diff --git a/404.html b/404.html new file mode 100644 index 0000000..85264bf --- /dev/null +++ b/404.html @@ -0,0 +1,8 @@ +--- +title: Page not found +--- + +

+ The page you were looking for does not exist. You might want to + go back home. +

diff --git a/README.md b/README.md new file mode 100644 index 0000000..7fb4d14 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Personal blog + +## Build and deploy + +Uses [Cloudflare Pages direct upload](https://developers.cloudflare.com/pages/get-started/direct-upload/). + +```sh +cabal run site build +npx wrangler login # Only needed once +npx wrangler pages deploy +``` diff --git a/README.org b/README.org deleted file mode 100644 index ce4b5ee..0000000 --- a/README.org +++ /dev/null @@ -1,16 +0,0 @@ -* Personal blog - -** Setup - -#+begin_src sh - npm install netlify-cli -g - netlify login # login to Netlify - netlify link # link the directory to the site -#+end_src - -** Deploy - -#+begin_src sh - # deploy, omit "--prod" if you want to preview - netlify deploy --dir=_site --prod -#+end_src diff --git a/android-chrome-192x192.png b/android-chrome-192x192.png deleted file mode 100644 index 220bf96..0000000 Binary files a/android-chrome-192x192.png and /dev/null differ diff --git a/android-chrome-512x512.png b/android-chrome-512x512.png deleted file mode 100644 index d8bf98a..0000000 Binary files a/android-chrome-512x512.png and /dev/null differ diff --git a/apple-touch-icon.png b/apple-touch-icon.png index 53e0531..5b02c6a 100644 Binary files a/apple-touch-icon.png and b/apple-touch-icon.png differ diff --git a/bib/bibliography.bib b/bib/bibliography.bib index d16f503..885be7d 100644 --- a/bib/bibliography.bib +++ b/bib/bibliography.bib @@ -740,3 +740,49 @@ volume = 21, year = 1999, } + +@misc{liu2024_kan, + author = {Liu, Ziming and Wang, Yixuan and Vaidya, Sachin and + Ruehle, Fabian and Halverson, James and Soljačić, + Marin and Hou, Thomas Y. and Tegmark, Max}, + title = {{KAN}: {Kolmogorov}-{Arnold} {Networks}}, + year = 2024, + month = may, + publisher = {arXiv}, + doi = {10.48550/arXiv.2404.19756}, + url = {http://arxiv.org/abs/2404.19756}, + note = {arXiv:2404.19756 [cond-mat, stat]}, + keywords = {Computer Science - Machine Learning, Condensed + Matter - Disordered Systems and Neural Networks, + Computer Science - Artificial Intelligence, + Statistics - Machine Learning}, +} + +@article{chenNeuralOrdinaryDifferential2018, + archivePrefix = {arXiv}, + eprinttype = {arxiv}, + eprint = {1806.07366}, + primaryClass = {cs, stat}, + title = {Neural {{Ordinary Differential Equations}}}, + url = {http://arxiv.org/abs/1806.07366}, + abstract = {We introduce a new family of deep neural network models. Instead of specifying a discrete sequence of hidden layers, we parameterize the derivative of the hidden state using a neural network. The output of the network is computed using a black-box differential equation solver. These continuous-depth models have constant memory cost, adapt their evaluation strategy to each input, and can explicitly trade numerical precision for speed. We demonstrate these properties in continuous-depth residual networks and continuous-time latent variable models. We also construct continuous normalizing flows, a generative model that can train by maximum likelihood, without partitioning or ordering the data dimensions. For training, we show how to scalably backpropagate through any ODE solver, without access to its internal operations. This allows end-to-end training of ODEs within larger models.}, + urldate = {2019-01-05}, + date = {2018-06-19}, + keywords = {Statistics - Machine Learning,Computer Science - Artificial Intelligence,Computer Science - Machine Learning}, + author = {Chen, Ricky T. Q. and Rubanova, Yulia and Bettencourt, Jesse and Duvenaud, David}, + file = {/home/dimitri/Nextcloud/Zotero/storage/26D4Y3GG/Chen et al. - 2018 - Neural Ordinary Differential Equations.pdf;/home/dimitri/Nextcloud/Zotero/storage/RNXT4EQV/1806.html} +} + +@article{ruthotto2024_differ_equat, + author = {Ruthotto, Lars}, + title = {Differential {Equations} for {Continuous}-{Time} + {Deep} {Learning}}, + journal = {Notices of the American Mathematical Society}, + year = 2024, + month = may, + volume = 71, + number = 05, + issn = {0002-9920, 1088-9477}, + doi = {10.1090/noti2930}, + url = {https://www.ams.org/notices/202405/rnoti-p613.pdf}, +} diff --git a/favicon-16x16.png b/favicon-16x16.png deleted file mode 100644 index 2ee434e..0000000 Binary files a/favicon-16x16.png and /dev/null differ diff --git a/favicon-32x32.png b/favicon-32x32.png deleted file mode 100644 index adec1fb..0000000 Binary files a/favicon-32x32.png and /dev/null differ diff --git a/favicon-96x96.png b/favicon-96x96.png new file mode 100644 index 0000000..3550660 Binary files /dev/null and b/favicon-96x96.png differ diff --git a/favicon.ico b/favicon.ico index 616702d..2c2a54e 100644 Binary files a/favicon.ico and b/favicon.ico differ diff --git a/favicon.svg b/favicon.svg new file mode 100644 index 0000000..bfc45c4 --- /dev/null +++ b/favicon.svg @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/index.html b/index.html index 3c151b9..967fedb 100644 --- a/index.html +++ b/index.html @@ -16,17 +16,17 @@ title: Dimitri Lozeve />

- Hi! I am an Engineering Manager - at Doctrine, where we develop - productivity tools to help lawyers manage their data. + Hi! I am an Engineering Manager at + Doctrine, where we develop productivity + tools to help lawyers analyze and write their private documents.

- I was previously a Research Engineer and Project Manager - at Sysnav, where I worked on - developing algorithms to help physicians measure the progress of - neuromuscular diseases. We used data from inertial sensors to - extract meaningful variables for pharmaceutical studies. + I was previously a Research Engineer and Project Manager at + Sysnav, where I worked on developing + algorithms to help physicians measure the progress of neuromuscular diseases. + We used data from inertial sensors to extract meaningful variables for + pharmaceutical studies.

@@ -41,10 +41,10 @@ title: Dimitri Lozeve

- You can contact me by email (at firstname@lastname.com), - reach me via Mastodon, - find me on LinkedIn, - or see what I do on GitHub. + You can contact me by email (at firstname@lastname.com), reach me via + Mastodon, find me on + LinkedIn, or see what + I do on GitHub.

Recent Posts

diff --git a/posts/kolmogorov-arnold-networks.org b/posts/kolmogorov-arnold-networks.org new file mode 100644 index 0000000..0d0c6cf --- /dev/null +++ b/posts/kolmogorov-arnold-networks.org @@ -0,0 +1,81 @@ +--- +title: "Reading notes: Kolmogorov-Arnold Networks" +date: 2024-06-08 +tags: machine learning, paper +toc: false +--- + +This paper [cite:@liu2024_kan] proposes an alternative to multi-layer +perceptrons (MLPs) in machine learning. + +The basic idea is that MLPs have parameters on the nodes of the +computation graph (the weights and biases on each cell), and that KANs +have the parameters on the edges. Each edge has a learnable activation +function parameterized as a spline. + +The network is learned at two levels, which allows for "adjusting +locally": +- the overall shape of the computation graph and its connexions + (external degrees of freedom, to learn the compositional structure), +- the parameters of each activation function (internal degrees of + freedom). + +It is based on the [[https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Arnold_representation_theorem][Kolmogorov-Arnold representation theorem]], which +says that any continuous multivariate function can be represented as a +sum of continuous univariate functions. We recover the distinction +between the compositional structure of the sum and the structure of +each internal univariate function. + +The theorem can be interpreted as two layers, and the paper then +generalizes it to multiple layer of arbitrary width. In the theorem, +the univariate functions are arbitrary and can be complex (even +fractal), so the hope is that allowing for arbitrary depth and width +will allow to only use splines. They derive an approximation theorem: +when replacing the arbitrary continuous functions of the +Kolmogorov-Arnold representation with splines, we can bound the error +independently of the dimension. (However there is a constant which +depends on the function and its representation, and therefore on the +dimension...) Theoretical scaling laws in the number of parameters are +much better than for MLPs, and moreover, experiments show that KANs +are much closer to their theoretical bounds than MLPs. + +KANs have interesting properties: +- The splines are interpolated on grid points which can be iteratively + refined. The fact that there is a notion of "fine-grainedness" is + very interesting, it allows to add parameters without having to + retrain everything. +- Larger is not always better: the quality of the reconstruction + depends on finding the optimal shape of the network, which should + match the structure of the function we want to approximate. Finding + this optimal shape is found via sparsification, pruning, and + regularization (non-trivial). +- We can have a "human in the loop" during training, guiding pruning, + and "symbolifying" some activations (i.e. by recognizing that an + activation function is actually a cos function, replace it + directly). This symbolic discovery can be guided by a symbolic + system recognizing some functions. It's therefore a mix of symbolic + regression and numerical regression. + +They test mostly with scientific applications in mind: reconstructing +equations from physics and pure maths. Conceptually, it has a lot of +overlap with Neural Differential Equations +[cite:@chenNeuralOrdinaryDifferential2018;@ruthotto2024_differ_equat] +and "scientific ML" in general. + +There is an interesting discussion at the end about KANs as the model +of choice for the "language of science". The idea is that LLMs are + important because they are useful for natural language, and KANs +could fill the same role for the language of functions. The +interpretability and adaptability (being able to be manipulated and +guided during training by a domain expert) is thus a core feature that +traditional deep learning models lack. + +There are still challenges, mostly it's unclear how it performs on +other types of data and other modalities, but it is very +encouraging. There is also a computational challenges, they are +obviously much slower to train, but there has been almost no +engineering work on them to optimize this, so it's expected. The fact +that the operations are not easily batchable (compared to matrix +multiplication) is however worrying for scalability to large networks. + +* References diff --git a/projects.org b/projects.org index fe62fda..c9bd6d3 100644 --- a/projects.org +++ b/projects.org @@ -106,6 +106,7 @@ was gathered using a Software Defined Radio to listen to aircrafts. - My submission [[https://github.com/dlozeve/apl-competition-2018][on GitHub]] - Nominated as a [[https://www.dyalog.com/nnews/128/456/Winners-Announced-for-the-2018-APL-Programming-Contest.htm][Phase I winner]] - [[https://adventofcode.com/][Advent of code]] + - [[https://git.sr.ht/~dlozeve/advent-of-code/tree/main/item/2024][2024]] (BQN) - [[https://github.com/dlozeve/aoc2023][2023]] (BQN) - [[https://github.com/dlozeve/aoc2022][2022]] (mostly BQN) - [[https://github.com/dlozeve/aoc2021][2021]] (mostly Chicken Scheme) diff --git a/site.hs b/site.hs index 5bc941b..1de646a 100644 --- a/site.hs +++ b/site.hs @@ -18,7 +18,7 @@ main = hakyll $ do route idRoute compile copyFileCompiler - match "favicon.ico" $ do + match (fromList ["favicon.ico", "favicon.svg", "favicon-96x96.png", "web-app-manifest-192x192.png", "web-app-manifest-512x512.png", "apple-touch-icon.png", "site.webmanifest"]) $ do route idRoute compile copyFileCompiler @@ -112,6 +112,12 @@ main = hakyll $ do match "templates/*" $ compile templateBodyCompiler + match "404.html" $ do + route idRoute + compile $ + customPandocCompiler False + >>= loadAndApplyTemplate "templates/default.html" defaultContext + -------------------------------------------------------------------------------- feedConfiguration :: FeedConfiguration feedConfiguration = diff --git a/site.webmanifest b/site.webmanifest index 45dc8a2..b17f5e5 100644 --- a/site.webmanifest +++ b/site.webmanifest @@ -1 +1,21 @@ -{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} \ No newline at end of file +{ + "name": "Dimitri Lozeve", + "short_name": "Dimitri Lozeve", + "icons": [ + { + "src": "/web-app-manifest-192x192.png", + "sizes": "192x192", + "type": "image/png", + "purpose": "maskable" + }, + { + "src": "/web-app-manifest-512x512.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "maskable" + } + ], + "theme_color": "#ffffff", + "background_color": "#ffffff", + "display": "standalone" +} diff --git a/templates/default.html b/templates/default.html index fd0b0fe..044fb2c 100644 --- a/templates/default.html +++ b/templates/default.html @@ -1,31 +1,54 @@ - - - + + + - - - - + + + + + - - $if(keywords)$ - - $endif$ Dimitri Lozeve - $title$ + + + + + $if(keywords)$ + + $endif$ + + - + + - - +
@@ -33,31 +56,25 @@ $if(title)$ -

$title$

- $if(subtitle)$ -

$subtitle$

- $endif$ - $if(date)$ -

$date$$if(author)$ – $author$$endif$

- $else$ - $if(author)$ -

$author$

- $endif$ - $endif$ - $if(tags)$ -

Tags: $tags$

- $endif$ - $endif$ +

$title$

+ $if(subtitle)$ +

$subtitle$

+ $endif$ $if(date)$ +

$date$$if(author)$ – $author$$endif$

+ $else$ $if(author)$ +

$author$

+ $endif$ $endif$ $if(tags)$ +

Tags: $tags$

+ $endif$ $endif$ $body$ -