Implement a Fourier feature neural network codec (#11)

* Implement a fourier feature neural network codec * Implement mini batching for the FourierNetwork codec * Bump MSRV to 1.81, begrudgingly * Add MAE, RMSE, and Linf metrics to epoch logs * Restore model with lowest loss after training * Test and fix FourierNetwork schema and config * Test RandomProjection schema and config * Reduce the CI test matrix size * Publish v0.1.0 of numcodecs-fourier-network
juntyr · Dec 11, 2024 · 9acf23c · 9acf23c
1 parent c11055e
commit 9acf23c
Show file tree

Hide file tree

Showing 53 changed files with 1,325 additions and 50 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -52,10 +52,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        rust: ["1.77", stable, nightly]
+        rust: ["1.81", stable, nightly]
         lock: ["Cargo.lock", "Cargo.lock.min"]
         include:
-          - rust: "1.77"
+          - rust: "1.81"
             wasm32-wasip1: wasm32-wasi
           - rust: stable
             wasm32-wasip1: wasm32-wasip1
@@ -108,7 +108,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        rust: ["1.77", stable, nightly]
+        rust: [stable]
         lock: ["Cargo.lock", "Cargo.lock.min"]
         python: ["3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.os }}
@@ -175,10 +175,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        rust: ["1.77", stable]
+        rust: ["1.81", stable]
         lock: ["Cargo.lock", "Cargo.lock.min"]
         include:
-          - rust: "1.77"
+          - rust: "1.81"
             wasm32-wasip1: wasm32-wasi
           - rust: stable
             wasm32-wasip1: wasm32-wasip1

diff --git a/Cargo.toml b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
     "codecs/asinh",
     "codecs/bit-round",
     "codecs/fixed-offset-scale",
+    "codecs/fourier-network",
     "codecs/identity",
     "codecs/linear-quantize",
     "codecs/log",
@@ -27,7 +28,7 @@ edition = "2021"
 authors = ["Juniper Tyree <[email protected]>"]
 repository = "https://github.com/juntyr/numcodecs-rs"
 license = "MPL-2.0"
-rust-version = "1.77"
+rust-version = "1.81"
 
 [workspace.dependencies]
 # workspace-internal numcodecs crates
@@ -39,6 +40,7 @@ numcodecs-wasm-guest = { version = "0.2", path = "crates/numcodecs-wasm-guest",
 numcodecs-asinh = { version = "0.2", path = "codecs/asinh", default-features = false }
 numcodecs-bit-round = { version = "0.2", path = "codecs/bit-round", default-features = false }
 numcodecs-fixed-offset-scale = { version = "0.2", path = "codecs/fixed-offset-scale", default-features = false }
+numcodecs-fourier-network = { version = "0.1", path = "codecs/fourier-network", default-features = false }
 numcodecs-identity = { version = "0.2", path = "codecs/identity", default-features = false }
 numcodecs-linear-quantize = { version = "0.3", path = "codecs/linear-quantize", default-features = false }
 numcodecs-log = { version = "0.3", path = "codecs/log", default-features = false }
@@ -53,8 +55,12 @@ numcodecs-zlib = { version = "0.2", path = "codecs/zlib", default-features = fal
 numcodecs-zstd = { version = "0.2", path = "codecs/zstd", default-features = false }
 
 # crates.io third-party dependencies
+burn = { version = "0.15", default-features = false }
 convert_case = { version = "0.6", default-features = false }
 format_serde_error = { version = "0.3", default-features = false }
+itertools = { version = "0.13", default-features = false }
+log = { version = "0.4.22", default-features = false }
+simple_logger = { version = "5.0", default-features = false }
 miniz_oxide = { version = "0.8", default-features = false }
 ndarray = { version = "0.16", default-features = false } # keep in sync with numpy
 ndarray-rand = { version = "0.15", default-features = false }
@@ -64,14 +70,14 @@ postcard = { version = "1.0.8", default-features = false }
 pyo3 = { version = "0.23.1", default-features = false }
 pyo3-error = { version = "0.3", default-features = false }
 pythonize = { version = "0.23", default-features = false }
-rand = { version = "0.8", default-features = false }
+rand = { version = "0.8.5", default-features = false }
 schemars = { version = "=1.0.0-alpha.15", default-features = false }
-serde = { version = "1.0.194", default-features = false }
+serde = { version = "1.0.210", default-features = false }
 serde-transcode = { version = "1.1", default-features = false }
-serde_json = { version = "1.0.127", default-features = false }
+serde_json = { version = "1.0.132", default-features = false }
 serde_repr = { version = "0.1.3", default-features = false }
 sz3 = { version = "0.1.1", default-features = false }
-thiserror = { version = "1.0.61", default-features = false }
+thiserror = { version = "1.0.62", default-features = false }
 twofloat = { version = "0.7", default-features = false }
 wit-bindgen = { version = "0.34", default-features = false }
 wyhash = { version = "0.5", default-features = false }

diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
 [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
 
-[MSRV]: https://img.shields.io/badge/MSRV-1.77.0-blue
+[MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
 [repo]: https://github.com/juntyr/numcodecs-rs
 
 [Latest Version]: https://img.shields.io/crates/v/numcodecs

diff --git a/codecs/asinh/README.md b/codecs/asinh/README.md
@@ -3,7 +3,7 @@
 [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
 [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
 
-[MSRV]: https://img.shields.io/badge/MSRV-1.77.0-blue
+[MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
 [repo]: https://github.com/juntyr/numcodecs-rs
 
 [Latest Version]: https://img.shields.io/crates/v/numcodecs-asinh

diff --git a/codecs/asinh/src/lib.rs b/codecs/asinh/src/lib.rs
@@ -3,7 +3,7 @@
 //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
 //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
 //!
-//! [MSRV]: https://img.shields.io/badge/MSRV-1.77.0-blue
+//! [MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
 //! [repo]: https://github.com/juntyr/numcodecs-rs
 //!
 //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-asinh

diff --git a/codecs/bit-round/README.md b/codecs/bit-round/README.md
@@ -3,7 +3,7 @@
 [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
 [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
 
-[MSRV]: https://img.shields.io/badge/MSRV-1.77.0-blue
+[MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
 [repo]: https://github.com/juntyr/numcodecs-rs
 
 [Latest Version]: https://img.shields.io/crates/v/numcodecs-bit-round

diff --git a/codecs/bit-round/src/lib.rs b/codecs/bit-round/src/lib.rs
@@ -3,7 +3,7 @@
 //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
 //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
 //!
-//! [MSRV]: https://img.shields.io/badge/MSRV-1.77.0-blue
+//! [MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
 //! [repo]: https://github.com/juntyr/numcodecs-rs
 //!
 //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-bit-round

diff --git a/codecs/fixed-offset-scale/README.md b/codecs/fixed-offset-scale/README.md
@@ -3,7 +3,7 @@
 [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
 [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
 
-[MSRV]: https://img.shields.io/badge/MSRV-1.77.0-blue
+[MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
 [repo]: https://github.com/juntyr/numcodecs-rs
 
 [Latest Version]: https://img.shields.io/crates/v/numcodecs-fixed-offset-scale

diff --git a/codecs/fixed-offset-scale/src/lib.rs b/codecs/fixed-offset-scale/src/lib.rs
@@ -3,7 +3,7 @@
 //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
 //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
 //!
-//! [MSRV]: https://img.shields.io/badge/MSRV-1.77.0-blue
+//! [MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
 //! [repo]: https://github.com/juntyr/numcodecs-rs
 //!
 //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-fixed-offset-scale

diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+name = "numcodecs-fourier-network"
+version = "0.1.0"
+edition = { workspace = true }
+authors = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+rust-version = { workspace = true }
+
+description = "Fourier feature neural network codec implementation for the numcodecs API"
+readme = "README.md"
+categories = ["compression", "encoding"]
+keywords = ["fourier", "network", "numcodecs", "compression", "encoding"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+burn = { workspace = true, features = ["std", "autodiff", "ndarray"] }
+itertools = { workspace = true, features = ["use_alloc"] }
+log = { workspace = true }
+ndarray = { workspace = true, features = ["std"] }
+numcodecs = { workspace = true }
+num-traits = { workspace = true, features = ["std"] }
+schemars = { workspace = true, features = ["derive", "preserve_order"] }
+serde = { workspace = true, features = ["std", "derive"] }
+thiserror = { workspace = true }
+
+[dev-dependencies]
+serde_json = { workspace = true, features = ["std"] }
+simple_logger = { workspace = true }
+
+[lints]
+workspace = true
diff --git a/codecs/fourier-network/LICENSE b/codecs/fourier-network/LICENSE
@@ -0,0 +1 @@
+../../LICENSE
diff --git a/codecs/fourier-network/README.md b/codecs/fourier-network/README.md
@@ -0,0 +1,32 @@
+[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs]
+
+[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main
+[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain
+
+[MSRV]: https://img.shields.io/badge/MSRV-1.81.0-blue
+[repo]: https://github.com/juntyr/numcodecs-rs
+
+[Latest Version]: https://img.shields.io/crates/v/numcodecs-fourier-network
+[crates.io]: https://crates.io/crates/numcodecs-fourier-network
+
+[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-fourier-network
+[docs.rs]: https://docs.rs/numcodecs-fourier-network/
+
+[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue
+[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_fourier_network
+
+# numcodecs-fourier-network
+
+Fourier feature neural network codec implementation for the [`numcodecs`] API.
+
+[`numcodecs`]: https://docs.rs/numcodecs/0.1/numcodecs/
+
+## License
+
+Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/).
+
+## Funding
+
+The `numcodecs-fourier-network` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe.
+
+Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054.