Rework save system to use ZarrV3

ext/Cargo.toml

@@ -23,6 +23,8 @@ sseq = { path = "crates/sseq", default-features = false }
 
 adler = "1"
 anyhow = "1.0.98"
+bitcode = { version = "0.6", features = ["serde"] }
+serde = { version = "1", features = ["derive"] }
 byteorder = "1.5.0"
 dashmap = "6.1.0"
 itertools = { version = "0.14.0", default-features = false, features = [
@@ -35,6 +37,20 @@ tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
 
 zstd = { version = "0.13.3", optional = true }
 
+# Zarrs is declared twice with target-specific feature sets. Where there is a real OS
+# (`cfg(any(windows, unix))`) we enable `filesystem` so the save system persists to disk; on
+# every other target (in practice `wasm32-unknown-unknown`, the web frontend) we fall back to an
+# in-memory store (see the `platform` modules in `src/save.rs`). The gate is `any(windows, unix)`,
+# not `not(target_arch = "wasm32")`, because that's exactly the `cfg` that gates the `filesystem`
+# feature's `positioned-io::RandomAccessFile`; the `zstd` feature's `zstd-sys` C build (which
+# expects POSIX symbols like `qsort_r` that the bare-wasm libc shim doesn't expose) builds under
+# the same condition. So `wasm32-unknown-emscripten` (which is `unix`) keeps both features.
+[target.'cfg(any(windows, unix))'.dependencies]
+zarrs = { version = "0.23", default-features = false, features = ["filesystem", "sharding", "crc32c", "zstd"] }
+
+[target.'cfg(not(any(windows, unix)))'.dependencies]
+zarrs = { version = "0.23", default-features = false, features = ["sharding", "crc32c"] }
+
 [target.'cfg(unix)'.dependencies]
 ctrlc = { version = "3", features = ["termination"] }
 

ext/crates/fp/src/matrix/quasi_inverse.rs

@@ -1,13 +1,12 @@
 use std::io;
 
 use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
-use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
 use super::Matrix;
 use crate::{
     prime::ValidPrime,
-    vector::{FpSlice, FpSliceMut, FpVector},
+    vector::{FpSlice, FpSliceMut},
 };
 
 /// Given a matrix M, a quasi-inverse Q is a map from the co-domain to the domain such that xQM = x
@@ -75,44 +74,6 @@ impl QuasiInverse {
         })
     }
 
-    /// Given a data file containing a quasi-inverse, apply it to all the vectors in `input`
-    /// and write the results to the corresponding vectors in `results`. This reads in the
-    /// quasi-inverse row by row to minimize memory usage.
-    pub fn stream_quasi_inverse<T, S>(
-        p: ValidPrime,
-        data: &mut impl io::Read,
-        results: &mut [T],
-        inputs: &[S],
-    ) -> io::Result<()>
-    where
-        for<'a> &'a mut T: Into<FpSliceMut<'a>>,
-        for<'a> &'a S: Into<FpSlice<'a>>,
-    {
-        let source_dim = data.read_u64::<LittleEndian>()? as usize;
-        let target_dim = data.read_u64::<LittleEndian>()? as usize;
-        let _image_dim = data.read_u64::<LittleEndian>()? as usize;
-
-        let image = Matrix::read_pivot(target_dim, data)?;
-        let mut v = FpVector::new(p, source_dim);
-
-        assert_eq!(results.len(), inputs.len());
-        for result in &mut *results {
-            assert_eq!(result.into().as_slice().len(), source_dim);
-        }
-
-        for (i, r) in image.into_iter().enumerate() {
-            if r < 0 {
-                continue;
-            }
-
-            v.update_from_bytes(data)?;
-            for (input, result) in inputs.iter().zip_eq(&mut *results) {
-                result.into().add(v.as_slice(), input.into().entry(i));
-            }
-        }
-        Ok(())
-    }
-
     pub fn preimage(&self) -> &Matrix {
         &self.preimage
     }
@@ -148,53 +109,3 @@ impl QuasiInverse {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_stream_qi() {
-        let p = ValidPrime::new(2);
-        let qi = QuasiInverse {
-            image: Some(vec![0, -1, 1, -1, 2, 3]),
-            preimage: Matrix::from_vec(
-                p,
-                &[
-                    vec![1, 0, 1, 1],
-                    vec![1, 1, 0, 0],
-                    vec![0, 1, 0, 1],
-                    vec![1, 1, 1, 0],
-                ],
-            ),
-        };
-        let v0 = FpVector::from_slice(p, &[1, 1, 0, 0, 1, 0]);
-        let v1 = FpVector::from_slice(p, &[0, 0, 1, 0, 1, 1]);
-
-        let mut out0 = FpVector::new(p, 4);
-        let mut out1 = FpVector::new(p, 4);
-
-        let mut cursor = io::Cursor::new(Vec::<u8>::new());
-        qi.to_bytes(&mut cursor).unwrap();
-        cursor.set_position(0);
-
-        QuasiInverse::stream_quasi_inverse(
-            p,
-            &mut cursor,
-            &mut [out0.as_slice_mut(), out1.as_slice_mut()],
-            &[v0.as_slice(), v1.as_slice()],
-        )
-        .unwrap();
-
-        let mut bench0 = FpVector::new(p, 4);
-        let mut bench1 = FpVector::new(p, 4);
-
-        qi.apply(bench0.as_slice_mut(), 1, v0.as_slice());
-        qi.apply(bench1.as_slice_mut(), 1, v1.as_slice());
-
-        assert_eq!(out0, bench0, "{out0} != {bench0}");
-        assert_eq!(out1, bench1, "{out1} != {bench1}");
-
-        assert_eq!(cursor.position() as usize, cursor.get_ref().len());
-    }
-}

ext/crates/fp/src/vector/fp_wrapper/mod.rs

@@ -112,8 +112,11 @@ impl FpVector {
         v
     }
 
-    // Convenient for some matrix methods
-    pub(crate) fn num_limbs(p: ValidPrime, len: usize) -> usize {
+    /// The number of `Limb`s required to store an [`FpVector`] of length
+    /// `len` over `F_p`. Useful for sizing a buffer that holds the
+    /// little-endian byte serialization of an `FpVector` (each limb is 8
+    /// bytes).
+    pub fn num_limbs(p: ValidPrime, len: usize) -> usize {
         Fp::new(p).number(len)
     }
 

ext/src/chain_complex/chain_homotopy.rs

@@ -31,6 +31,10 @@ pub struct ChainHomotopy<
     lock: Mutex<()>,
     /// Homotopies, indexed by the filtration of the target of f - g.
     homotopies: OnceBiVec<Arc<FreeModuleHomomorphism<U::Module>>>,
+    /// Save directory for this homotopy and any secondary lift built from it.
+    /// Set to a `homotopies/{left}__{right}` subgroup of the source's save_dir
+    /// when both `left` and `right` are named, `None` otherwise so that
+    /// homotopies between anonymous homs don't pollute the store.
     save_dir: SaveDirectory,
 }
 
@@ -44,23 +48,19 @@ impl<
         left: Arc<ResolutionHomomorphism<S, T>>,
         right: Arc<ResolutionHomomorphism<T, U>>,
     ) -> Self {
-        let save_dir = if left.source.save_dir().is_some()
-            && !left.name().is_empty()
+        assert!(Arc::ptr_eq(&left.target, &right.source));
+        let save_dir = if !left.name().is_empty()
             && !right.name().is_empty()
+            && let Some(parent) = left.source.save_dir().store()
         {
-            let mut save_dir = left.source.save_dir().clone();
-            save_dir.push(format!("massey/{},{}/", left.name(), right.name()));
-
-            SaveKind::ChainHomotopy
-                .create_dir(save_dir.write().unwrap())
-                .unwrap();
-
-            save_dir
+            SaveDirectory::Store(
+                parent
+                    .subgroup(&format!("homotopies/{}__{}", left.name(), right.name()))
+                    .expect("Failed to create homotopies subgroup"),
+            )
         } else {
             SaveDirectory::None
         };
-
-        assert!(Arc::ptr_eq(&left.target, &right.source));
         Self {
             homotopies: OnceBiVec::new((left.shift + right.shift).s() - 1),
             left,
@@ -178,16 +178,13 @@ impl<
             return self.homotopies[source.s()].add_generators_from_rows_ooo(source.t(), outputs);
         }
 
-        if let Some(dir) = self.save_dir.read()
-            && let Some(mut f) = self
-                .left
-                .source
-                .save_file(SaveKind::ChainHomotopy, source)
-                .open_file(dir.to_owned())
+        if let Some(store) = self.save_dir.store()
+            && let Some(data) = store.read(SaveKind::ChainHomotopy, source).unwrap()
         {
+            let mut cursor = &data[..];
             let mut outputs = Vec::with_capacity(num_gens);
             for _ in 0..num_gens {
-                outputs.push(FpVector::from_bytes(p, target_dim, &mut f).unwrap());
+                outputs.push(FpVector::from_bytes(p, target_dim, &mut cursor).unwrap());
             }
             return self.homotopies[source.s()].add_generators_from_rows_ooo(source.t(), outputs);
         }
@@ -263,15 +260,12 @@ impl<
             &scratches,
         ));
 
-        if let Some(dir) = self.save_dir.write() {
-            let mut f = self
-                .left
-                .source
-                .save_file(SaveKind::ChainHomotopy, source)
-                .create_file(dir.to_owned(), false);
+        if let Some(store) = self.save_dir.store() {
+            let mut buf = Vec::new();
             for row in &outputs {
-                row.to_bytes(&mut f).unwrap();
+                row.to_bytes(&mut buf).unwrap();
             }
+            store.write(SaveKind::ChainHomotopy, source, &buf).unwrap();
         }
         self.homotopies[source.s()].add_generators_from_rows_ooo(source.t(), outputs)
     }
@@ -306,7 +300,7 @@ pub(crate) mod secondary {
     use crate::{
         chain_complex::FreeChainComplex,
         resolution_homomorphism::ResolutionHomomorphism,
-        save::{SaveDirectory, SaveKind},
+        save::SaveDirectory,
         secondary::{
             CompositeData, LAMBDA_BIDEGREE, SecondaryHomotopy, SecondaryLift,
             SecondaryResolutionHomomorphism,
@@ -570,12 +564,6 @@ pub(crate) mod secondary {
                 );
             }
 
-            if let Some(p) = underlying.save_dir().write() {
-                for subdir in SaveKind::secondary_data() {
-                    subdir.create_dir(p).unwrap();
-                }
-            }
-
             Self {
                 left,
                 right,

ext/src/chain_complex/mod.rs

@@ -262,20 +262,6 @@ pub trait ChainComplex: Send + Sync {
     fn save_dir(&self) -> &SaveDirectory {
         &SaveDirectory::None
     }
-
-    /// Get the save file of a bidegree
-    fn save_file(
-        &self,
-        kind: crate::save::SaveKind,
-        b: Bidegree,
-    ) -> crate::save::SaveFile<Self::Algebra> {
-        crate::save::SaveFile {
-            algebra: self.algebra(),
-            kind,
-            b,
-            idx: None,
-        }
-    }
 }
 
 /// An iterator returned by [`ChainComplex::iter_stem`]