feat(core): enter CONVERT and move the cursor through the conversion session (#445)

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- **R-15 Gate 4D:** the conversion session enters `CONVERT` and moves the checkpoint cursor (part b-2 of the
+  owner's steps through the gate). `ConversionSession::enter_convert` (idempotent: a session already in
+  `CONVERT` writes nothing) and `advance_cursor` (forward only, inside the inventory, in `CONVERT` only) share
+  one private step with `renew_lease`, which now commits through `commit_lease_renewal_held` or the new
+  `commit_journal_checkpoint_held`, and every outcome after a landed commit is followed by the admission
+  check. The cursor-driven iteration is the next part. PR #1015.
 - **R-15 Gate 4D:** the conversion session renews its lease (part b-1 of the owner's steps through the
   gate). `ConversionSession::renew_lease` builds the renewal from the session's own snapshot (`renewed_lease`),
   takes the root event through the held admission, commits it with the key route and active epoch the

crates/worldscript-secure-storage/src/authority.rs

@@ -362,6 +362,20 @@ pub fn commit_journal_checkpoint<F: DurableFs, P: KeyProvider>(
     check_operation_id(&checkpoint.manifest.operation_id)
         .map_err(|_| AuthorityError::InvalidOperationId)?;
     let held = acquire_root_commit(layout)?;
+    commit_journal_checkpoint_held(fs, provider, layout, checkpoint, &held)
+}
+
+/// As [`commit_journal_checkpoint`], under a root event the caller already holds, as
+/// [`commit_lease_renewal_held`] is for the renewal. `held` must guard the root of `layout`.
+pub(crate) fn commit_journal_checkpoint_held<F: DurableFs, P: KeyProvider>(
+    fs: &mut F,
+    provider: &mut P,
+    layout: RootLayout<'_>,
+    checkpoint: JournalCheckpoint<'_>,
+    held: &RootCommitGuard,
+) -> Result<RootCommitted, AuthorityError> {
+    check_operation_id(&checkpoint.manifest.operation_id)
+        .map_err(|_| AuthorityError::InvalidOperationId)?;
     let commit = journal_catalog_commit(&checkpoint);
     let committed = committed_binding(fs, provider, layout, commit)?;
     let key = route_journal_key(fs, provider, layout, checkpoint.journal)?;
@@ -382,7 +396,7 @@ pub fn commit_journal_checkpoint<F: DurableFs, P: KeyProvider>(
         provider,
         layout,
         commit,
-        &held,
+        held,
         Some(JournalStep {
             binding: BindingStep::Checkpoint(&advance),
             key: &key,

crates/worldscript-secure-storage/src/conversion.rs

@@ -31,8 +31,9 @@
 //! still the caller's while nobody has taken over (the fence arbitrates).
 //!
 //! Beginning writes nothing and holds no key. The session then moves the journal only through the
-//! fenced journal-owner operations, one step at a time ([`ConversionSession::renew_lease`] is the
-//! first). A step checks the admission and the journal directory, builds the successor from the
+//! fenced journal-owner operations, one step at a time ([`ConversionSession::renew_lease`],
+//! [`ConversionSession::enter_convert`] and [`ConversionSession::advance_cursor`], which share one
+//! private step). A step checks the admission and the journal directory, builds the successor from the
 //! session's own snapshot, takes the root event through the held admission (so the identity check and
 //! the root lock are coupled), commits with the key route and active epoch the committed root itself
 //! names, reads the committed journal back under that same root event and checks the admission again.
@@ -47,22 +48,24 @@
 //! it names the successor (committed, even if the commit reported an error), or it does not (not
 //! committed, snapshot kept). If the read-back fails, or names a journal that is neither of the two
 //! where the commit reported success, the session is spent: every later step is refused and the caller
-//! begins again to learn the state. A lost admission is reported even when it was lost after the
-//! commit; the snapshot then is the committed journal.
+//! begins again to learn the state. After the commit call every outcome is followed by the admission
+//! check, and a lost admission is reported first: the step may have committed or left a candidate, and
+//! the caller begins again to learn the state.
 
 use std::fs::File;
 use std::path::{Path, PathBuf};
 
 use crate::admission::{AdmissionError, AdmissionScope, ExclusiveAdmissionGuard};
 use crate::authority::{
-    commit_lease_renewal_held, read_committed_journal, AuthorityError, CommittedJournal,
-    JournalCheckpoint, JournalSource,
+    commit_journal_checkpoint_held, commit_lease_renewal_held, read_committed_journal,
+    AuthorityError, CommittedJournal, JournalCheckpoint, JournalSource,
 };
 use crate::durable::{DurableFs, WriteOperationId};
 use crate::error::SealError;
 use crate::journal::{
-    phase_code, renewed_lease, CandidateConflict, JournalManifest, MigrationExecutionError,
-    MigrationFence,
+    checkpoint_progress, phase_code, renewed_lease, transition_phase, CandidateConflict,
+    JournalCheckpointCursor, JournalManifest, MigrationExecutionError, MigrationFence,
+    MigrationPhase,
 };
 use crate::provider::KeyProvider;
 use crate::root::LiveMigration;
@@ -106,13 +109,15 @@ pub enum ConversionError {
     FinalInventoryNotCaptured,
     /// The committed lease is owned by another owner, or by none.
     ForeignLeaseOwner,
-    /// A step's successor is not a valid one (a renewal with no lease to renew, or an expiry that does
-    /// not move strictly forward).
+    /// A step's successor is not a valid one (a renewal with no lease to renew, an expiry that does not
+    /// move strictly forward, a cursor that regresses or lies outside the inventory).
     Migration(MigrationExecutionError),
+    /// The step belongs to another phase: the cursor moves in `CONVERT` only.
+    WrongPhase,
     /// No write-operation identifier could be drawn from the operating system.
     OperationId(SealError),
-    /// The step committed, but the journal read back is not the one it committed: the session is
-    /// spent, begin again.
+    /// The committed journal is not the one this session holds (a step's read-back named another
+    /// journal, or another owner advanced it since): the session is spent, begin again.
     Superseded,
     /// The step committed, but the committed journal could not be read back: the session is spent,
     /// begin again.
@@ -167,6 +172,14 @@ impl PinnedDirectory {
     }
 }
 
+/// Which journal-owner operation commits a step's successor: the lease renewal and the ordinary
+/// checkpoint are different operations over the same root event.
+#[derive(Debug, Clone, Copy)]
+enum StepKind {
+    Renewal,
+    Checkpoint,
+}
+
 /// The exclusive conversion entry: the committed journal state, read under an admission that no
 /// ordinary operation can share.
 #[derive(Debug)]
@@ -276,22 +289,112 @@ impl<'a> ConversionSession<'a> {
     ///
     /// The expiry must move strictly forward (`Migration(InvalidLeaseRenewal)`); a lease long past its
     /// expiry is still renewed while nobody has taken over, because a takeover advances the fence and
-    /// the committed manifest then names another owner. The renewal is committed by
-    /// [`commit_lease_renewal_held`] under the root event the admission hands out, so a snapshot that
-    /// went stale is refused before any write. See the module documentation for what a failed step
+    /// the committed manifest then names another owner. See the module documentation for what a step
     /// leaves behind.
     pub fn renew_lease<F: DurableFs, P: KeyProvider>(
         &mut self,
         fs: &mut F,
         provider: &mut P,
         expires_unix_ms: u64,
     ) -> Result<RootCommitted, ConversionError> {
+        self.step(fs, provider, StepKind::Renewal, |manifest, fence| {
+            renewed_lease(manifest, fence, expires_unix_ms).map_err(ConversionError::Migration)
+        })
+    }
+
+    /// Enters `CONVERT` from `ADMIT`, at cursor `(0, 0)` (§10.3), and returns the root commit. A
+    /// session that already is in `CONVERT` (a resumed conversion) has nothing to do: nothing is
+    /// written and `None` is returned.
+    pub fn enter_convert<F: DurableFs, P: KeyProvider>(
+        &mut self,
+        fs: &mut F,
+        provider: &mut P,
+    ) -> Result<Option<RootCommitted>, ConversionError> {
+        self.ready()?;
+        if self.journal.manifest.phase == phase_code::CONVERT {
+            // Nothing to write, but the answer must not rest on a snapshot another owner has overtaken.
+            self.confirm_snapshot(fs, provider)?;
+            return Ok(None);
+        }
+        let convert = MigrationPhase::from_wire(phase_code::CONVERT);
+        self.step(fs, provider, StepKind::Checkpoint, |manifest, fence| {
+            transition_phase(manifest, fence, convert).map_err(ConversionError::Migration)
+        })
+        .map(Some)
+    }
+
+    /// Records durable progress in `CONVERT`: moves the checkpoint cursor to `cursor` and returns the
+    /// root commit. The cursor never moves backwards and stays inside the manifest's extent, its page
+    /// count and its total entry count (`Migration(RegressiveCheckpoint)` and the extent refusals,
+    /// before any write); an equal cursor is accepted and records a revision with the same cursor. The
+    /// manifest does not carry the entry count of a page, so this does not check the entry index
+    /// against the selected page: what the index means within a page is fixed by the iteration that
+    /// reads the pages. In any other phase the step is `WrongPhase`.
+    pub fn advance_cursor<F: DurableFs, P: KeyProvider>(
+        &mut self,
+        fs: &mut F,
+        provider: &mut P,
+        cursor: JournalCheckpointCursor,
+    ) -> Result<RootCommitted, ConversionError> {
+        self.step(fs, provider, StepKind::Checkpoint, |manifest, fence| {
+            if manifest.phase != phase_code::CONVERT {
+                return Err(ConversionError::WrongPhase);
+            }
+            checkpoint_progress(manifest, fence, cursor).map_err(ConversionError::Migration)
+        })
+    }
+
+    /// Requires the committed journal to be the one the session holds. If it is not (another owner
+    /// advanced it), the session is spent; if it cannot be read, nothing was written and the step may
+    /// be tried again.
+    fn confirm_snapshot<F: DurableFs, P: KeyProvider>(
+        &mut self,
+        fs: &mut F,
+        provider: &P,
+    ) -> Result<(), ConversionError> {
+        let operation = WriteOperationId::generate().map_err(ConversionError::OperationId)?;
+        let layout = RootLayout {
+            root_dir: &self.root,
+        };
+        let journal = JournalSource {
+            dir: &self.journal_pin.canonical,
+            operation: &operation,
+        };
+        match read_committed_journal(fs, provider, layout, journal) {
+            Ok(current) if current.manifest == self.journal.manifest => self.check_admitted(),
+            Ok(_) => {
+                self.spent = true;
+                Err(ConversionError::Superseded)
+            }
+            Err(error) => Err(ConversionError::Authority(error)),
+        }
+    }
+
+    /// Whether the session may take a step: not spent, and the admission and the pinned journal
+    /// directory still hold.
+    fn ready(&self) -> Result<(), ConversionError> {
         if self.spent {
             return Err(ConversionError::Spent);
         }
-        self.check_admitted()?;
-        let renewed = renewed_lease(&self.journal.manifest, &self.fence(), expires_unix_ms)
-            .map_err(ConversionError::Migration)?;
+        self.check_admitted()
+    }
+
+    /// The step shared by the public methods: `build` makes the successor from the session's snapshot
+    /// and `kind` says which journal-owner operation commits it. See the module documentation.
+    fn step<F, P, B>(
+        &mut self,
+        fs: &mut F,
+        provider: &mut P,
+        kind: StepKind,
+        build: B,
+    ) -> Result<RootCommitted, ConversionError>
+    where
+        F: DurableFs,
+        P: KeyProvider,
+        B: FnOnce(&JournalManifest, &MigrationFence) -> Result<JournalManifest, ConversionError>,
+    {
+        self.ready()?;
+        let successor = build(&self.journal.manifest, &self.fence())?;
         let operation = WriteOperationId::generate().map_err(ConversionError::OperationId)?;
         let layout = RootLayout {
             root_dir: &self.root,
@@ -300,9 +403,9 @@ impl<'a> ConversionSession<'a> {
             dir: &self.journal_pin.canonical,
             operation: &operation,
         };
-        let fence = MigrationFence::from_manifest(&renewed);
+        let fence = MigrationFence::from_manifest(&successor);
         let step = JournalCheckpoint {
-            manifest: &renewed,
+            manifest: &successor,
             fence: &fence,
             journal,
             root_key_ref: &self.journal.root_key_ref,
@@ -316,19 +419,23 @@ impl<'a> ConversionSession<'a> {
                 .map_err(ConversionError::Admission)?
                 .ok_or(ConversionError::RootBusy)?;
             let root = event.root_guard().map_err(ConversionError::Admission)?;
-            let committed = commit_lease_renewal_held(fs, provider, layout, step, root);
+            let committed = match kind {
+                StepKind::Renewal => commit_lease_renewal_held(fs, provider, layout, step, root),
+                StepKind::Checkpoint => {
+                    commit_journal_checkpoint_held(fs, provider, layout, step, root)
+                }
+            };
             // Whatever the commit reported, read the root back while the event is still held: no other
             // root commit comes between, and the root says whether the step committed.
             (
                 committed,
                 read_committed_journal(fs, &*provider, layout, journal),
             )
         };
-        let settled = self.settle(&renewed, committed, read_back);
-        // A step that landed, whatever the commit reported, is followed by the admission check.
-        if matches!(settled, Ok(_) | Err(ConversionError::Committed(_))) {
-            self.check_admitted()?;
-        }
+        let settled = self.settle(&successor, committed, read_back);
+        // Every outcome of the commit call is followed by the admission check, and a lost admission is
+        // reported first: the step may have committed or left a candidate, whatever it reported.
+        self.check_admitted()?;
         settled
     }