Serialize startup prepared artifact reclamation

crates/lazaret-host-coordinator/src/recovery/reconciliation.rs

@@ -232,20 +232,8 @@ where
         .recover_preparations()
         .map_err(|error| error.to_string())?;
     failpoint(RecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
-    let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
-    let selected = catalog_snapshot
-        .prepared_artifacts
-        .iter()
-        .filter(|artifact| {
-            !catalog_snapshot
-                .artifact_leases
-                .iter()
-                .any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id)
-        })
-        .map(|artifact| artifact.prepared_artifact_id)
-        .collect::<Vec<_>>();
     prepared
-        .gc_artifacts(&selected)
+        .gc_unleased_artifacts()
         .map_err(|error| error.to_string())?;
     Ok(())
 }

crates/lazaret-prepared-lower/src/store.rs

@@ -13,7 +13,9 @@ use lazaret_core::identity::{
     DeviceId, HostPath, IdMapDigest, InodeId, PolicyId, PrepareOperationId, PreparedArtifactId,
     SelectedManifestDigest, SourceIdentity,
 };
-use lazaret_core::records::{ArtifactState, PrepareOperation, PrepareStageState, PreparedArtifact};
+use lazaret_core::records::{
+    ArtifactState, PrepareOperation, PrepareStageState, PreparedArtifact, StorageCatalog,
+};
 use lazaret_linux::filesystem::{
     ExclusiveFileLock, SharedFileLock, try_exclusive_flock, try_shared_flock,
 };
@@ -522,12 +524,14 @@ impl PreparedLowerStore {
         &self,
         selected: &[PreparedArtifactId],
     ) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
-        self.gc_artifacts_inner(selected)
+        let physical_lock = self.physical_lock()?;
+        self.gc_artifacts_inner(selected, &physical_lock)
     }
 
     fn gc_artifacts_inner(
         &self,
         selected: &[PreparedArtifactId],
+        physical_lock: &ExclusiveFileLock,
     ) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
         let mut unique = std::collections::HashSet::new();
         if selected.iter().any(|id| !unique.insert(*id)) {
@@ -539,6 +543,40 @@ impl PreparedLowerStore {
             .catalog
             .snapshot()
             .map_err(PreparedLowerError::catalog)?;
+        self.gc_artifacts_from_snapshot(selected, &snapshot, physical_lock)
+    }
+
+    /// Reclaims the exact unleased artifacts observed while holding physical authority.
+    ///
+    /// # Errors
+    /// Refuses catalog drift, leases, physical ownership mismatch, foreign paths, and
+    /// interrupted reclaim without exact durable evidence.
+    pub fn gc_unleased_artifacts(&self) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
+        let physical_lock = self.physical_lock()?;
+        let snapshot = self
+            .catalog
+            .snapshot()
+            .map_err(PreparedLowerError::catalog)?;
+        let selected = snapshot
+            .prepared_artifacts
+            .iter()
+            .filter(|artifact| {
+                !snapshot
+                    .artifact_leases
+                    .iter()
+                    .any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id)
+            })
+            .map(|artifact| artifact.prepared_artifact_id)
+            .collect::<Vec<_>>();
+        self.gc_artifacts_from_snapshot(&selected, &snapshot, &physical_lock)
+    }
+
+    fn gc_artifacts_from_snapshot(
+        &self,
+        selected: &[PreparedArtifactId],
+        snapshot: &StorageCatalog,
+        physical_lock: &ExclusiveFileLock,
+    ) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
         let mut results = Vec::with_capacity(selected.len());
         for id in selected {
             let artifact = snapshot
@@ -548,8 +586,8 @@ impl PreparedLowerStore {
                 .context("prepared-artifact GC target is absent")
                 .map_err(PreparedLowerError::reclaim)?;
             let outcome = match artifact.state {
-                ArtifactState::Available => self.reclaim(artifact)?,
+                ArtifactState::Available => self.reclaim_inner(artifact, physical_lock)?,
-                ArtifactState::Reclaiming => self.resume_reclaim(artifact)?,
+                ArtifactState::Reclaiming => self.resume_reclaim_locked(artifact, physical_lock)?,
             };
             results.push(PreparedArtifactGcResult {
                 prepared_artifact_id: *id,
@@ -566,7 +604,15 @@ impl PreparedLowerStore {
     /// Refuses leases, forged catalog rows, physical identity mismatch, foreign paths, and
     /// filesystem or catalog failures.
     pub fn reclaim(&self, expected: &PreparedArtifact) -> PreparedLowerResult<ReclaimOutcome> {
-        let _physical_lock = self.physical_lock()?;
+        let physical_lock = self.physical_lock()?;
+        self.reclaim_inner(expected, &physical_lock)
+    }
+
+    fn reclaim_inner(
+        &self,
+        expected: &PreparedArtifact,
+        _physical_lock: &ExclusiveFileLock,
+    ) -> PreparedLowerResult<ReclaimOutcome> {
         let artifact = self
             .open_relative(&artifact_path(expected.prepared_artifact_id))
             .map_err(PreparedLowerError::reclaim)?
@@ -598,7 +644,15 @@ impl PreparedLowerStore {
         &self,
         reclaiming: &PreparedArtifact,
     ) -> PreparedLowerResult<ReclaimOutcome> {
-        let _physical_lock = self.physical_lock()?;
+        let physical_lock = self.physical_lock()?;
+        self.resume_reclaim_locked(reclaiming, &physical_lock)
+    }
+
+    fn resume_reclaim_locked(
+        &self,
+        reclaiming: &PreparedArtifact,
+        _physical_lock: &ExclusiveFileLock,
+    ) -> PreparedLowerResult<ReclaimOutcome> {
         let _claim = self
             .lock_reclaiming_artifact(reclaiming)
             .map_err(PreparedLowerError::reclaim)?;

crates/lazaret-prepared-lower/tests/prepared_lower/publication.rs

@@ -449,6 +449,60 @@ fn selected_gc_is_idempotent_only_at_the_durable_reclaim_boundary() {
     assert!(fixture.store.gc_artifacts(&[id]).is_err());
 }
 
+#[test]
+fn concurrent_unleased_gc_serializes_observation_with_reclaim() {
+    let fixture = Fixture::new();
+    let artifact = fixture
+        .store
+        .prepare(&fixture.request(
+            75,
+            &[(
+                layer(&[Entry {
+                    path: "payload",
+                    kind: EntryType::Regular,
+                    data: b"concurrent-gc",
+                    target: None,
+                    mode: 0o644,
+                }]),
+                OCI_TAR,
+            )],
+        ))
+        .unwrap();
+    let (_, first) = fixture.independent_store();
+    let (_, second) = fixture.independent_store();
+    let barrier = Arc::new(Barrier::new(3));
+    let first_barrier = Arc::clone(&barrier);
+    let first = thread::spawn(move || {
+        first_barrier.wait();
+        first.gc_unleased_artifacts()
+    });
+    let second_barrier = Arc::clone(&barrier);
+    let second = thread::spawn(move || {
+        second_barrier.wait();
+        second.gc_unleased_artifacts()
+    });
+
+    barrier.wait();
+    let results = [
+        first.join().unwrap().unwrap(),
+        second.join().unwrap().unwrap(),
+    ];
+    assert_eq!(results.iter().map(Vec::len).sum::<usize>(), 1);
+    assert_eq!(
+        results.iter().flatten().next().unwrap().outcome,
+        ReclaimOutcome::Removed
+    );
+    assert!(
+        fixture
+            .catalog
+            .snapshot()
+            .unwrap()
+            .prepared_artifacts
+            .is_empty()
+    );
+    assert!(!fixture.artifact_root(&artifact).exists());
+}
+
 #[test]
 fn selected_artifact_excludes_gc_until_the_claim_is_released() {
     let fixture = Fixture::new();

docs/protocol/PreparedLower.cfg

@@ -25,8 +25,9 @@ INVARIANTS
     LeaseBindingConsistent
     MountedGenerationLeased
     LeasedArtifactPublished
-    GCExcludesLeases
+    GCReclaimExcludesLeases
-    GCExcludesSelections
+    GCReclaimExcludesSelections
     SelectedArtifactStable
+    CompletedReclaimConverges
     ReplacementOnlyDuringGC
     ForeignStagePreserved

docs/protocol/PreparedLower.tla

@@ -45,10 +45,13 @@ VARIABLES
     selectionBinding,
     mounts,
     artifactBinding,
-    gcRequested,
+    gcObservation,
     replacementPresent,
     foreignObserved
 
+FreshArtifactIncarnations ==
+    ArtifactIncarnations \ {gcObservation[g] : g \in Generations}
+
 vars == <<
     artifactState,
     artifactIncarnation,
@@ -59,7 +62,7 @@ vars == <<
     selectionBinding,
     mounts,
     artifactBinding,
-    gcRequested,
+    gcObservation,
     replacementPresent,
     foreignObserved
 >>
@@ -74,7 +77,7 @@ Init ==
     /\ selectionBinding = [g \in Generations |-> NoArtifact]
     /\ mounts = {}
     /\ artifactBinding = [g \in Generations |-> NoArtifact]
-    /\ gcRequested = FALSE
+    /\ gcObservation = [g \in Generations |-> NoArtifact]
     /\ replacementPresent = FALSE
     /\ foreignObserved = FALSE
 
@@ -86,7 +89,7 @@ BeginPrepare(p) ==
     /\ prepareIntent' = [prepareIntent EXCEPT ![p] = TRUE]
     /\ materializerState' = [materializerState EXCEPT ![p] = "Live"]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState, leases,
-        selectionBinding, mounts, artifactBinding, gcRequested,
+        selectionBinding, mounts, artifactBinding, gcObservation,
         replacementPresent, foreignObserved>>
 
 CreateOwnedStage(p) ==
@@ -96,7 +99,7 @@ CreateOwnedStage(p) ==
     /\ stageState' = [stageState EXCEPT ![p] = "Owned"]
     /\ UNCHANGED <<artifactState, artifactIncarnation, prepareIntent,
         materializerState, leases, selectionBinding, mounts, artifactBinding,
-        gcRequested, replacementPresent, foreignObserved>>
+        gcObservation, replacementPresent, foreignObserved>>
 
 LoseMaterializer(p) ==
     /\ prepareIntent[p]
@@ -104,24 +107,24 @@ LoseMaterializer(p) ==
     /\ materializerState' = [materializerState EXCEPT ![p] = "Lost"]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, leases, selectionBinding, mounts, artifactBinding,
-        gcRequested, replacementPresent, foreignObserved>>
+        gcObservation, replacementPresent, foreignObserved>>
 
 PublishArtifact(p) ==
     /\ artifactState = "Absent"
+    /\ FreshArtifactIncarnations # {}
     /\ stageState[p] = "Owned"
     /\ prepareIntent[p]
     /\ materializerState[p] = "Live"
     /\ artifactState' = "Published"
-    /\ artifactIncarnation' = A1
+    /\ artifactIncarnation' \in FreshArtifactIncarnations
     /\ stageState' = [stageState EXCEPT ![p] = "Absent"]
     /\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
     /\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
     /\ UNCHANGED <<leases, selectionBinding, mounts, artifactBinding,
-        gcRequested, replacementPresent, foreignObserved>>
+        gcObservation, replacementPresent, foreignObserved>>
 
 ReusePublishedArtifact(p) ==
     /\ artifactState = "Published"
-    /\ ~gcRequested
     /\ stageState[p] = "Owned"
     /\ prepareIntent[p]
     /\ materializerState[p] = "Live"
@@ -129,7 +132,7 @@ ReusePublishedArtifact(p) ==
     /\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
     /\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
     /\ UNCHANGED <<artifactState, artifactIncarnation, leases,
-        selectionBinding, mounts, artifactBinding, gcRequested,
+        selectionBinding, mounts, artifactBinding, gcObservation,
         replacementPresent, foreignObserved>>
 
 RecoverUnmaterializedIntent(p) ==
@@ -139,7 +142,7 @@ RecoverUnmaterializedIntent(p) ==
     /\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
     /\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState, leases,
-        selectionBinding, mounts, artifactBinding, gcRequested,
+        selectionBinding, mounts, artifactBinding, gcObservation,
         replacementPresent, foreignObserved>>
 
 RecoverOwnedStage(p) ==
@@ -150,7 +153,7 @@ RecoverOwnedStage(p) ==
     /\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
     /\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
     /\ UNCHANGED <<artifactState, artifactIncarnation, leases,
-        selectionBinding, mounts, artifactBinding, gcRequested,
+        selectionBinding, mounts, artifactBinding, gcObservation,
         replacementPresent, foreignObserved>>
 
 ObserveForeignStage(p) ==
@@ -161,7 +164,7 @@ ObserveForeignStage(p) ==
     /\ foreignObserved' = TRUE
     /\ UNCHANGED <<artifactState, artifactIncarnation, prepareIntent,
         materializerState, leases, selectionBinding, mounts, artifactBinding,
-        gcRequested, replacementPresent>>
+        gcObservation, replacementPresent>>
 
 RefuseForeignStage(p) ==
     /\ stageState[p] = "Foreign"
@@ -169,14 +172,13 @@ RefuseForeignStage(p) ==
 
 SelectPublishedArtifact(g) ==
     /\ artifactState = "Published"
-    /\ ~gcRequested
     /\ g \notin leases
     /\ selectionBinding[g] = NoArtifact
     /\ selectionBinding' =
         [selectionBinding EXCEPT ![g] = artifactIncarnation]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, materializerState, leases, mounts, artifactBinding,
-        gcRequested, replacementPresent, foreignObserved>>
+        gcObservation, replacementPresent, foreignObserved>>
 
 ReleaseSelection(g) ==
     /\ selectionBinding[g] # NoArtifact
@@ -184,18 +186,17 @@ ReleaseSelection(g) ==
     /\ selectionBinding' = [selectionBinding EXCEPT ![g] = NoArtifact]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, materializerState, leases, mounts, artifactBinding,
-        gcRequested, replacementPresent, foreignObserved>>
+        gcObservation, replacementPresent, foreignObserved>>
 
 AcquireLease(g) ==
     /\ artifactState = "Published"
-    /\ ~gcRequested
     /\ g \notin leases
     /\ selectionBinding[g] = artifactIncarnation
     /\ leases' = leases \cup {g}
     /\ selectionBinding' = [selectionBinding EXCEPT ![g] = NoArtifact]
     /\ artifactBinding' = [artifactBinding EXCEPT ![g] = Artifact]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState, prepareIntent,
-        materializerState, mounts, gcRequested, replacementPresent,
+        materializerState, mounts, gcObservation, replacementPresent,
         foreignObserved>>
 
 MountStateAnchor(g) ==
@@ -205,14 +206,14 @@ MountStateAnchor(g) ==
     /\ mounts' = mounts \cup {g}
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, materializerState, leases, selectionBinding,
-        artifactBinding, gcRequested, replacementPresent, foreignObserved>>
+        artifactBinding, gcObservation, replacementPresent, foreignObserved>>
 
 UnmountStateAnchor(g) ==
     /\ g \in mounts
     /\ mounts' = mounts \ {g}
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, materializerState, leases, selectionBinding,
-        artifactBinding, gcRequested, replacementPresent, foreignObserved>>
+        artifactBinding, gcObservation, replacementPresent, foreignObserved>>
 
 ReleaseLease(g) ==
     /\ g \in leases
@@ -221,43 +222,60 @@ ReleaseLease(g) ==
     /\ artifactBinding' = [artifactBinding EXCEPT ![g] = NoArtifact]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, materializerState, selectionBinding, mounts,
-        gcRequested, replacementPresent, foreignObserved>>
+        gcObservation, replacementPresent, foreignObserved>>
 
-RequestGC ==
+ObserveGCCandidate(g) ==
     /\ artifactState = "Published"
     /\ leases = {}
-    /\ \A g \in Generations : selectionBinding[g] = NoArtifact
+    /\ \A selected \in Generations :
-    /\ ~gcRequested
+        selectionBinding[selected] = NoArtifact
-    /\ gcRequested' = TRUE
+    /\ gcObservation[g] = NoArtifact
+    /\ gcObservation' =
+        [gcObservation EXCEPT ![g] = artifactIncarnation]
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, materializerState, leases, selectionBinding, mounts,
         artifactBinding, replacementPresent, foreignObserved>>
 
 PublishReplacementDuringGC ==
     /\ artifactState = "Published"
-    /\ gcRequested
+    /\ \E g \in Generations :
+        gcObservation[g] = artifactIncarnation
+    /\ (IF artifactIncarnation = A1 THEN A2 ELSE A1)
+        \in FreshArtifactIncarnations
     /\ ~replacementPresent
     /\ replacementPresent' = TRUE
     /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
         prepareIntent, materializerState, leases, selectionBinding, mounts,
-        artifactBinding, gcRequested, foreignObserved>>
+        artifactBinding, gcObservation, foreignObserved>>
 
-ReclaimArtifact ==
+ReclaimObservedArtifact(g) ==
     /\ artifactState = "Published"
-    /\ gcRequested
+    /\ gcObservation[g] = artifactIncarnation
     /\ leases = {}
-    /\ \A g \in Generations : selectionBinding[g] = NoArtifact
+    /\ \A selected \in Generations :
+        selectionBinding[selected] = NoArtifact
     /\ mounts = {}
     /\ artifactState' = IF replacementPresent THEN "Published" ELSE "Absent"
     /\ artifactIncarnation' =
         IF replacementPresent
         THEN IF artifactIncarnation = A1 THEN A2 ELSE A1
         ELSE NoArtifact
-    /\ gcRequested' = FALSE
+    /\ gcObservation' =
+        [gcObservation EXCEPT ![g] = NoArtifact]
     /\ replacementPresent' = FALSE
     /\ UNCHANGED <<stageState, prepareIntent, materializerState, leases,
         selectionBinding, mounts, artifactBinding, foreignObserved>>
 
+ConvergeCompletedReclaim(g) ==
+    /\ gcObservation[g] # NoArtifact
+    /\ \/ artifactState = "Absent"
+       \/ gcObservation[g] # artifactIncarnation
+    /\ gcObservation' =
+        [gcObservation EXCEPT ![g] = NoArtifact]
+    /\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
+        prepareIntent, materializerState, leases, selectionBinding, mounts,
+        artifactBinding, replacementPresent, foreignObserved>>
+
 Next ==
     \/ (\E p \in Preparations : BeginPrepare(p))
     \/ (\E p \in Preparations : CreateOwnedStage(p))
@@ -274,9 +292,10 @@ Next ==
     \/ (\E g \in Generations : MountStateAnchor(g))
     \/ (\E g \in Generations : UnmountStateAnchor(g))
     \/ (\E g \in Generations : ReleaseLease(g))
-    \/ RequestGC
+    \/ (\E g \in Generations : ObserveGCCandidate(g))
     \/ PublishReplacementDuringGC
-    \/ ReclaimArtifact
+    \/ (\E g \in Generations : ReclaimObservedArtifact(g))
+    \/ (\E g \in Generations : ConvergeCompletedReclaim(g))
 
 Spec == Init /\ [][Next]_vars
 
@@ -291,7 +310,8 @@ TypeOK ==
         [Generations -> ArtifactIncarnations \cup {NoArtifact}]
     /\ mounts \subseteq Generations
     /\ artifactBinding \in [Generations -> {NoArtifact, Artifact}]
-    /\ gcRequested \in BOOLEAN
+    /\ gcObservation \in
+        [Generations -> ArtifactIncarnations \cup {NoArtifact}]
     /\ replacementPresent \in BOOLEAN
     /\ foreignObserved \in BOOLEAN
 
@@ -332,7 +352,6 @@ LostIntentRecoverable ==
 PublishedOwnedStageReusable ==
     \A p \in Preparations :
         /\ artifactState = "Published"
-        /\ ~gcRequested
         /\ materializerState[p] = "Live"
         /\ stageState[p] = "Owned"
         => ENABLED ReusePublishedArtifact(p)
@@ -348,24 +367,36 @@ MountedGenerationLeased ==
 LeasedArtifactPublished ==
     leases # {} => artifactState = "Published"
 
-GCExcludesLeases ==
+GCReclaimExcludesLeases ==
-    gcRequested =>
+    \A g \in Generations :
-        /\ artifactState = "Published"
+        ENABLED ReclaimObservedArtifact(g) =>
             /\ leases = {}
             /\ mounts = {}
 
-GCExcludesSelections ==
+GCReclaimExcludesSelections ==
-    gcRequested =>
+    \A g \in Generations :
-        \A g \in Generations : selectionBinding[g] = NoArtifact
+        ENABLED ReclaimObservedArtifact(g) =>
+            \A selected \in Generations :
+                selectionBinding[selected] = NoArtifact
 
 SelectedArtifactStable ==
     \A g \in Generations :
         selectionBinding[g] # NoArtifact =>
-            /\ ~ENABLED RequestGC
+            \A observer \in Generations :
-            /\ ~ENABLED ReclaimArtifact
+                ~ENABLED ReclaimObservedArtifact(observer)
+
+CompletedReclaimConverges ==
+    \A g \in Generations :
+        /\ gcObservation[g] # NoArtifact
+        /\ \/ artifactState = "Absent"
+           \/ gcObservation[g] # artifactIncarnation
+        => /\ ENABLED ConvergeCompletedReclaim(g)
+           /\ ~ENABLED ReclaimObservedArtifact(g)
 
 ReplacementOnlyDuringGC ==
-    replacementPresent => gcRequested
+    replacementPresent =>
+        \E g \in Generations :
+            gcObservation[g] = artifactIncarnation
 
 ForeignStagePreserved ==
     foreignObserved => \E p \in Preparations : stageState[p] = "Foreign"