ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Serialize startup prepared artifact reclamation

Browse source
This commit is contained in:
ms2k 2026-07-22 03:56:44 +09:00
commit 7cd8a90af0
5 changed files with 195 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crates/lazaret-host-coordinator/src/recovery/reconciliation.rs
View file

@ -232,20 +232,8 @@ where
.recover_preparations()
.map_err(|error| error.to_string())?;
failpoint(RecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
let selected = catalog_snapshot
.prepared_artifacts
.iter()
.filter(|artifact| {
!catalog_snapshot
.artifact_leases
.iter()
.any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id)
})
.map(|artifact| artifact.prepared_artifact_id)
.collect::<Vec<_>>();
prepared
.gc_artifacts(&selected)
.gc_unleased_artifacts()
.map_err(|error| error.to_string())?;
Ok(())
}

66
crates/lazaret-prepared-lower/src/store.rs
View file

@ -13,7 +13,9 @@ use lazaret_core::identity::{
DeviceId, HostPath, IdMapDigest, InodeId, PolicyId, PrepareOperationId, PreparedArtifactId,
SelectedManifestDigest, SourceIdentity,
};
use lazaret_core::records::{ArtifactState, PrepareOperation, PrepareStageState, PreparedArtifact};
use lazaret_core::records::{
ArtifactState, PrepareOperation, PrepareStageState, PreparedArtifact, StorageCatalog,
};
use lazaret_linux::filesystem::{
ExclusiveFileLock, SharedFileLock, try_exclusive_flock, try_shared_flock,
};
@ -522,12 +524,14 @@ impl PreparedLowerStore {
&self,
selected: &[PreparedArtifactId],
) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
self.gc_artifacts_inner(selected)
let physical_lock = self.physical_lock()?;
self.gc_artifacts_inner(selected, &physical_lock)
}
fn gc_artifacts_inner(
&self,
selected: &[PreparedArtifactId],
physical_lock: &ExclusiveFileLock,
) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
let mut unique = std::collections::HashSet::new();
if selected.iter().any(|id| !unique.insert(*id)) {
@ -539,6 +543,40 @@ impl PreparedLowerStore {
.catalog
.snapshot()
.map_err(PreparedLowerError::catalog)?;
self.gc_artifacts_from_snapshot(selected, &snapshot, physical_lock)
}
/// Reclaims the exact unleased artifacts observed while holding physical authority.
///
/// # Errors
/// Refuses catalog drift, leases, physical ownership mismatch, foreign paths, and
/// interrupted reclaim without exact durable evidence.
pub fn gc_unleased_artifacts(&self) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
let physical_lock = self.physical_lock()?;
let snapshot = self
.catalog
.snapshot()
.map_err(PreparedLowerError::catalog)?;
let selected = snapshot
.prepared_artifacts
.iter()
.filter(|artifact| {
!snapshot
.artifact_leases
.iter()
.any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id)
})
.map(|artifact| artifact.prepared_artifact_id)
.collect::<Vec<_>>();
self.gc_artifacts_from_snapshot(&selected, &snapshot, &physical_lock)
}
fn gc_artifacts_from_snapshot(
&self,
selected: &[PreparedArtifactId],
snapshot: &StorageCatalog,
physical_lock: &ExclusiveFileLock,
) -> PreparedLowerResult<Vec<PreparedArtifactGcResult>> {
let mut results = Vec::with_capacity(selected.len());
for id in selected {
let artifact = snapshot
@ -548,8 +586,8 @@ impl PreparedLowerStore {
.context("prepared-artifact GC target is absent")
.map_err(PreparedLowerError::reclaim)?;
let outcome = match artifact.state {
ArtifactState::Available => self.reclaim(artifact)?,
ArtifactState::Reclaiming => self.resume_reclaim(artifact)?,
ArtifactState::Available => self.reclaim_inner(artifact, physical_lock)?,
ArtifactState::Reclaiming => self.resume_reclaim_locked(artifact, physical_lock)?,
};
results.push(PreparedArtifactGcResult {
prepared_artifact_id: *id,
@ -566,7 +604,15 @@ impl PreparedLowerStore {
/// Refuses leases, forged catalog rows, physical identity mismatch, foreign paths, and
/// filesystem or catalog failures.
pub fn reclaim(&self, expected: &PreparedArtifact) -> PreparedLowerResult<ReclaimOutcome> {
let _physical_lock = self.physical_lock()?;
let physical_lock = self.physical_lock()?;
self.reclaim_inner(expected, &physical_lock)
}
fn reclaim_inner(
&self,
expected: &PreparedArtifact,
_physical_lock: &ExclusiveFileLock,
) -> PreparedLowerResult<ReclaimOutcome> {
let artifact = self
.open_relative(&artifact_path(expected.prepared_artifact_id))
.map_err(PreparedLowerError::reclaim)?
@ -598,7 +644,15 @@ impl PreparedLowerStore {
&self,
reclaiming: &PreparedArtifact,
) -> PreparedLowerResult<ReclaimOutcome> {
let _physical_lock = self.physical_lock()?;
let physical_lock = self.physical_lock()?;
self.resume_reclaim_locked(reclaiming, &physical_lock)
}
fn resume_reclaim_locked(
&self,
reclaiming: &PreparedArtifact,
_physical_lock: &ExclusiveFileLock,
) -> PreparedLowerResult<ReclaimOutcome> {
let _claim = self
.lock_reclaiming_artifact(reclaiming)
.map_err(PreparedLowerError::reclaim)?;

54
crates/lazaret-prepared-lower/tests/prepared_lower/publication.rs
View file

@ -449,6 +449,60 @@ fn selected_gc_is_idempotent_only_at_the_durable_reclaim_boundary() {
assert!(fixture.store.gc_artifacts(&[id]).is_err());
}
#[test]
fn concurrent_unleased_gc_serializes_observation_with_reclaim() {
let fixture = Fixture::new();
let artifact = fixture
.store
.prepare(&fixture.request(
75,
&[(
layer(&[Entry {
path: "payload",
kind: EntryType::Regular,
data: b"concurrent-gc",
target: None,
mode: 0o644,
}]),
OCI_TAR,
)],
))
.unwrap();
let (_, first) = fixture.independent_store();
let (_, second) = fixture.independent_store();
let barrier = Arc::new(Barrier::new(3));
let first_barrier = Arc::clone(&barrier);
let first = thread::spawn(move || {
first_barrier.wait();
first.gc_unleased_artifacts()
});
let second_barrier = Arc::clone(&barrier);
let second = thread::spawn(move || {
second_barrier.wait();
second.gc_unleased_artifacts()
});
barrier.wait();
let results = [
first.join().unwrap().unwrap(),
second.join().unwrap().unwrap(),
];
assert_eq!(results.iter().map(Vec::len).sum::<usize>(), 1);
assert_eq!(
results.iter().flatten().next().unwrap().outcome,
ReclaimOutcome::Removed
);
assert!(
fixture
.catalog
.snapshot()
.unwrap()
.prepared_artifacts
.is_empty()
);
assert!(!fixture.artifact_root(&artifact).exists());
}
#[test]
fn selected_artifact_excludes_gc_until_the_claim_is_released() {
let fixture = Fixture::new();

5
docs/protocol/PreparedLower.cfg
View file

@ -25,8 +25,9 @@ INVARIANTS
LeaseBindingConsistent
MountedGenerationLeased
LeasedArtifactPublished
GCExcludesLeases
GCExcludesSelections
GCReclaimExcludesLeases
GCReclaimExcludesSelections
SelectedArtifactStable
CompletedReclaimConverges
ReplacementOnlyDuringGC
ForeignStagePreserved

123
docs/protocol/PreparedLower.tla
View file

@ -45,10 +45,13 @@ VARIABLES
selectionBinding,
mounts,
artifactBinding,
gcRequested,
gcObservation,
replacementPresent,
foreignObserved
FreshArtifactIncarnations ==
ArtifactIncarnations \ {gcObservation[g] : g \in Generations}
vars == <<
artifactState,
artifactIncarnation,
@ -59,7 +62,7 @@ vars == <<
selectionBinding,
mounts,
artifactBinding,
gcRequested,
gcObservation,
replacementPresent,
foreignObserved
>>
@ -74,7 +77,7 @@ Init ==
/\ selectionBinding = [g \in Generations |-> NoArtifact]
/\ mounts = {}
/\ artifactBinding = [g \in Generations |-> NoArtifact]
/\ gcRequested = FALSE
/\ gcObservation = [g \in Generations |-> NoArtifact]
/\ replacementPresent = FALSE
/\ foreignObserved = FALSE
@ -86,7 +89,7 @@ BeginPrepare(p) ==
/\ prepareIntent' = [prepareIntent EXCEPT ![p] = TRUE]
/\ materializerState' = [materializerState EXCEPT ![p] = "Live"]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState, leases,
selectionBinding, mounts, artifactBinding, gcRequested,
selectionBinding, mounts, artifactBinding, gcObservation,
replacementPresent, foreignObserved>>
CreateOwnedStage(p) ==
@ -96,7 +99,7 @@ CreateOwnedStage(p) ==
/\ stageState' = [stageState EXCEPT ![p] = "Owned"]
/\ UNCHANGED <<artifactState, artifactIncarnation, prepareIntent,
materializerState, leases, selectionBinding, mounts, artifactBinding,
gcRequested, replacementPresent, foreignObserved>>
gcObservation, replacementPresent, foreignObserved>>
LoseMaterializer(p) ==
/\ prepareIntent[p]
@ -104,24 +107,24 @@ LoseMaterializer(p) ==
/\ materializerState' = [materializerState EXCEPT ![p] = "Lost"]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, leases, selectionBinding, mounts, artifactBinding,
gcRequested, replacementPresent, foreignObserved>>
gcObservation, replacementPresent, foreignObserved>>
PublishArtifact(p) ==
/\ artifactState = "Absent"
/\ FreshArtifactIncarnations # {}
/\ stageState[p] = "Owned"
/\ prepareIntent[p]
/\ materializerState[p] = "Live"
/\ artifactState' = "Published"
/\ artifactIncarnation' = A1
/\ artifactIncarnation' \in FreshArtifactIncarnations
/\ stageState' = [stageState EXCEPT ![p] = "Absent"]
/\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
/\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
/\ UNCHANGED <<leases, selectionBinding, mounts, artifactBinding,
gcRequested, replacementPresent, foreignObserved>>
gcObservation, replacementPresent, foreignObserved>>
ReusePublishedArtifact(p) ==
/\ artifactState = "Published"
/\ ~gcRequested
/\ stageState[p] = "Owned"
/\ prepareIntent[p]
/\ materializerState[p] = "Live"
@ -129,7 +132,7 @@ ReusePublishedArtifact(p) ==
/\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
/\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
/\ UNCHANGED <<artifactState, artifactIncarnation, leases,
selectionBinding, mounts, artifactBinding, gcRequested,
selectionBinding, mounts, artifactBinding, gcObservation,
replacementPresent, foreignObserved>>
RecoverUnmaterializedIntent(p) ==
@ -139,7 +142,7 @@ RecoverUnmaterializedIntent(p) ==
/\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
/\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState, leases,
selectionBinding, mounts, artifactBinding, gcRequested,
selectionBinding, mounts, artifactBinding, gcObservation,
replacementPresent, foreignObserved>>
RecoverOwnedStage(p) ==
@ -150,7 +153,7 @@ RecoverOwnedStage(p) ==
/\ prepareIntent' = [prepareIntent EXCEPT ![p] = FALSE]
/\ materializerState' = [materializerState EXCEPT ![p] = "Absent"]
/\ UNCHANGED <<artifactState, artifactIncarnation, leases,
selectionBinding, mounts, artifactBinding, gcRequested,
selectionBinding, mounts, artifactBinding, gcObservation,
replacementPresent, foreignObserved>>
ObserveForeignStage(p) ==
@ -161,7 +164,7 @@ ObserveForeignStage(p) ==
/\ foreignObserved' = TRUE
/\ UNCHANGED <<artifactState, artifactIncarnation, prepareIntent,
materializerState, leases, selectionBinding, mounts, artifactBinding,
gcRequested, replacementPresent>>
gcObservation, replacementPresent>>
RefuseForeignStage(p) ==
/\ stageState[p] = "Foreign"
@ -169,14 +172,13 @@ RefuseForeignStage(p) ==
SelectPublishedArtifact(g) ==
/\ artifactState = "Published"
/\ ~gcRequested
/\ g \notin leases
/\ selectionBinding[g] = NoArtifact
/\ selectionBinding' =
[selectionBinding EXCEPT ![g] = artifactIncarnation]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, leases, mounts, artifactBinding,
gcRequested, replacementPresent, foreignObserved>>
gcObservation, replacementPresent, foreignObserved>>
ReleaseSelection(g) ==
/\ selectionBinding[g] # NoArtifact
@ -184,18 +186,17 @@ ReleaseSelection(g) ==
/\ selectionBinding' = [selectionBinding EXCEPT ![g] = NoArtifact]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, leases, mounts, artifactBinding,
gcRequested, replacementPresent, foreignObserved>>
gcObservation, replacementPresent, foreignObserved>>
AcquireLease(g) ==
/\ artifactState = "Published"
/\ ~gcRequested
/\ g \notin leases
/\ selectionBinding[g] = artifactIncarnation
/\ leases' = leases \cup {g}
/\ selectionBinding' = [selectionBinding EXCEPT ![g] = NoArtifact]
/\ artifactBinding' = [artifactBinding EXCEPT ![g] = Artifact]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState, prepareIntent,
materializerState, mounts, gcRequested, replacementPresent,
materializerState, mounts, gcObservation, replacementPresent,
foreignObserved>>
MountStateAnchor(g) ==
@ -205,14 +206,14 @@ MountStateAnchor(g) ==
/\ mounts' = mounts \cup {g}
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, leases, selectionBinding,
artifactBinding, gcRequested, replacementPresent, foreignObserved>>
artifactBinding, gcObservation, replacementPresent, foreignObserved>>
UnmountStateAnchor(g) ==
/\ g \in mounts
/\ mounts' = mounts \ {g}
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, leases, selectionBinding,
artifactBinding, gcRequested, replacementPresent, foreignObserved>>
artifactBinding, gcObservation, replacementPresent, foreignObserved>>
ReleaseLease(g) ==
/\ g \in leases
@ -221,43 +222,60 @@ ReleaseLease(g) ==
/\ artifactBinding' = [artifactBinding EXCEPT ![g] = NoArtifact]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, selectionBinding, mounts,
gcRequested, replacementPresent, foreignObserved>>
gcObservation, replacementPresent, foreignObserved>>
RequestGC ==
ObserveGCCandidate(g) ==
/\ artifactState = "Published"
/\ leases = {}
/\ \A g \in Generations : selectionBinding[g] = NoArtifact
/\ ~gcRequested
/\ gcRequested' = TRUE
/\ \A selected \in Generations :
selectionBinding[selected] = NoArtifact
/\ gcObservation[g] = NoArtifact
/\ gcObservation' =
[gcObservation EXCEPT ![g] = artifactIncarnation]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, leases, selectionBinding, mounts,
artifactBinding, replacementPresent, foreignObserved>>
PublishReplacementDuringGC ==
/\ artifactState = "Published"
/\ gcRequested
/\ \E g \in Generations :
gcObservation[g] = artifactIncarnation
/\ (IF artifactIncarnation = A1 THEN A2 ELSE A1)
\in FreshArtifactIncarnations
/\ ~replacementPresent
/\ replacementPresent' = TRUE
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, leases, selectionBinding, mounts,
artifactBinding, gcRequested, foreignObserved>>
artifactBinding, gcObservation, foreignObserved>>
ReclaimArtifact ==
ReclaimObservedArtifact(g) ==
/\ artifactState = "Published"
/\ gcRequested
/\ gcObservation[g] = artifactIncarnation
/\ leases = {}
/\ \A g \in Generations : selectionBinding[g] = NoArtifact
/\ \A selected \in Generations :
selectionBinding[selected] = NoArtifact
/\ mounts = {}
/\ artifactState' = IF replacementPresent THEN "Published" ELSE "Absent"
/\ artifactIncarnation' =
IF replacementPresent
THEN IF artifactIncarnation = A1 THEN A2 ELSE A1
ELSE NoArtifact
/\ gcRequested' = FALSE
/\ gcObservation' =
[gcObservation EXCEPT ![g] = NoArtifact]
/\ replacementPresent' = FALSE
/\ UNCHANGED <<stageState, prepareIntent, materializerState, leases,
selectionBinding, mounts, artifactBinding, foreignObserved>>
ConvergeCompletedReclaim(g) ==
/\ gcObservation[g] # NoArtifact
/\ \/ artifactState = "Absent"
\/ gcObservation[g] # artifactIncarnation
/\ gcObservation' =
[gcObservation EXCEPT ![g] = NoArtifact]
/\ UNCHANGED <<artifactState, artifactIncarnation, stageState,
prepareIntent, materializerState, leases, selectionBinding, mounts,
artifactBinding, replacementPresent, foreignObserved>>
Next ==
\/ (\E p \in Preparations : BeginPrepare(p))
\/ (\E p \in Preparations : CreateOwnedStage(p))
@ -274,9 +292,10 @@ Next ==
\/ (\E g \in Generations : MountStateAnchor(g))
\/ (\E g \in Generations : UnmountStateAnchor(g))
\/ (\E g \in Generations : ReleaseLease(g))
\/ RequestGC
\/ (\E g \in Generations : ObserveGCCandidate(g))
\/ PublishReplacementDuringGC
\/ ReclaimArtifact
\/ (\E g \in Generations : ReclaimObservedArtifact(g))
\/ (\E g \in Generations : ConvergeCompletedReclaim(g))
Spec == Init /\ [][Next]_vars
@ -291,7 +310,8 @@ TypeOK ==
[Generations -> ArtifactIncarnations \cup {NoArtifact}]
/\ mounts \subseteq Generations
/\ artifactBinding \in [Generations -> {NoArtifact, Artifact}]
/\ gcRequested \in BOOLEAN
/\ gcObservation \in
[Generations -> ArtifactIncarnations \cup {NoArtifact}]
/\ replacementPresent \in BOOLEAN
/\ foreignObserved \in BOOLEAN
@ -332,7 +352,6 @@ LostIntentRecoverable ==
PublishedOwnedStageReusable ==
\A p \in Preparations :
/\ artifactState = "Published"
/\ ~gcRequested
/\ materializerState[p] = "Live"
/\ stageState[p] = "Owned"
=> ENABLED ReusePublishedArtifact(p)
@ -348,24 +367,36 @@ MountedGenerationLeased ==
LeasedArtifactPublished ==
leases # {} => artifactState = "Published"
GCExcludesLeases ==
gcRequested =>
/\ artifactState = "Published"
/\ leases = {}
/\ mounts = {}
GCReclaimExcludesLeases ==
\A g \in Generations :
ENABLED ReclaimObservedArtifact(g) =>
/\ leases = {}
/\ mounts = {}
GCExcludesSelections ==
gcRequested =>
\A g \in Generations : selectionBinding[g] = NoArtifact
GCReclaimExcludesSelections ==
\A g \in Generations :
ENABLED ReclaimObservedArtifact(g) =>
\A selected \in Generations :
selectionBinding[selected] = NoArtifact
SelectedArtifactStable ==
\A g \in Generations :
selectionBinding[g] # NoArtifact =>
/\ ~ENABLED RequestGC
/\ ~ENABLED ReclaimArtifact
\A observer \in Generations :
~ENABLED ReclaimObservedArtifact(observer)
CompletedReclaimConverges ==
\A g \in Generations :
/\ gcObservation[g] # NoArtifact
/\ \/ artifactState = "Absent"
\/ gcObservation[g] # artifactIncarnation
=> /\ ENABLED ConvergeCompletedReclaim(g)
/\ ~ENABLED ReclaimObservedArtifact(g)
ReplacementOnlyDuringGC ==
replacementPresent => gcRequested
replacementPresent =>
\E g \in Generations :
gcObservation[g] = artifactIncarnation
ForeignStagePreserved ==
foreignObserved => \E p \in Preparations : stageState[p] = "Foreign"