ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Make concurrency verification deterministic and exhaustive

Browse source
This commit is contained in:
ms2k 2026-07-22 01:51:55 +09:00
commit 84bb22a7db
2 changed files with 216 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/lazaret-prepared-lower/tests/prepared_lower.rs
View file

@ -1,11 +1,10 @@
use std::fs::File;
use std::io::{Cursor, Read, Write};
use std::os::unix::ffi::OsStringExt;
use std::os::unix::ffi::{OsStrExt, OsStringExt};
use std::os::unix::fs::{MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Barrier};
use std::thread;
use std::time::{Duration, Instant};
use flate2::Compression;
use flate2::write::GzEncoder;
@ -14,6 +13,7 @@ use lazaret_core::identity::{
SelectedManifestDigest, SourceIdentity,
};
use lazaret_core::records::{PrepareOperation, PrepareStageState};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_prepared_lower::{
IdMap, IdMapRange, PrepareRecovery, PrepareRequest, PreparedDestination,
PreparedDestinationKind, PreparedLowerError, PreparedLowerStore, ReclaimOutcome,

214
crates/lazaret-prepared-lower/tests/prepared_lower/publication.rs
View file

@ -76,112 +76,60 @@ fn concurrent_publication_reuses_one_immutable_artifact() {
}
#[test]
fn startup_recovery_preserves_a_stage_owned_by_a_live_materializer() {
fn startup_recovery_preserves_an_independently_locked_owned_stage() {
let fixture = Fixture::new();
let first_layer = layer(&[Entry {
path: "large",
kind: EntryType::Regular,
data: &vec![0u8; 16 * 1024 * 1024],
target: None,
mode: 0o644,
}]);
let second_layer = layer(&[Entry {
path: "marker",
kind: EntryType::Regular,
data: b"complete",
target: None,
mode: 0o644,
}]);
let first_request = fixture.request(
42,
&[(first_layer, OCI_TAR), (second_layer.clone(), OCI_TAR)],
);
let second_request = PrepareRequest {
operation_id: PrepareOperationId::new([43; 32]),
..first_request.clone()
let operation_id = PrepareOperationId::new([42; 32]);
let stage_relative = PathBuf::from("stages").join(hex(operation_id.as_bytes()));
let stage_path = fixture.root.join(&stage_relative);
std::fs::create_dir(&stage_path).unwrap();
let metadata = stage_path.metadata().unwrap();
let intent = PrepareOperation {
operation_id,
expected_artifact_id: PreparedArtifactId::new([43; 32]),
source_identity: SourceIdentity::new(b"synthetic-live-owner".to_vec()),
selected_manifest_digest: SelectedManifestDigest::new([44; 32]),
policy_id: PolicyId::new([45; 32]),
id_map_digest: IdMapDigest::new([46; 32]),
stage_path: HostPath::new(stage_relative.as_os_str().as_bytes().to_vec()),
stage_state: PrepareStageState::Intent,
device: None,
inode: None,
};
let second_blob = first_request
.layout_path
.join("blobs/sha256")
.join(hex(&Sha256::digest(&second_layer)));
let (_, first_store) = fixture.independent_store();
let (second_catalog, second_store) = fixture.independent_store();
let first_handle = thread::spawn(move || first_store.prepare(&first_request));
let owned = PrepareOperation {
stage_state: PrepareStageState::Owned,
device: Some(DeviceId::new(metadata.dev())),
inode: Some(InodeId::new(metadata.ino())),
..intent.clone()
};
fixture.catalog.record_prepare(&intent).unwrap();
fixture.catalog.bind_prepare_stage(&intent, &owned).unwrap();
let live_owner = try_exclusive_flock(File::open(&stage_path).unwrap()).unwrap();
let (_, recovery_store) = fixture.independent_store();
let deadline = Instant::now() + Duration::from_secs(5);
let operation = loop {
if let Some(operation) = second_catalog
assert_eq!(
recovery_store.recover_preparations().unwrap(),
vec![(operation_id, PrepareRecovery::PreservedLiveOwnedStage,)]
);
assert_eq!(
fixture.catalog.snapshot().unwrap().prepare_operations,
vec![owned]
);
assert!(stage_path.is_dir());
drop(live_owner);
assert_eq!(
recovery_store.recover_preparations().unwrap(),
vec![(operation_id, PrepareRecovery::RemovedOwnedStage)]
);
assert!(
fixture
.catalog
.snapshot()
.unwrap()
.prepare_operations
.into_iter()
.find(|operation| operation.stage_state == PrepareStageState::Owned)
{
break operation;
}
assert!(
!first_handle.is_finished(),
"materialization finished before synchronization"
);
assert!(
Instant::now() < deadline,
"materialization did not publish stage ownership"
);
thread::yield_now();
};
std::fs::remove_file(&second_blob).unwrap();
rustix::fs::mknodat(
rustix::fs::CWD,
&second_blob,
rustix::fs::FileType::Fifo,
rustix::fs::Mode::from_raw_mode(0o600),
0,
)
.unwrap();
thread::sleep(Duration::from_millis(25));
assert!(
!first_handle.is_finished(),
"materializer passed the synchronization FIFO"
);
assert_eq!(
second_store.recover_preparations().unwrap(),
vec![(
operation.operation_id,
PrepareRecovery::PreservedLiveOwnedStage,
)]
);
let writer_path = second_blob.clone();
let writer_bytes = second_layer.clone();
let writer = thread::spawn(move || {
std::fs::OpenOptions::new()
.write(true)
.open(writer_path)
.unwrap()
.write_all(&writer_bytes)
.unwrap();
});
let first = first_handle.join().unwrap().unwrap();
writer.join().unwrap();
std::fs::remove_file(&second_blob).unwrap();
std::fs::write(&second_blob, &second_layer).unwrap();
let second = second_store.prepare(&second_request).unwrap();
assert_eq!(first, second);
let snapshot = second_catalog.snapshot().unwrap();
assert_eq!(snapshot.prepared_artifacts, vec![first]);
assert!(snapshot.prepare_operations.is_empty());
assert!(
fixture
.root
.join("stages")
.read_dir()
.unwrap()
.next()
.is_none()
.is_empty()
);
assert!(!stage_path.exists());
}
#[test]
@ -501,6 +449,74 @@ fn selected_gc_is_idempotent_only_at_the_durable_reclaim_boundary() {
assert!(fixture.store.gc_artifacts(&[id]).is_err());
}
#[test]
fn selected_artifact_excludes_gc_until_the_claim_is_released() {
let fixture = Fixture::new();
let artifact = fixture
.store
.prepare(&fixture.request(
72,
&[(
layer(&[Entry {
path: "payload",
kind: EntryType::Regular,
data: b"selected",
target: None,
mode: 0o644,
}]),
OCI_TAR,
)],
))
.unwrap();
let selected = fixture.store.select_artifact(&artifact).unwrap().unwrap();
let (_, gc_store) = fixture.independent_store();
let retained = gc_store
.gc_artifacts(&[artifact.prepared_artifact_id])
.unwrap();
assert_eq!(retained[0].outcome, ReclaimOutcome::RetainedBySelection);
assert_eq!(
fixture.catalog.snapshot().unwrap().prepared_artifacts,
vec![artifact.clone()]
);
assert!(fixture.artifact_root(&artifact).exists());
drop(selected);
let removed = gc_store
.gc_artifacts(&[artifact.prepared_artifact_id])
.unwrap();
assert_eq!(removed[0].outcome, ReclaimOutcome::Removed);
}
#[test]
fn selection_rebinds_to_an_equivalent_replacement() {
let fixture = Fixture::new();
let bytes = layer(&[Entry {
path: "payload",
kind: EntryType::Regular,
data: b"replacement",
target: None,
mode: 0o644,
}]);
let request = fixture.request(73, &[(bytes, OCI_TAR)]);
let first = fixture.store.prepare(&request).unwrap();
assert_eq!(
fixture.store.reclaim(&first).unwrap(),
ReclaimOutcome::Removed
);
let replacement = fixture
.store
.prepare(&PrepareRequest {
operation_id: PrepareOperationId::new([74; 32]),
..request
})
.unwrap();
let selection = fixture.store.select_artifact(&first).unwrap().unwrap();
assert_eq!(selection.artifact(), &replacement);
assert_ne!(selection.artifact().inode, first.inode);
}
#[test]
fn destination_layout_is_materialized_and_canonical_in_artifact_identity() {
let fixture = Fixture::new();