Make concurrency verification deterministic and exhaustive
This commit is contained in:
parent
1bd1d395d9
commit
84bb22a7db
1 changed files with 216 additions and 102 deletions
|
|
@ -76,112 +76,60 @@ fn concurrent_publication_reuses_one_immutable_artifact() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn startup_recovery_preserves_a_stage_owned_by_a_live_materializer() {
|
||||
fn startup_recovery_preserves_an_independently_locked_owned_stage() {
|
||||
let fixture = Fixture::new();
|
||||
let first_layer = layer(&[Entry {
|
||||
path: "large",
|
||||
kind: EntryType::Regular,
|
||||
data: &vec![0u8; 16 * 1024 * 1024],
|
||||
target: None,
|
||||
mode: 0o644,
|
||||
}]);
|
||||
let second_layer = layer(&[Entry {
|
||||
path: "marker",
|
||||
kind: EntryType::Regular,
|
||||
data: b"complete",
|
||||
target: None,
|
||||
mode: 0o644,
|
||||
}]);
|
||||
let first_request = fixture.request(
|
||||
42,
|
||||
&[(first_layer, OCI_TAR), (second_layer.clone(), OCI_TAR)],
|
||||
);
|
||||
let second_request = PrepareRequest {
|
||||
operation_id: PrepareOperationId::new([43; 32]),
|
||||
..first_request.clone()
|
||||
let operation_id = PrepareOperationId::new([42; 32]);
|
||||
let stage_relative = PathBuf::from("stages").join(hex(operation_id.as_bytes()));
|
||||
let stage_path = fixture.root.join(&stage_relative);
|
||||
std::fs::create_dir(&stage_path).unwrap();
|
||||
let metadata = stage_path.metadata().unwrap();
|
||||
let intent = PrepareOperation {
|
||||
operation_id,
|
||||
expected_artifact_id: PreparedArtifactId::new([43; 32]),
|
||||
source_identity: SourceIdentity::new(b"synthetic-live-owner".to_vec()),
|
||||
selected_manifest_digest: SelectedManifestDigest::new([44; 32]),
|
||||
policy_id: PolicyId::new([45; 32]),
|
||||
id_map_digest: IdMapDigest::new([46; 32]),
|
||||
stage_path: HostPath::new(stage_relative.as_os_str().as_bytes().to_vec()),
|
||||
stage_state: PrepareStageState::Intent,
|
||||
device: None,
|
||||
inode: None,
|
||||
};
|
||||
let second_blob = first_request
|
||||
.layout_path
|
||||
.join("blobs/sha256")
|
||||
.join(hex(&Sha256::digest(&second_layer)));
|
||||
let (_, first_store) = fixture.independent_store();
|
||||
let (second_catalog, second_store) = fixture.independent_store();
|
||||
let first_handle = thread::spawn(move || first_store.prepare(&first_request));
|
||||
let owned = PrepareOperation {
|
||||
stage_state: PrepareStageState::Owned,
|
||||
device: Some(DeviceId::new(metadata.dev())),
|
||||
inode: Some(InodeId::new(metadata.ino())),
|
||||
..intent.clone()
|
||||
};
|
||||
fixture.catalog.record_prepare(&intent).unwrap();
|
||||
fixture.catalog.bind_prepare_stage(&intent, &owned).unwrap();
|
||||
let live_owner = try_exclusive_flock(File::open(&stage_path).unwrap()).unwrap();
|
||||
let (_, recovery_store) = fixture.independent_store();
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
let operation = loop {
|
||||
if let Some(operation) = second_catalog
|
||||
assert_eq!(
|
||||
recovery_store.recover_preparations().unwrap(),
|
||||
vec![(operation_id, PrepareRecovery::PreservedLiveOwnedStage,)]
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.catalog.snapshot().unwrap().prepare_operations,
|
||||
vec![owned]
|
||||
);
|
||||
assert!(stage_path.is_dir());
|
||||
|
||||
drop(live_owner);
|
||||
assert_eq!(
|
||||
recovery_store.recover_preparations().unwrap(),
|
||||
vec![(operation_id, PrepareRecovery::RemovedOwnedStage)]
|
||||
);
|
||||
assert!(
|
||||
fixture
|
||||
.catalog
|
||||
.snapshot()
|
||||
.unwrap()
|
||||
.prepare_operations
|
||||
.into_iter()
|
||||
.find(|operation| operation.stage_state == PrepareStageState::Owned)
|
||||
{
|
||||
break operation;
|
||||
}
|
||||
assert!(
|
||||
!first_handle.is_finished(),
|
||||
"materialization finished before synchronization"
|
||||
);
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"materialization did not publish stage ownership"
|
||||
);
|
||||
thread::yield_now();
|
||||
};
|
||||
|
||||
std::fs::remove_file(&second_blob).unwrap();
|
||||
rustix::fs::mknodat(
|
||||
rustix::fs::CWD,
|
||||
&second_blob,
|
||||
rustix::fs::FileType::Fifo,
|
||||
rustix::fs::Mode::from_raw_mode(0o600),
|
||||
0,
|
||||
)
|
||||
.unwrap();
|
||||
thread::sleep(Duration::from_millis(25));
|
||||
assert!(
|
||||
!first_handle.is_finished(),
|
||||
"materializer passed the synchronization FIFO"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
second_store.recover_preparations().unwrap(),
|
||||
vec![(
|
||||
operation.operation_id,
|
||||
PrepareRecovery::PreservedLiveOwnedStage,
|
||||
)]
|
||||
);
|
||||
|
||||
let writer_path = second_blob.clone();
|
||||
let writer_bytes = second_layer.clone();
|
||||
let writer = thread::spawn(move || {
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.open(writer_path)
|
||||
.unwrap()
|
||||
.write_all(&writer_bytes)
|
||||
.unwrap();
|
||||
});
|
||||
let first = first_handle.join().unwrap().unwrap();
|
||||
writer.join().unwrap();
|
||||
std::fs::remove_file(&second_blob).unwrap();
|
||||
std::fs::write(&second_blob, &second_layer).unwrap();
|
||||
|
||||
let second = second_store.prepare(&second_request).unwrap();
|
||||
assert_eq!(first, second);
|
||||
let snapshot = second_catalog.snapshot().unwrap();
|
||||
assert_eq!(snapshot.prepared_artifacts, vec![first]);
|
||||
assert!(snapshot.prepare_operations.is_empty());
|
||||
assert!(
|
||||
fixture
|
||||
.root
|
||||
.join("stages")
|
||||
.read_dir()
|
||||
.unwrap()
|
||||
.next()
|
||||
.is_none()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(!stage_path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -501,6 +449,74 @@ fn selected_gc_is_idempotent_only_at_the_durable_reclaim_boundary() {
|
|||
assert!(fixture.store.gc_artifacts(&[id]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selected_artifact_excludes_gc_until_the_claim_is_released() {
|
||||
let fixture = Fixture::new();
|
||||
let artifact = fixture
|
||||
.store
|
||||
.prepare(&fixture.request(
|
||||
72,
|
||||
&[(
|
||||
layer(&[Entry {
|
||||
path: "payload",
|
||||
kind: EntryType::Regular,
|
||||
data: b"selected",
|
||||
target: None,
|
||||
mode: 0o644,
|
||||
}]),
|
||||
OCI_TAR,
|
||||
)],
|
||||
))
|
||||
.unwrap();
|
||||
let selected = fixture.store.select_artifact(&artifact).unwrap().unwrap();
|
||||
let (_, gc_store) = fixture.independent_store();
|
||||
|
||||
let retained = gc_store
|
||||
.gc_artifacts(&[artifact.prepared_artifact_id])
|
||||
.unwrap();
|
||||
assert_eq!(retained[0].outcome, ReclaimOutcome::RetainedBySelection);
|
||||
assert_eq!(
|
||||
fixture.catalog.snapshot().unwrap().prepared_artifacts,
|
||||
vec![artifact.clone()]
|
||||
);
|
||||
assert!(fixture.artifact_root(&artifact).exists());
|
||||
|
||||
drop(selected);
|
||||
let removed = gc_store
|
||||
.gc_artifacts(&[artifact.prepared_artifact_id])
|
||||
.unwrap();
|
||||
assert_eq!(removed[0].outcome, ReclaimOutcome::Removed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selection_rebinds_to_an_equivalent_replacement() {
|
||||
let fixture = Fixture::new();
|
||||
let bytes = layer(&[Entry {
|
||||
path: "payload",
|
||||
kind: EntryType::Regular,
|
||||
data: b"replacement",
|
||||
target: None,
|
||||
mode: 0o644,
|
||||
}]);
|
||||
let request = fixture.request(73, &[(bytes, OCI_TAR)]);
|
||||
let first = fixture.store.prepare(&request).unwrap();
|
||||
assert_eq!(
|
||||
fixture.store.reclaim(&first).unwrap(),
|
||||
ReclaimOutcome::Removed
|
||||
);
|
||||
let replacement = fixture
|
||||
.store
|
||||
.prepare(&PrepareRequest {
|
||||
operation_id: PrepareOperationId::new([74; 32]),
|
||||
..request
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let selection = fixture.store.select_artifact(&first).unwrap().unwrap();
|
||||
assert_eq!(selection.artifact(), &replacement);
|
||||
assert_ne!(selection.artifact().inode, first.inode);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn destination_layout_is_materialized_and_canonical_in_artifact_identity() {
|
||||
let fixture = Fixture::new();
|
||||
|
|
|
|||
Loading…
Reference in a new issue