Preserve live prepared-lower materializations during recovery

crates/lazaret-prepared-lower/tests/prepared_lower/publication.rs

@@ -75,6 +75,115 @@ fn concurrent_publication_reuses_one_immutable_artifact() {
     assert!(snapshot.prepare_operations.is_empty());
 }
 
+#[test]
+fn startup_recovery_preserves_a_stage_owned_by_a_live_materializer() {
+    let fixture = Fixture::new();
+    let first_layer = layer(&[Entry {
+        path: "large",
+        kind: EntryType::Regular,
+        data: &vec![0u8; 16 * 1024 * 1024],
+        target: None,
+        mode: 0o644,
+    }]);
+    let second_layer = layer(&[Entry {
+        path: "marker",
+        kind: EntryType::Regular,
+        data: b"complete",
+        target: None,
+        mode: 0o644,
+    }]);
+    let first_request = fixture.request(
+        42,
+        &[(first_layer, OCI_TAR), (second_layer.clone(), OCI_TAR)],
+    );
+    let second_request = PrepareRequest {
+        operation_id: PrepareOperationId::new([43; 32]),
+        ..first_request.clone()
+    };
+    let second_blob = first_request
+        .layout_path
+        .join("blobs/sha256")
+        .join(hex(&Sha256::digest(&second_layer)));
+    let (_, first_store) = fixture.independent_store();
+    let (second_catalog, second_store) = fixture.independent_store();
+    let first_handle = thread::spawn(move || first_store.prepare(&first_request));
+
+    let deadline = Instant::now() + Duration::from_secs(5);
+    let operation = loop {
+        if let Some(operation) = second_catalog
+            .snapshot()
+            .unwrap()
+            .prepare_operations
+            .into_iter()
+            .find(|operation| operation.stage_state == PrepareStageState::Owned)
+        {
+            break operation;
+        }
+        assert!(
+            !first_handle.is_finished(),
+            "materialization finished before synchronization"
+        );
+        assert!(
+            Instant::now() < deadline,
+            "materialization did not publish stage ownership"
+        );
+        thread::yield_now();
+    };
+
+    std::fs::remove_file(&second_blob).unwrap();
+    rustix::fs::mknodat(
+        rustix::fs::CWD,
+        &second_blob,
+        rustix::fs::FileType::Fifo,
+        rustix::fs::Mode::from_raw_mode(0o600),
+        0,
+    )
+    .unwrap();
+    thread::sleep(Duration::from_millis(25));
+    assert!(
+        !first_handle.is_finished(),
+        "materializer passed the synchronization FIFO"
+    );
+
+    assert_eq!(
+        second_store.recover_preparations().unwrap(),
+        vec![(
+            operation.operation_id,
+            PrepareRecovery::PreservedLiveOwnedStage,
+        )]
+    );
+
+    let writer_path = second_blob.clone();
+    let writer_bytes = second_layer.clone();
+    let writer = thread::spawn(move || {
+        std::fs::OpenOptions::new()
+            .write(true)
+            .open(writer_path)
+            .unwrap()
+            .write_all(&writer_bytes)
+            .unwrap();
+    });
+    let first = first_handle.join().unwrap().unwrap();
+    writer.join().unwrap();
+    std::fs::remove_file(&second_blob).unwrap();
+    std::fs::write(&second_blob, &second_layer).unwrap();
+
+    let second = second_store.prepare(&second_request).unwrap();
+    assert_eq!(first, second);
+    let snapshot = second_catalog.snapshot().unwrap();
+    assert_eq!(snapshot.prepared_artifacts, vec![first]);
+    assert!(snapshot.prepare_operations.is_empty());
+    assert!(
+        fixture
+            .root
+            .join("stages")
+            .read_dir()
+            .unwrap()
+            .next()
+            .is_none()
+    );
+}
+
 #[test]
 fn publication_during_reclaim_preserves_replacement_and_deletes_old_inode() {
     let fixture = Fixture::new();