Close hostile recovery and concurrency gaps

crates/lazaret-client/src/lib.rs

@@ -212,8 +212,8 @@ pub fn run<B: ClientBackend>(
     let endpoint = match backend.observe().map_err(ClientError::Backend)? {
         DomainStatus::Ready(endpoint) => endpoint,
         DomainStatus::Starting => backend.wait_until_ready()?,
-        DomainStatus::Stale => return Err(ClientError::StaleCoordinator),
+        DomainStatus::Stale | DomainStatus::Absent => {
-        DomainStatus::Absent => match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
+            match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
                 BootstrapAcquisition::Leader => match backend.bootstrap() {
                     Ok(endpoint) => endpoint,
                     Err(ClientError::BootstrapOwnerDisconnected) => {
@@ -222,7 +222,8 @@ pub fn run<B: ClientBackend>(
                     Err(error) => return Err(error),
                 },
                 BootstrapAcquisition::Waiter => backend.wait_until_ready()?,
-        },
+            }
+        }
     };
     backend.execute(endpoint, &command.argv, stdout, stderr)
 }

crates/lazaret-client/src/production.rs

@@ -84,6 +84,9 @@ impl ProductionClientBackend {
             Ok((authority, socket_path, publication))
                 if publication_matches(configuration, &publication) =>
             {
+                if !control_socket_is_live(&socket_path) {
+                    return Ok(Discovery::Stale);
+                }
                 Ok(Discovery::Ready(Box::new(ControlEndpoint::confined(
                     socket_path,
                     publication,
@@ -177,8 +180,7 @@ impl ProductionClientBackend {
         loop {
             match self.discover().map_err(ClientError::Backend)? {
                 Discovery::Ready(endpoint) => return Ok(*endpoint),
-                Discovery::Stale => return Err(ClientError::StaleCoordinator),
+                Discovery::Stale | Discovery::Absent => {}
-                Discovery::Absent => {}
             }
             if self
                 .coordinator
@@ -231,11 +233,7 @@ impl ClientBackend for ProductionClientBackend {
             Discovery::Stale => return Ok(DomainStatus::Stale),
             Discovery::Absent => {}
         }
-        if self.domain_lock_is_available()? {
         Ok(DomainStatus::Absent)
-        } else {
-            Ok(DomainStatus::Starting)
-        }
     }
 
     fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
@@ -273,8 +271,7 @@ impl ClientBackend for ProductionClientBackend {
         loop {
             match self.discover().map_err(ClientError::Backend)? {
                 Discovery::Ready(endpoint) => return Ok(*endpoint),
-                Discovery::Stale => return Err(ClientError::StaleCoordinator),
+                Discovery::Stale | Discovery::Absent => {}
-                Discovery::Absent => {}
             }
             match std::os::unix::net::UnixStream::connect(&socket) {
                 Ok(mut stream) => {
@@ -407,6 +404,10 @@ pub(crate) fn open_private_domain_lock(root: &impl AsFd) -> Result<File, Product
     Ok(file)
 }
 
+fn control_socket_is_live(socket_path: &std::path::Path) -> bool {
+    std::os::unix::net::UnixStream::connect(socket_path).is_ok()
+}
+
 fn publication_matches(
     configuration: &ResolvedDomainConfiguration,
     publication: &lazaret_core::records::ProtocolPublicationIdentity,
@@ -587,7 +588,10 @@ mod tests {
         let (server, _) = listener.accept().unwrap();
 
         assert_eq!(observed, publication);
-        drop((authority, client, server));
+        assert!(control_socket_is_live(&runtime_path.join("c.sock")));
+        let (probe, _) = listener.accept().unwrap();
+        drop((authority, client, server, probe, listener));
+        assert!(!control_socket_is_live(&runtime_path.join("c.sock")));
     }
 
     #[test]

crates/lazaret-client/tests/cli_contract.rs

@@ -177,12 +177,27 @@ fn nonzero_guest_exit_is_the_cli_status() {
 }
 
 #[test]
-fn stale_and_lost_coordinators_are_actionable_failures() {
+fn stale_domain_enters_lock_arbitration_while_live_stale_and_lost_faces_refuse() {
-    let mut stale = FakeBackend {
+    let mut unowned = FakeBackend {
         observations: [DomainStatus::Stale].into(),
+        acquisition: Some(BootstrapAcquisition::Leader),
+        bootstrap: Some(Ok(Endpoint::Published)),
         ..FakeBackend::default()
     };
-    assert!(invoke(&mut stale).unwrap_err().contains("stale"));
+    assert_eq!(invoke(&mut unowned).unwrap().0, 0);
+    assert_eq!(
+        unowned.actions,
+        ["validate", "observe", "acquire", "bootstrap", "execute"]
+    );
+
+    let mut live = FakeBackend {
+        observations: [DomainStatus::Stale].into(),
+        acquisition: Some(BootstrapAcquisition::Waiter),
+        waited: Some(Err(ClientError::StaleCoordinator)),
+        ..FakeBackend::default()
+    };
+    assert!(invoke(&mut live).unwrap_err().contains("stale"));
+    assert_eq!(live.actions, ["validate", "observe", "acquire", "wait"]);
 
     let mut lost = FakeBackend {
         observations: [DomainStatus::Ready(Endpoint::Existing)].into(),

crates/lazaret-core/src/configuration.rs

@@ -1597,7 +1597,10 @@ fn ensure_owned_directory(path: &Path, uid: u32) -> Result<PathBuf, Configuratio
     let mut current = open_absolute_directory(&existing_canonical)?;
     let mut canonical = existing_canonical;
     for component in missing.into_iter().rev() {
-        rustix::fs::mkdirat(&current, &component, Mode::RWXU).map_err(io_error)?;
+        match rustix::fs::mkdirat(&current, &component, Mode::RWXU) {
+            Ok(()) | Err(rustix::io::Errno::EXIST) => {}
+            Err(error) => return Err(io_error(error)),
+        }
         let next = openat2(
             &current,
             &component,

crates/lazaret-core/tests/configuration.rs

@@ -3,6 +3,7 @@ use std::fs;
 use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Barrier};
 
 use lazaret_core::codec::{CodecError, decode, encode};
 use lazaret_core::configuration::{
@@ -645,6 +646,31 @@ fn volume_symlinks_and_lazaret_owned_mode_drift_are_rejected() {
     );
 }
 
+#[test]
+fn concurrent_loaders_share_exact_owned_directory_creation() {
+    let tree = TestTree::new();
+    let config = write_configuration(&tree, &valid_configuration(&tree));
+    let barrier = Arc::new(Barrier::new(8));
+    std::thread::scope(|scope| {
+        let handles = (0..8)
+            .map(|_| {
+                let barrier = Arc::clone(&barrier);
+                let config = config.clone();
+                let tree = &tree;
+                scope.spawn(move || {
+                    let mut environment = BTreeMap::new();
+                    environment.insert("HOST_TOKEN".into(), "value".into());
+                    barrier.wait();
+                    load_trusted_configuration(request(tree, config, environment))
+                })
+            })
+            .collect::<Vec<_>>();
+        for handle in handles {
+            handle.join().unwrap().unwrap();
+        }
+    });
+}
+
 #[test]
 fn configuration_source_permissions_links_and_size_are_bounded() {
     for mutation in 0..3 {

crates/lazaret-host-coordinator/src/client_bootstrap.rs

@@ -183,7 +183,7 @@ fn serve_published(
         loop {
             while let Ok((connection_id, result)) = completions.try_recv() {
                 active.remove(&connection_id);
-                connection_completion(result, false)?;
+                connection_completion(result)?;
             }
             require_escalation_running(&escalation_completions, escalation_enabled)?;
             let face = {
@@ -196,7 +196,7 @@ fn serve_published(
                     }
                     drop(lifecycle);
                     while !active.is_empty() {
-                        wait_for_connection_completion(&completions, &mut active, true)?;
+                        wait_for_connection_completion(&completions, &mut active)?;
                     }
                     if let Some(escalation) = &escalation_handle {
                         escalation.stop();
@@ -222,7 +222,7 @@ fn serve_published(
                 face
             };
             if active.len() == MAX_CONCURRENT_CONNECTIONS {
-                wait_for_connection_completion(&completions, &mut active, false)?;
+                wait_for_connection_completion(&completions, &mut active)?;
                 continue;
             }
             match face.accept_connection() {
@@ -299,24 +299,19 @@ fn require_escalation_running(
 fn wait_for_connection_completion(
     completions: &std::sync::mpsc::Receiver<(u64, Result<(), ControlError>)>,
     active: &mut BTreeMap<u64, std::os::unix::net::UnixStream>,
-    cancellation_requested: bool,
 ) -> Result<(), ProductionColdStartError> {
     let (connection_id, result) = completions
         .recv()
         .map_err(ProductionColdStartError::service)?;
     active.remove(&connection_id);
-    connection_completion(result, cancellation_requested)
+    connection_completion(result)
 }
 
-fn connection_completion(
+fn connection_completion(result: Result<(), ControlError>) -> Result<(), ProductionColdStartError> {
-    result: Result<(), ControlError>,
-    cancellation_requested: bool,
-) -> Result<(), ProductionColdStartError> {
     match result {
         Ok(()) => Ok(()),
         Err(ControlError::Io(error))
-            if cancellation_requested
+            if matches!(
-                && matches!(
                 error.kind(),
                 std::io::ErrorKind::UnexpectedEof
                     | std::io::ErrorKind::ConnectionReset
@@ -583,30 +578,23 @@ mod tests {
     }
 
     #[test]
-    fn withdrawal_suppresses_only_cancelled_socket_io() {
+    fn client_disconnects_and_withdrawal_are_connection_local() {
         assert!(
-            connection_completion(
+            connection_completion(Err(ControlError::Io(std::io::Error::new(
-                Err(ControlError::Io(std::io::Error::new(
                 std::io::ErrorKind::UnexpectedEof,
                 "withdrawn socket",
-                ))),
+            ))))
-                true,
-            )
             .is_ok()
         );
         assert!(
-            connection_completion(
+            connection_completion(Err(ControlError::Io(std::io::Error::new(
-                Err(ControlError::Io(std::io::Error::new(
                 std::io::ErrorKind::UnexpectedEof,
                 "ordinary socket failure",
-                ))),
+            ))))
-                false,
+            .is_ok()
-            )
-            .is_err()
         );
         assert!(
-            connection_completion(Err(ControlError::Lifecycle("teardown failed".into())), true,)
+            connection_completion(Err(ControlError::Lifecycle("teardown failed".into()))).is_err()
-                .is_err()
         );
     }
 }

crates/lazaret-host-coordinator/src/control.rs

@@ -2,10 +2,10 @@ use std::collections::BTreeMap;
 use std::ffi::{OsStr, OsString};
 use std::fmt;
 use std::fs::{File, OpenOptions};
-use std::io::{self, Read, Seek, SeekFrom, Write};
+use std::io::{self, Read, Write};
 use std::os::fd::{AsFd, AsRawFd};
 use std::os::unix::ffi::OsStringExt;
-use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
+use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt, PermissionsExt};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex, MutexGuard};
 
@@ -1743,10 +1743,8 @@ impl PublishedControlFace {
                 "publication objects changed identity, ownership, or mode",
             ));
         }
-        let mut exact_identity = self.identity_file.try_clone()?;
+        let mut bytes = vec![0; self.publication_bytes.len()];
-        exact_identity.seek(SeekFrom::Start(0))?;
+        self.identity_file.read_exact_at(&mut bytes, 0)?;
-        let mut bytes = Vec::new();
-        exact_identity.read_to_end(&mut bytes)?;
         if bytes != self.publication_bytes {
             return Err(ControlError::InvalidPublication(
                 "publication identity bytes changed",

crates/lazaret-host-coordinator/src/recovery.rs

@@ -630,7 +630,9 @@ where
 {
     recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
     failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
-    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
+    let snapshot = store
+        .snapshot()
+        .map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
     if snapshot.domain.domain_id != configuration.domain_id() {
         return Err("recovery Domain differs from resolved configuration".into());
     }
@@ -651,7 +653,13 @@ where
             &mut failpoint,
             &mut open,
             &mut delete,
-        )?;
+        )
+        .map_err(|error| {
+            format!(
+                "generation {} recovery failed: {error}",
+                generation.generation_id.get()
+            )
+        })?;
     }
     failpoint(
         ProductionRecoveryBoundary::PrepareOperations,
@@ -725,26 +733,34 @@ where
     if !session_updates.is_empty() {
         store
             .apply(&session_updates)
-            .map_err(|error| error.to_string())?;
+            .map_err(|error| format!("Session failure update failed: {error}"))?;
     }
     let participants = store
         .snapshot()
-        .map_err(|error| error.to_string())?
+        .map_err(|error| format!("Participant snapshot failed: {error}"))?
         .participants;
-    let ownership = runtime_recovery_ownership(&participants, generation)?;
+    let ownership = runtime_recovery_ownership(&participants, generation)
-    let mut owners = seal_runtime_owners(&ownership, open).map_err(|error| error.to_string())?;
+        .map_err(|error| format!("runtime ownership inventory failed: {error}"))?;
+    let mut owners = seal_runtime_owners(&ownership, open)
+        .map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
 
     failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
-    recover_publication(configuration, store, generation_id)?;
+    recover_publication(configuration, store, generation_id)
+        .map_err(|error| format!("Publication recovery failed: {error}"))?;
 
     failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
-    reconcile_runtime_before_state_anchor(&mut owners).map_err(|error| error.to_string())?;
+    reconcile_runtime_before_state_anchor(&mut owners)
-    require_sandbox_absent(configuration, store, generation_id, delete)?;
+        .map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
-    clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)?;
+    require_sandbox_absent(configuration, store, generation_id, delete)
+        .map_err(|error| format!("Sandbox absence failed: {error}"))?;
+    clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
+        .map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
 
     failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
-    clean_role_after_absence(store, generation_id, ParticipantRole::Helper)?;
+    clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
-    clean_role_after_absence(store, generation_id, ParticipantRole::Network)?;
+        .map_err(|error| format!("Helper cleanup failed: {error}"))?;
+    clean_role_after_absence(store, generation_id, ParticipantRole::Network)
+        .map_err(|error| format!("Network cleanup failed: {error}"))?;
 
     failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
     require_mount_absent(
@@ -753,13 +769,16 @@ where
         "Git-routing",
     )
     .map_err(|error| error.to_string())?;
-    clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)?;
+    clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
+        .map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
 
     failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
-    recover_state_anchor(configuration, store, catalog, generation, &mut owners)?;
+    recover_state_anchor(configuration, store, catalog, generation, &mut owners)
+        .map_err(|error| format!("State Anchor recovery failed: {error}"))?;
 
     failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
-    recover_partial_prepared_lower(configuration, store, catalog, generation_id)?;
+    recover_partial_prepared_lower(configuration, store, catalog, generation_id)
+        .map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
 
     let snapshot = store.snapshot().map_err(|error| error.to_string())?;
     if snapshot.participants.iter().any(|participant| {

crates/lazaret-host-coordinator/src/runtime.rs

@@ -946,6 +946,17 @@ pub struct CommittedSandboxRuntime {
 }
 
 impl CommittedSandboxRuntime {
+    fn receive_runtime_event(&mut self) -> Result<SupervisorEvent, SandboxCoordinatorError> {
+        loop {
+            let event = self.control.receive()?;
+            if event == SupervisorEvent::HelperExited {
+                self.network_evidence = None;
+            } else {
+                return Ok(event);
+            }
+        }
+    }
+
     #[must_use]
     pub const fn network_evidence(&self) -> Option<&NetworkReadinessEvidence> {
         self.network_evidence.as_ref()
@@ -984,11 +995,11 @@ impl CommittedSandboxRuntime {
     /// # Errors
     /// Returns an error if the RS cannot apply the exact retained-runsc pidfd action.
     pub fn signal_attached_runsc(
-        &self,
+        &mut self,
         signal: AttachedRunscSignal,
     ) -> Result<(), SandboxCoordinatorError> {
         self.control.signal_attached_runsc(signal)?;
-        if self.control.receive()? == SupervisorEvent::SignalApplied(signal) {
+        if self.receive_runtime_event()? == SupervisorEvent::SignalApplied(signal) {
             Ok(())
         } else {
             Err(SandboxCoordinatorError::InvalidState(
@@ -1007,7 +1018,7 @@ impl CommittedSandboxRuntime {
     ) -> Result<(), SandboxCoordinatorError> {
         coordinator.begin_release(self.generation_id)?;
         self.control.stop_sandbox()?;
-        if self.control.receive()? != SupervisorEvent::SandboxStopped {
+        if self.receive_runtime_event()? != SupervisorEvent::SandboxStopped {
             return Err(SandboxCoordinatorError::InvalidState(
                 "Runtime Supervisor did not stop the sandbox",
             ));
@@ -1027,7 +1038,7 @@ impl CommittedSandboxRuntime {
             .begin_release(self.generation_id)
             .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
         self.control.stop_git_routing()?;
-        match self.control.receive()? {
+        match self.receive_runtime_event()? {
             SupervisorEvent::GitRoutingStopped { mount } if mount == expected => coordinator
                 .complete_clean(self.generation_id, mount)
                 .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error))),
@@ -1046,7 +1057,7 @@ impl CommittedSandboxRuntime {
     /// Refuses an unexpected RS response or channel failure.
     pub fn stop_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
         self.control.stop_sidecar()?;
-        if self.control.receive()? == SupervisorEvent::SidecarStopped {
+        if self.receive_runtime_event()? == SupervisorEvent::SidecarStopped {
             self.network_evidence = None;
             Ok(())
         } else {
@@ -1065,7 +1076,7 @@ impl CommittedSandboxRuntime {
         expected_mount: MountIdentity,
     ) -> Result<(), SandboxCoordinatorError> {
         self.control.unmount_state_anchor()?;
-        match self.control.receive()? {
+        match self.receive_runtime_event()? {
             SupervisorEvent::StateAnchorUnmounted { mount } if mount == expected_mount => Ok(()),
             _ => Err(SandboxCoordinatorError::InvalidState(
                 "Runtime Supervisor did not confirm the exact State Anchor unmount",
@@ -1085,7 +1096,7 @@ impl CommittedSandboxRuntime {
     ) -> Result<StateAnchorAbsenceObservation, SandboxCoordinatorError> {
         self.control
             .observe_state_anchor_absence(user_namespace, mount_namespace, mount)?;
-        match self.control.receive()? {
+        match self.receive_runtime_event()? {
             SupervisorEvent::StateAnchorAbsenceObserved {
                 user_namespace: observed_user_namespace,
                 mount_namespace: observed_mount_namespace,
@@ -1118,7 +1129,7 @@ impl CommittedSandboxRuntime {
     /// Refuses an unexpected response or an RS that does not exit.
     pub fn graceful_exit(mut self) -> Result<(), SandboxCoordinatorError> {
         self.control.exit()?;
-        if self.control.receive()? != SupervisorEvent::Exited {
+        if self.receive_runtime_event()? != SupervisorEvent::Exited {
             return Err(SandboxCoordinatorError::InvalidState(
                 "Runtime Supervisor did not acknowledge graceful exit",
             ));
@@ -1132,7 +1143,7 @@ impl CommittedSandboxRuntime {
     /// # Errors
     /// Refuses any other event or an RS that does not then exit.
     pub fn wait_for_attached_runsc_exit(mut self) -> Result<(), SandboxCoordinatorError> {
-        if self.control.receive()? != SupervisorEvent::AttachedRunscExited {
+        if self.receive_runtime_event()? != SupervisorEvent::AttachedRunscExited {
             return Err(SandboxCoordinatorError::InvalidState(
                 "Runtime Supervisor did not report attached-runsc loss",
             ));
@@ -1160,7 +1171,7 @@ impl CommittedSandboxRuntime {
     ) -> Result<(), SandboxCoordinatorError> {
         coordinator.begin_release(self.generation_id)?;
         self.control.stop()?;
-        if self.control.receive()? != SupervisorEvent::Stopped {
+        if self.receive_runtime_event()? != SupervisorEvent::Stopped {
             return Err(SandboxCoordinatorError::InvalidState(
                 "Runtime Supervisor did not complete sandbox stop",
             ));

crates/lazaret-host-coordinator/tests/control.rs

@@ -516,6 +516,22 @@ fn duplicate_session_cli_and_anchor_identities_are_refused_while_registered() {
     first_server.join().unwrap().unwrap();
 }
 
+#[test]
+fn concurrent_connections_validate_publication_without_shared_file_offsets() {
+    let directory = TestDirectory::new("concurrent-publication");
+    let identity = generation_identity(7);
+    let (face, _store) = publish(&directory, &identity);
+    let clients = (40..56)
+        .map(|value| {
+            let face = Arc::clone(&face);
+            thread::spawn(move || enroll(&face, ClientId::new(bytes(value))))
+        })
+        .collect::<Vec<_>>();
+    for client in clients {
+        client.join().unwrap();
+    }
+}
+
 #[test]
 fn anonymous_connections_are_enrollment_only_bounded_and_exact_alpn_is_required() {
     let directory = TestDirectory::new("anonymous");

crates/lazaret-host-coordinator/tests/runtime-composition.rs

@@ -753,7 +753,7 @@ fn cross_bundle_handoff_refuses_target_work() -> Result<(), MatrixError> {
 
 fn normal_stop_and_stop_resume() -> Result<(), MatrixError> {
     let fixture = Fixture::new()?;
-    let runtime = fixture.launch_committed()?;
+    let mut runtime = fixture.launch_committed()?;
     wait_for_activity(&fixture.activity)?;
     runtime
         .signal_attached_runsc(AttachedRunscSignal::Stop)
@@ -789,7 +789,7 @@ fn normal_stop_and_stop_resume() -> Result<(), MatrixError> {
 
 fn attached_runsc_loss_stops_guest() -> Result<(), MatrixError> {
     let fixture = Fixture::new()?;
-    let runtime = fixture.launch_committed()?;
+    let mut runtime = fixture.launch_committed()?;
     wait_for_activity(&fixture.activity)?;
     runtime
         .signal_attached_runsc(AttachedRunscSignal::Kill)

crates/lazaret-linux/src/process.rs

@@ -1070,12 +1070,12 @@ where
         return Ok(PidfsRecoveryMatch::BootMismatch);
     }
     let host_pid = identity.host_pid().get();
-    let pidfd = open(host_pid)?;
+    let pidfd = normalize_process_absence(open(host_pid))?;
     let observed = inspect(pidfd.as_fd(), current_boot)?;
     if observed != identity.authority() {
         return Err(io::Error::other("pidfs process authority mismatch"));
     }
-    if start(host_pid)? != identity.start_time_ticks().get() {
+    if normalize_process_absence(start(host_pid))? != identity.start_time_ticks().get() {
         return Err(io::Error::other("process start time mismatch"));
     }
     Ok(PidfsRecoveryMatch::Matched(RetainedProcess {
@@ -1084,6 +1084,16 @@ where
     }))
 }
 
+fn normalize_process_absence<T>(result: io::Result<T>) -> io::Result<T> {
+    result.map_err(|error| {
+        if error.raw_os_error() == Some(rustix::io::Errno::NOENT.raw_os_error()) {
+            io::Error::from_raw_os_error(rustix::io::Errno::SRCH.raw_os_error())
+        } else {
+            error
+        }
+    })
+}
+
 /// # Errors
 /// Refuses if arming fails or the exact parent changes during the recheck.
 pub fn arm_parent_death_and_recheck(
@@ -1745,6 +1755,50 @@ mod tests {
         }
     }
 
+    #[test]
+    fn recovery_normalizes_vanished_proc_evidence_to_esrch() {
+        let authority = PidfsProcessAuthorityV1 {
+            boot_id: BootId::new([1; 16]),
+            pidfs_st_dev: PidfsDeviceId::new(7),
+            pidfs_st_ino: PidfsInodeId::new(9),
+        };
+        let identity = PidfsProcessIdentityV1::new(
+            authority,
+            HostPidLocator::new(41),
+            ProcessStartTimeTicks::new(43),
+        );
+        for missing in ["open", "start"] {
+            let result = match_recovery_process_with(
+                identity,
+                || Ok(authority.boot_id),
+                |_| {
+                    if missing == "open" {
+                        Err(io::Error::from_raw_os_error(
+                            rustix::io::Errno::NOENT.raw_os_error(),
+                        ))
+                    } else {
+                        Ok(File::open("/dev/null").unwrap().into())
+                    }
+                },
+                |_, _| Ok(authority),
+                |_| {
+                    if missing == "start" {
+                        Err(io::Error::from_raw_os_error(
+                            rustix::io::Errno::NOENT.raw_os_error(),
+                        ))
+                    } else {
+                        Ok(43)
+                    }
+                },
+            )
+            .unwrap_err();
+            assert_eq!(
+                result.raw_os_error(),
+                Some(rustix::io::Errno::SRCH.raw_os_error())
+            );
+        }
+    }
+
     #[test]
     fn recovery_action_reconciles_boot_without_process_action() {
         let mut old_boot = *read_boot_id().unwrap().as_bytes();

crates/lazaret-linux/tests/capability-drop-helper.rs

@@ -8,42 +8,55 @@ use rustix::thread::{
 #[allow(clippy::disallowed_methods, clippy::disallowed_types)]
 fn main() -> ExitCode {
     if std::env::var_os("LAZARET_CAPABILITY_DROP_ISOLATED").is_none() {
-        let before = lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN)
+        let isolated = match std::process::Command::new("unshare")
-            .expect("inspect unprivileged capability state");
-        if lazaret_linux::capability::irreversibly_drop(CapabilitySet::NET_ADMIN).is_err()
-            || lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN).ok() != Some(before)
-        {
-            return ExitCode::FAILURE;
-        }
-        return match std::process::Command::new("unshare")
             .args(["--user", "--map-root-user"])
             .arg(std::env::current_exe().expect("test executable path"))
             .env("LAZARET_CAPABILITY_DROP_ISOLATED", "1")
             .output()
         {
-            Ok(output) if output.status.success() => ExitCode::SUCCESS,
+            Ok(output) if output.status.success() => true,
             Ok(output)
                 if String::from_utf8_lossy(&output.stderr).contains("Operation not permitted") =>
             {
                 eprintln!("SKIP isolated capability-positive branch: user namespaces unavailable");
-                ExitCode::SUCCESS
+                true
             }
             Ok(output) => {
                 eprintln!(
                     "isolated capability-positive branch failed: {}",
                     String::from_utf8_lossy(&output.stderr)
                 );
-                ExitCode::FAILURE
+                false
             }
             Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
                 eprintln!("SKIP isolated capability-positive branch: unshare is unavailable");
-                ExitCode::SUCCESS
+                true
             }
             Err(error) => {
                 eprintln!("failed to run isolated capability-positive branch: {error}");
-                ExitCode::FAILURE
+                false
             }
         };
+        if !isolated {
+            return ExitCode::FAILURE;
+        }
+        let before = lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN)
+            .expect("inspect unprivileged capability state");
+        return match lazaret_linux::capability::irreversibly_drop(CapabilitySet::NET_ADMIN) {
+            Ok(())
+                if lazaret_linux::capability::require_regain_refused(CapabilitySet::NET_ADMIN)
+                    .is_ok() =>
+            {
+                ExitCode::SUCCESS
+            }
+            Err(_)
+                if lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN).ok()
+                    == Some(before) =>
+            {
+                ExitCode::SUCCESS
+            }
+            Ok(()) | Err(_) => ExitCode::FAILURE,
+        };
     }
 
     let capability = CapabilitySet::NET_ADMIN;

deny.toml

@@ -6,6 +6,7 @@ multiple-versions = "deny"
 wildcards = "deny"
 skip = [
     { name = "getrandom", version = "0.2.17", reason = "oci-client authentication uses jsonwebtoken's current getrandom line" },
+    { name = "nix", version = "0.30.1", reason = "fuser 0.17.0 uses its current nix line while Lazaret directly uses 0.31.3" },
     { name = "oci-spec", version = "0.9.0", reason = "oci-client 0.17.0 uses its current OCI types while the runtime supervisor directly uses 0.10.0" },
     { name = "untrusted", version = "0.7.1", reason = "oci-client rustls authentication uses aws-lc-rs's compatibility line" },
 ]

tests/tooling/test_verify.py

@@ -119,6 +119,20 @@ class VerificationTests(unittest.TestCase):
         self.assertFalse(VERIFY.license_permitted("GPL-3.0-only", allowed))
         self.assertFalse(VERIFY.license_permitted("MIT AND GPL-3.0-only", allowed))
 
+    def test_only_the_exact_materialized_fuser_source_is_approved(self) -> None:
+        package = {
+            "name": "fuser",
+            "version": "0.17.0",
+            "manifest_path": str(VERIFY.MATERIALIZED_FUSER_MANIFEST),
+        }
+        self.assertTrue(VERIFY.is_materialized_fuser(package))
+        self.assertFalse(VERIFY.is_materialized_fuser({**package, "version": "0.18.0"}))
+        self.assertFalse(
+            VERIFY.is_materialized_fuser(
+                {**package, "manifest_path": str(ROOT / "fuser/Cargo.toml")}
+            )
+        )
+
     def test_workspace_source_discovery_is_not_confined_to_crates(self) -> None:
         with tempfile.TemporaryDirectory(dir=ROOT) as directory:
             package_root = Path(directory)
@@ -192,8 +206,12 @@ class VerificationTests(unittest.TestCase):
                 VERIFY.audit_process_syscalls({})
 
     def test_process_syscall_audit_rejects_failed_binary(self) -> None:
-        with self.assertRaisesRegex(VERIFY.VerificationError, "exercised path exited 1"):
+        for binary, returncode in VERIFY.PROCESS_TRACE_EXIT_CODES.items():
-            VERIFY.validate_trace_exit("example", 1)
+            VERIFY.validate_trace_exit(binary, returncode)
+            with self.assertRaisesRegex(VERIFY.VerificationError, "expected"):
+                VERIFY.validate_trace_exit(binary, returncode + 1)
+        with self.assertRaisesRegex(VERIFY.VerificationError, "expected None"):
+            VERIFY.validate_trace_exit("example", 0)
 
     def test_workspace_binary_discovery_covers_every_current_target(self) -> None:
         self.assertEqual(

tools/lazaret_verify.py

@@ -14,6 +14,9 @@ import tempfile
 import tomllib
 
 ROOT = Path(__file__).resolve().parent.parent
+MATERIALIZED_FUSER_MANIFEST = (
+    ROOT / ".lazaret/vendor/fuser-0.17.0/Cargo.toml"
+).resolve()
 MODEL_CONFIGURATIONS = (
     (ROOT / "docs/protocol/Lazaret.tla", ROOT / "docs/protocol/Lazaret.cfg"),
     (ROOT / "docs/protocol/PreparedLower.tla", ROOT / "docs/protocol/PreparedLower.cfg"),
@@ -227,6 +230,16 @@ FORBIDDEN_PROCESS_SYSCALLS = tuple(
         r"^rt_sigaction\(SIGCHLD,",
     )
 )
+PROCESS_TRACE_EXIT_CODES = {
+    "lazaret": 2,
+    "lazaret-escalation-client": 2,
+    "lazaret-git-sentry": 1,
+    "lazaret-host-coordinator": 0,
+    "lazaret-ignored-workspace": 1,
+    "lazaret-launch-wrapper": 0,
+    "lazaret-runtime-supervisor": 0,
+    "lazaret-workspace-benchmark": 1,
+}
 LINT_BYPASS = re.compile(
     r"#\s*!?\s*\[[^]]*(?:allow|expect)\s*\([^]]*"
     r"(?:warnings|clippy\s*::\s*(?:all|style|restriction)|disallowed_methods|disallowed_types)"
@@ -397,7 +410,8 @@ def audit_dependencies(metadata: dict) -> None:
         if package["id"] not in workspace_members:
             source = package.get("source") or ""
             require(
-                source.startswith("registry+https://github.com/rust-lang/crates.io-index"),
+                source.startswith("registry+https://github.com/rust-lang/crates.io-index")
+                or is_materialized_fuser(package),
                 f"unapproved dependency source: {package['name']}",
             )
             license_expression = package.get("license") or ""
@@ -429,6 +443,14 @@ def audit_dependencies(metadata: dict) -> None:
     print("PASS dependency source, wildcard, and license audit")
 
 
+def is_materialized_fuser(package: dict) -> bool:
+    return (
+        package.get("name") == "fuser"
+        and package.get("version") == "0.17.0"
+        and Path(package.get("manifest_path", "")).resolve() == MATERIALIZED_FUSER_MANIFEST
+    )
+
+
 def license_permitted(expression: str, allowed: set[str]) -> bool:
     expression = re.sub(r"\s+WITH\s+[A-Za-z0-9.-]+", "", expression)
     expression = re.sub(r"\s*/\s*", " OR ", expression)
@@ -573,7 +595,11 @@ def audit_process_syscalls(metadata: dict) -> None:
 
 
 def validate_trace_exit(binary: str, returncode: int) -> None:
-    require(returncode == 0, f"{binary}: exercised path exited {returncode}")
+    expected = PROCESS_TRACE_EXIT_CODES.get(binary)
+    require(
+        returncode == expected,
+        f"{binary}: exercised path exited {returncode}, expected {expected}",
+    )
 
 
 def refresh_advisories() -> None: