Implement exact bootstrap recovery and artifact GC

crates/lazaret-state-anchor/src/lib.rs

@@ -6,28 +6,35 @@ use std::fs::File;
 use std::io::{self, Write};
 use std::os::fd::{AsFd, AsRawFd, OwnedFd};
 use std::os::unix::ffi::{OsStrExt, OsStringExt};
-use std::path::{Path, PathBuf};
+use std::path::{Component, Path, PathBuf};
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::thread;
 use std::time::{Duration, Instant};
 
 use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration};
-use lazaret_core::identity::{HostPath, MountIdentity, PreparedArtifactId};
+use lazaret_core::identity::{
+    DeviceId, HostPath, InodeId, MountIdentity, NamespaceIdentity, PreparedArtifactId,
+};
 use lazaret_core::records::{
     HostProcessOwnershipSlot, PersistedHostProcessOwnership, PreparedArtifact,
     ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
 };
-use lazaret_linux::filesystem::mount_identity;
+use lazaret_linux::filesystem::{mount_identity, namespace_identity};
+use lazaret_linux::ipc::{
+    private_seqpacket_pair, receive_descriptors, receive_packet, send_descriptors, send_packet,
+};
 use lazaret_linux::process::{
-    ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker,
+    ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, require_pidfs_magic,
 };
 use rustix::fs::{
     AtFlags, FileType, Mode, OFlags, ResolveFlags, mkdirat, openat2, renameat, unlinkat,
 };
 use rustix::io::dup;
+use rustix::thread::{ThreadNameSpaceType, move_into_thread_name_spaces};
 
 const MOUNT_TIMEOUT: Duration = Duration::from_secs(5);
 const MOUNT_POLL_INTERVAL: Duration = Duration::from_millis(2);
+const PRIVATE_RECOVERY_ROLE: &str = "--lazaret-private-state-anchor-recovery";
 static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
 pub const PROTECTED_RUNTIME_FILES: [&str; 3] = ["resolv.conf", "hosts", "hostname"];
 
@@ -181,6 +188,12 @@ impl<'configuration> StateAnchorLayout<'configuration> {
         &self.merged
     }
 
+    /// Returns the ordinary host path used as the merged mount label.
+    #[must_use]
+    pub fn merged_label(&self) -> &Path {
+        &self.merged_label
+    }
+
     #[must_use]
     pub fn runtime_files(&self) -> &Path {
         &self.runtime_files
@@ -1007,6 +1020,259 @@ pub fn unmount_after_coordinator_loss(
     Ok(())
 }
 
+/// Enters an exact live holder's private namespaces and unmounts only its recorded State Anchor.
+///
+/// # Errors
+/// Refuses descriptor transfer, pidfs, namespace, path, mount, helper, or acknowledgment drift.
+pub fn recover_private_state_anchor_mount(
+    layout: &StateAnchorLayout<'_>,
+    holder: &RetainedProcess,
+    user_namespace: NamespaceIdentity,
+    mount_namespace: NamespaceIdentity,
+    expected_mount: MountIdentity,
+) -> Result<MountIdentity, StateAnchorError> {
+    let merged = &layout.merged_label;
+    let (coordinator, helper) = private_seqpacket_pair()?;
+    send_descriptors(
+        &coordinator,
+        &[holder.as_fd(), layout.runtime_anchor.as_fd()],
+    )?;
+    let authority = holder.identity().authority();
+    let target = TargetCommand::new("/proc/self/exe")
+        .argument(PRIVATE_RECOVERY_ROLE)
+        .argument(authority.pidfs_st_dev.get().to_string())
+        .argument(authority.pidfs_st_ino.get().to_string())
+        .argument(user_namespace.device.get().to_string())
+        .argument(user_namespace.inode.get().to_string())
+        .argument(mount_namespace.device.get().to_string())
+        .argument(mount_namespace.inode.get().to_string())
+        .argument(expected_mount.mount_id.get().to_string())
+        .argument(expected_mount.device.get().to_string())
+        .argument(expected_mount.inode.get().to_string())
+        .argument(merged.as_os_str());
+    child_broker().run_transient_with_private_stdout(&target, helper)?;
+    let observed: MountIdentity = receive_packet(coordinator)?;
+    if observed == expected_mount {
+        Err(StateAnchorError::OwnershipMismatch)
+    } else {
+        Ok(observed)
+    }
+}
+
+/// Dispatches the bounded private-namespace State Anchor recovery helper.
+#[must_use]
+pub fn dispatch_private_state_anchor_recovery(arguments: &[OsString]) -> Option<io::Result<()>> {
+    if arguments
+        .first()
+        .is_none_or(|arg| arg != PRIVATE_RECOVERY_ROLE)
+    {
+        return None;
+    }
+    Some(run_private_state_anchor_recovery(arguments))
+}
+
+fn run_private_state_anchor_recovery(arguments: &[OsString]) -> io::Result<()> {
+    if arguments.len() != 11 {
+        return Err(io::Error::other(
+            "private State Anchor recovery arguments are malformed",
+        ));
+    }
+    let values = arguments[1..10]
+        .iter()
+        .map(|value| {
+            value
+                .to_str()
+                .ok_or_else(|| io::Error::other("private recovery integer is not UTF-8"))?
+                .parse::<u64>()
+                .map_err(|_| io::Error::other("private recovery integer is invalid"))
+        })
+        .collect::<io::Result<Vec<_>>>()?;
+    let expected_pidfs = (values[0], values[1]);
+    let expected_user = NamespaceIdentity {
+        device: DeviceId::new(values[2]),
+        inode: InodeId::new(values[3]),
+    };
+    let expected_mount_namespace = NamespaceIdentity {
+        device: DeviceId::new(values[4]),
+        inode: InodeId::new(values[5]),
+    };
+    let expected_mount = MountIdentity {
+        mount_id: lazaret_core::identity::MountId::new(values[6]),
+        device: DeviceId::new(values[7]),
+        inode: InodeId::new(values[8]),
+    };
+    let merged = PathBuf::from(&arguments[10]);
+    let stdout = io::stdout();
+    let mut descriptors = receive_descriptors(&stdout, 2)?;
+    let runtime_anchor = descriptors
+        .pop()
+        .ok_or_else(|| io::Error::other("runtime anchor descriptor is absent"))?;
+    let pidfd = descriptors
+        .pop()
+        .ok_or_else(|| io::Error::other("pidfd descriptor is absent"))?;
+    let mut operations = KernelPrivateRecovery {
+        pidfd,
+        runtime_anchor,
+        expected_pidfs,
+        merged,
+        reopened_parent: None,
+    };
+    let observed = execute_private_recovery(
+        &mut operations,
+        expected_user,
+        expected_mount_namespace,
+        expected_mount,
+    )?;
+    send_packet(stdout, &observed)
+}
+
+trait PrivateRecoveryOperations {
+    fn validate_pidfs(&mut self) -> io::Result<()>;
+    fn enter_user_namespace(&mut self) -> io::Result<()>;
+    fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity>;
+    fn enter_mount_namespace(&mut self) -> io::Result<()>;
+    fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity>;
+    fn verify_mount(&mut self) -> io::Result<MountIdentity>;
+    fn unmount(&mut self) -> io::Result<()>;
+    fn observe_after_unmount(&mut self) -> io::Result<MountIdentity>;
+}
+
+fn execute_private_recovery(
+    operations: &mut impl PrivateRecoveryOperations,
+    expected_user: NamespaceIdentity,
+    expected_mount_namespace: NamespaceIdentity,
+    expected_mount: MountIdentity,
+) -> io::Result<MountIdentity> {
+    operations
+        .validate_pidfs()
+        .map_err(|error| io::Error::other(format!("validate pidfs: {error}")))?;
+    operations
+        .enter_user_namespace()
+        .map_err(|error| io::Error::other(format!("enter user namespace: {error}")))?;
+    if operations
+        .observe_user_namespace()
+        .map_err(|error| io::Error::other(format!("observe user namespace: {error}")))?
+        != expected_user
+    {
+        return Err(io::Error::other("private recovery user namespace mismatch"));
+    }
+    operations
+        .enter_mount_namespace()
+        .map_err(|error| io::Error::other(format!("enter mount namespace: {error}")))?;
+    if operations
+        .observe_mount_namespace()
+        .map_err(|error| io::Error::other(format!("observe mount namespace: {error}")))?
+        != expected_mount_namespace
+    {
+        return Err(io::Error::other(
+            "private recovery mount namespace mismatch",
+        ));
+    }
+    if operations
+        .verify_mount()
+        .map_err(|error| io::Error::other(format!("verify mount: {error}")))?
+        != expected_mount
+    {
+        return Err(io::Error::other("private recovery mount identity mismatch"));
+    }
+    operations
+        .unmount()
+        .map_err(|error| io::Error::other(format!("unmount: {error}")))?;
+    let observed = operations
+        .observe_after_unmount()
+        .map_err(|error| io::Error::other(format!("observe after unmount: {error}")))?;
+    if observed == expected_mount {
+        Err(io::Error::other("private recovery mount remains present"))
+    } else {
+        Ok(observed)
+    }
+}
+
+struct KernelPrivateRecovery {
+    pidfd: OwnedFd,
+    runtime_anchor: OwnedFd,
+    expected_pidfs: (u64, u64),
+    merged: PathBuf,
+    reopened_parent: Option<OwnedFd>,
+}
+
+impl PrivateRecoveryOperations for KernelPrivateRecovery {
+    fn validate_pidfs(&mut self) -> io::Result<()> {
+        require_pidfs_magic(rustix::fs::fstatfs(&self.pidfd)?.f_type as i64)?;
+        let stat = rustix::fs::fstat(&self.pidfd)?;
+        if (stat.st_dev, stat.st_ino) == self.expected_pidfs {
+            Ok(())
+        } else {
+            Err(io::Error::other(
+                "private recovery pidfs authority mismatch",
+            ))
+        }
+    }
+
+    fn enter_user_namespace(&mut self) -> io::Result<()> {
+        move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::USER)?;
+        Ok(())
+    }
+
+    fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
+        namespace_identity(Path::new("/proc/self/ns/user"))
+    }
+
+    fn enter_mount_namespace(&mut self) -> io::Result<()> {
+        move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::MOUNT)?;
+        Ok(())
+    }
+
+    fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
+        namespace_identity(Path::new("/proc/self/ns/mnt"))
+    }
+
+    fn verify_mount(&mut self) -> io::Result<MountIdentity> {
+        let parent_path = self
+            .merged
+            .parent()
+            .ok_or_else(|| io::Error::other("private recovery merged path has no parent"))?;
+        let reopened_parent = openat2(
+            rustix::fs::CWD,
+            parent_path,
+            OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
+            Mode::empty(),
+            ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
+        )?;
+        let retained = rustix::fs::fstat(&self.runtime_anchor)?;
+        let reopened = rustix::fs::fstat(&reopened_parent)?;
+        if (retained.st_dev, retained.st_ino) != (reopened.st_dev, reopened.st_ino) {
+            return Err(io::Error::other("private recovery runtime anchor mismatch"));
+        }
+        let mounted = open_final_mount_directory(&reopened_parent, self.merged_name()?)?;
+        let identity = mount_identity(&mounted)?;
+        self.reopened_parent = Some(reopened_parent);
+        Ok(identity)
+    }
+
+    fn unmount(&mut self) -> io::Result<()> {
+        rustix::mount::unmount(&self.merged, rustix::mount::UnmountFlags::empty())?;
+        Ok(())
+    }
+
+    fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
+        let parent = self
+            .reopened_parent
+            .as_ref()
+            .ok_or_else(|| io::Error::other("private recovery parent was not verified"))?;
+        mount_identity(&open_final_mount_directory(parent, self.merged_name()?)?)
+    }
+}
+
+impl KernelPrivateRecovery {
+    fn merged_name(&self) -> io::Result<&Path> {
+        self.merged
+            .file_name()
+            .map(Path::new)
+            .ok_or_else(|| io::Error::other("private recovery merged path has no name"))
+    }
+}
+
 /// Independently verifies that no mount is attached at the exact merged path.
 ///
 /// # Errors
@@ -1015,6 +1281,22 @@ pub fn require_mount_absent(layout: &StateAnchorLayout<'_>) -> Result<(), StateA
     layout.require_unmounted()
 }
 
+/// Freshly observes absence of one exact retired mount in the caller's current namespace.
+///
+/// # Errors
+/// Refuses unreadable layout evidence or continued presence of the exact mount.
+pub fn observe_mount_absence(
+    layout: &StateAnchorLayout<'_>,
+    expected: MountIdentity,
+) -> Result<MountIdentity, StateAnchorError> {
+    let observed = layout.current_merged_identity()?;
+    if observed == expected {
+        Err(StateAnchorError::OwnershipMismatch)
+    } else {
+        Ok(observed)
+    }
+}
+
 fn create_directory(parent: impl AsFd, name: &str) -> Result<(), StateAnchorError> {
     match mkdirat(parent.as_fd(), name, Mode::from_raw_mode(0o700)) {
         Ok(()) | Err(rustix::io::Errno::EXIST) => {}
@@ -1045,6 +1327,16 @@ fn open_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchor
     Ok(descriptor)
 }
 
+fn open_final_mount_directory(parent: impl AsFd, path: &Path) -> io::Result<OwnedFd> {
+    let mut components = path.components();
+    if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() {
+        return Err(io::Error::other(
+            "final mount path is not one normal component",
+        ));
+    }
+    open_mounted_directory(parent, path).map_err(io::Error::other)
+}
+
 fn open_mounted_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchorError> {
     let descriptor = openat2(
         parent.as_fd(),
@@ -1331,3 +1623,200 @@ fn path_from_host(path: &HostPath) -> PathBuf {
 fn host_path(path: &Path) -> HostPath {
     HostPath::new(path.as_os_str().as_bytes().to_vec())
 }
+
+#[cfg(test)]
+mod recovery_tests {
+    use super::*;
+    use std::os::unix::fs::symlink;
+
+    struct FailpointRecovery {
+        fail_at: Option<usize>,
+        boundary: usize,
+        events: Vec<&'static str>,
+        user: NamespaceIdentity,
+        mount_namespace: NamespaceIdentity,
+        mount: MountIdentity,
+        observed: MountIdentity,
+    }
+
+    impl FailpointRecovery {
+        fn boundary(&mut self, name: &'static str) -> io::Result<()> {
+            self.boundary += 1;
+            self.events.push(name);
+            if self.fail_at == Some(self.boundary) {
+                Err(io::Error::other("injected recovery boundary failure"))
+            } else {
+                Ok(())
+            }
+        }
+    }
+
+    impl PrivateRecoveryOperations for FailpointRecovery {
+        fn validate_pidfs(&mut self) -> io::Result<()> {
+            self.boundary("pidfs")
+        }
+
+        fn enter_user_namespace(&mut self) -> io::Result<()> {
+            self.boundary("enter-user")
+        }
+
+        fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
+            self.boundary("observe-user")?;
+            Ok(self.user)
+        }
+
+        fn enter_mount_namespace(&mut self) -> io::Result<()> {
+            self.boundary("enter-mount")
+        }
+
+        fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
+            self.boundary("observe-mount")?;
+            Ok(self.mount_namespace)
+        }
+
+        fn verify_mount(&mut self) -> io::Result<MountIdentity> {
+            self.boundary("verify-mount")?;
+            Ok(self.mount)
+        }
+
+        fn unmount(&mut self) -> io::Result<()> {
+            self.boundary("unmount")
+        }
+
+        fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
+            self.boundary("observe-absence")?;
+            Ok(self.observed)
+        }
+    }
+
+    fn fixture(fail_at: Option<usize>) -> FailpointRecovery {
+        FailpointRecovery {
+            fail_at,
+            boundary: 0,
+            events: Vec::new(),
+            user: NamespaceIdentity {
+                device: DeviceId::new(1),
+                inode: InodeId::new(2),
+            },
+            mount_namespace: NamespaceIdentity {
+                device: DeviceId::new(3),
+                inode: InodeId::new(4),
+            },
+            mount: MountIdentity {
+                mount_id: lazaret_core::identity::MountId::new(5),
+                device: DeviceId::new(6),
+                inode: InodeId::new(7),
+            },
+            observed: MountIdentity {
+                mount_id: lazaret_core::identity::MountId::new(8),
+                device: DeviceId::new(6),
+                inode: InodeId::new(7),
+            },
+        }
+    }
+
+    #[test]
+    fn private_recovery_failpoints_refuse_before_every_later_boundary() {
+        for boundary in 1..=8 {
+            let mut operations = fixture(Some(boundary));
+            let expected_user = operations.user;
+            let expected_mount_namespace = operations.mount_namespace;
+            let expected_mount = operations.mount;
+            let result = execute_private_recovery(
+                &mut operations,
+                expected_user,
+                expected_mount_namespace,
+                expected_mount,
+            );
+            assert!(result.is_err());
+            assert_eq!(operations.events.len(), boundary);
+        }
+        let mut operations = fixture(None);
+        let expected_user = operations.user;
+        let expected_mount_namespace = operations.mount_namespace;
+        let expected_mount = operations.mount;
+        assert_eq!(
+            execute_private_recovery(
+                &mut operations,
+                expected_user,
+                expected_mount_namespace,
+                expected_mount,
+            )
+            .unwrap(),
+            operations.observed
+        );
+        assert_eq!(
+            operations.events,
+            [
+                "pidfs",
+                "enter-user",
+                "observe-user",
+                "enter-mount",
+                "observe-mount",
+                "verify-mount",
+                "unmount",
+                "observe-absence",
+            ]
+        );
+    }
+
+    #[test]
+    fn final_mount_open_refuses_intermediate_and_link_traversal() {
+        let root = std::env::temp_dir().join(format!(
+            "lazaret-final-mount-open-{}-{}",
+            std::process::id(),
+            TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
+        ));
+        std::fs::create_dir_all(root.join("nested/final")).unwrap();
+        symlink("nested", root.join("link")).unwrap();
+        let parent = File::open(&root).unwrap();
+        let descriptor = File::open(root.join("nested")).unwrap();
+        symlink(
+            format!("/proc/self/fd/{}", descriptor.as_raw_fd()),
+            root.join("magic"),
+        )
+        .unwrap();
+
+        assert!(open_final_mount_directory(&parent, Path::new("nested/final")).is_err());
+        assert!(open_final_mount_directory(&parent, Path::new("link")).is_err());
+        assert!(open_final_mount_directory(&parent, Path::new("magic")).is_err());
+        assert!(open_final_mount_directory(&parent, Path::new("nested")).is_ok());
+
+        std::fs::remove_dir_all(root).unwrap();
+    }
+
+    #[test]
+    fn private_recovery_refuses_wrong_or_absent_final_mount_before_action() {
+        for observed_mount_id in [4, 8] {
+            let mut operations = fixture(None);
+            operations.mount.mount_id = lazaret_core::identity::MountId::new(observed_mount_id);
+            let expected_user = operations.user;
+            let expected_mount_namespace = operations.mount_namespace;
+            let expected_mount = MountIdentity {
+                mount_id: lazaret_core::identity::MountId::new(5),
+                device: DeviceId::new(6),
+                inode: InodeId::new(7),
+            };
+            assert!(
+                execute_private_recovery(
+                    &mut operations,
+                    expected_user,
+                    expected_mount_namespace,
+                    expected_mount,
+                )
+                .is_err()
+            );
+            assert_eq!(
+                operations.events,
+                [
+                    "pidfs",
+                    "enter-user",
+                    "observe-user",
+                    "enter-mount",
+                    "observe-mount",
+                    "verify-mount",
+                ]
+            );
+        }
+    }
+}