Recover inactive bootstrap waiter sockets safely

crates/lazaret-host-coordinator/src/cold_start.rs

@@ -297,6 +297,7 @@ impl<'a> ColdStart<'a> {
         let waiters = RefCell::new(
             BootstrapWaiterFace::bind(
                 self.configuration,
+                &self.store,
                 self.generation.generation_id,
                 self.invoking_uid,
             )

crates/lazaret-host-coordinator/src/waiter.rs

@@ -5,7 +5,7 @@ use std::os::fd::AsFd;
 use std::os::unix::ffi::OsStringExt;
 use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
 use std::os::unix::net::{UnixListener, UnixStream};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
 use lazaret_core::codec;
 use lazaret_core::configuration::ResolvedDomainConfiguration;
@@ -18,6 +18,7 @@ use lazaret_core::records::{ProtocolVersion, Session, SessionPhase};
 use rustix::net::sockopt::socket_peercred;
 use serde::Serialize;
 
+use crate::domain_store::HostCoordinatorDomainStore;
 use crate::lifecycle::{HostCoordinatorLifecycle, LifecycleAdapter, SessionLifecycleAdapter};
 
 const MAX_FRAME_BYTES: usize = 4_096;
@@ -83,12 +84,19 @@ impl BootstrapWaiterFace {
     /// Publishes one bounded invoking-user-only waiter admission socket.
     ///
     /// # Errors
-    /// Refuses changed Runtime Domain authority or a preexisting, replaced, or non-private path.
+    /// Refuses changed Runtime Domain authority, a mismatched Domain Store, or a live, replaced, or
-    pub fn bind(
+    /// non-private waiter path. Recovers an inactive exact socket under the retained Domain lock.
+    pub(crate) fn bind(
         configuration: &ResolvedDomainConfiguration,
+        store: &HostCoordinatorDomainStore,
         generation_id: GenerationId,
         uid: u32,
     ) -> Result<Self, BootstrapWaiterError> {
+        if store.snapshot().map_err(failure)?.domain.domain_id != configuration.domain_id() {
+            return Err(failure(
+                "Domain Store authority does not match the bootstrap waiter Domain",
+            ));
+        }
         let root = File::from(
             configuration
                 .runtime_domain_root()
@@ -117,22 +125,11 @@ impl BootstrapWaiterFace {
         let socket_path = root_path.join(BOOTSTRAP_WAITER_SOCKET_NAME);
         match std::fs::symlink_metadata(&socket_path) {
             Err(error) if error.kind() == io::ErrorKind::NotFound => {}
-            Ok(_) => return Err(failure("bootstrap waiter path already exists")),
+            Ok(metadata) => recover_stale_socket(&socket_path, &metadata, uid)?,
             Err(error) => return Err(failure(error)),
         }
         let listener = UnixListener::bind(&socket_path).map_err(failure)?;
-        std::fs::set_permissions(&socket_path, std::fs::Permissions::from_mode(0o600))
+        let socket_metadata = configure_bound_socket(&listener, &socket_path, uid)?;
-            .map_err(failure)?;
-        listener.set_nonblocking(true).map_err(failure)?;
-        let socket_metadata = std::fs::symlink_metadata(&socket_path).map_err(failure)?;
-        if !socket_metadata.file_type().is_socket()
-            || socket_metadata.uid() != uid
-            || socket_metadata.mode() & 0o7777 != 0o600
-        {
-            return Err(failure(
-                "bootstrap waiter socket is not invoking-user mode 0600",
-            ));
-        }
         Ok(Self {
             root,
             root_path,
@@ -318,16 +315,80 @@ impl BootstrapWaiterFace {
     }
 }
 
+fn configure_bound_socket(
+    listener: &UnixListener,
+    socket_path: &Path,
+    uid: u32,
+) -> Result<std::fs::Metadata, BootstrapWaiterError> {
+    let initial_metadata = std::fs::symlink_metadata(socket_path).map_err(failure)?;
+    let initial_identity = FileIdentity::from_metadata(&initial_metadata);
+    listener.set_nonblocking(true).map_err(failure)?;
+    let probe = UnixStream::connect(socket_path).map_err(failure)?;
+    let (accepted, _) = listener.accept().map_err(failure)?;
+    drop((probe, accepted));
+    let configured = (|| {
+        std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
+            .map_err(failure)?;
+        let metadata = std::fs::symlink_metadata(socket_path).map_err(failure)?;
+        if FileIdentity::from_metadata(&metadata) != initial_identity
+            || !metadata.file_type().is_socket()
+            || metadata.uid() != uid
+            || metadata.mode() & 0o7777 != 0o600
+        {
+            return Err(failure(
+                "bootstrap waiter socket is not exact invoking-user mode 0600",
+            ));
+        }
+        Ok(metadata)
+    })();
+    if configured.is_err() {
+        remove_socket_if_identity(socket_path, initial_identity);
+    }
+    configured
+}
+
+fn recover_stale_socket(
+    socket_path: &Path,
+    metadata: &std::fs::Metadata,
+    uid: u32,
+) -> Result<(), BootstrapWaiterError> {
+    if !metadata.file_type().is_socket()
+        || metadata.uid() != uid
+        || metadata.mode() & 0o7777 != 0o600
+    {
+        return Err(failure("bootstrap waiter path already exists"));
+    }
+    match UnixStream::connect(socket_path) {
+        Ok(_) => {
+            return Err(failure(
+                "bootstrap waiter socket is still accepting connections",
+            ));
+        }
+        Err(error) if error.kind() == io::ErrorKind::ConnectionRefused => {}
+        Err(error) => return Err(failure(format!("probing bootstrap waiter socket: {error}"))),
+    }
+    let current = std::fs::symlink_metadata(socket_path).map_err(failure)?;
+    if FileIdentity::from_metadata(&current) != FileIdentity::from_metadata(metadata) {
+        return Err(failure(
+            "bootstrap waiter authority changed during recovery",
+        ));
+    }
+    std::fs::remove_file(socket_path).map_err(failure)
+}
+
+fn remove_socket_if_identity(socket_path: &Path, identity: FileIdentity) {
+    if std::fs::symlink_metadata(socket_path)
+        .ok()
+        .is_some_and(|metadata| FileIdentity::from_metadata(&metadata) == identity)
+    {
+        let _ = std::fs::remove_file(socket_path);
+    }
+}
+
 impl Drop for BootstrapWaiterFace {
     fn drop(&mut self) {
-        if !self.withdrawn
+        if !self.withdrawn {
-            && std::fs::symlink_metadata(&self.socket_path)
+            remove_socket_if_identity(&self.socket_path, self.socket_identity);
-                .ok()
-                .is_some_and(|metadata| {
-                    FileIdentity::from_metadata(&metadata) == self.socket_identity
-                })
-        {
-            let _ = std::fs::remove_file(&self.socket_path);
         }
     }
 }
@@ -494,6 +555,7 @@ mod tests {
             .unwrap();
         let mut face = BootstrapWaiterFace::bind(
             &configuration,
+            &store,
             GenerationId::new(1),
             rustix::process::getuid().as_raw(),
         )
@@ -580,6 +642,7 @@ mod tests {
             .unwrap();
         let mut face = BootstrapWaiterFace::bind(
             &configuration,
+            &store,
             GenerationId::new(1),
             rustix::process::getuid().as_raw(),
         )
@@ -644,6 +707,7 @@ mod tests {
             .unwrap();
         let mut face = BootstrapWaiterFace::bind(
             &configuration,
+            &store,
             GenerationId::new(1),
             rustix::process::getuid().as_raw(),
         )
@@ -659,6 +723,80 @@ mod tests {
         drop(replacement_listener);
     }
 
+    #[test]
+    fn inactive_waiter_socket_is_recovered_under_matching_domain_lock() {
+        let tree = tempfile::Builder::new()
+            .prefix("lz-")
+            .tempdir_in("/tmp")
+            .unwrap();
+        let configuration = configuration(tree.path());
+        let store = store(&configuration);
+        let socket = runtime_socket_path(&configuration);
+        let listener = UnixListener::bind(&socket).unwrap();
+        fs::set_permissions(&socket, fs::Permissions::from_mode(0o600)).unwrap();
+        drop(listener);
+
+        let face = BootstrapWaiterFace::bind(
+            &configuration,
+            &store,
+            GenerationId::new(1),
+            rustix::process::getuid().as_raw(),
+        )
+        .unwrap();
+
+        UnixStream::connect(&socket).unwrap();
+        drop(face);
+        assert!(!socket.exists());
+    }
+
+    #[test]
+    fn live_waiter_socket_is_not_replaced() {
+        let tree = tempfile::Builder::new()
+            .prefix("lz-")
+            .tempdir_in("/tmp")
+            .unwrap();
+        let configuration = configuration(tree.path());
+        let store = store(&configuration);
+        let socket = runtime_socket_path(&configuration);
+        let listener = UnixListener::bind(&socket).unwrap();
+        fs::set_permissions(&socket, fs::Permissions::from_mode(0o600)).unwrap();
+
+        assert!(
+            BootstrapWaiterFace::bind(
+                &configuration,
+                &store,
+                GenerationId::new(1),
+                rustix::process::getuid().as_raw(),
+            )
+            .is_err()
+        );
+        UnixStream::connect(&socket).unwrap();
+        drop(listener);
+    }
+
+    #[test]
+    fn non_socket_waiter_path_is_not_removed() {
+        let tree = tempfile::Builder::new()
+            .prefix("lz-")
+            .tempdir_in("/tmp")
+            .unwrap();
+        let configuration = configuration(tree.path());
+        let store = store(&configuration);
+        let socket = runtime_socket_path(&configuration);
+        fs::write(&socket, b"occupied").unwrap();
+
+        assert!(
+            BootstrapWaiterFace::bind(
+                &configuration,
+                &store,
+                GenerationId::new(1),
+                rustix::process::getuid().as_raw(),
+            )
+            .is_err()
+        );
+        assert_eq!(fs::read(&socket).unwrap(), b"occupied");
+    }
+
     fn complete_startup(lifecycle: &mut HostCoordinatorLifecycle<'_, Adapter, Sessions>) {
         lifecycle.create_trust_anchor().unwrap();
         lifecycle.start_runtime_supervisor().unwrap();
@@ -701,6 +839,17 @@ mod tests {
         )
     }
 
+    fn runtime_socket_path(configuration: &ResolvedDomainConfiguration) -> PathBuf {
+        PathBuf::from(std::ffi::OsString::from_vec(
+            configuration
+                .runtime_domain_root()
+                .canonical_label()
+                .as_bytes()
+                .to_vec(),
+        ))
+        .join(BOOTSTRAP_WAITER_SOCKET_NAME)
+    }
+
     fn configuration(root: &std::path::Path) -> ResolvedDomainConfiguration {
         let configuration_directory = root.join("configuration");
         let workspace = root.join("workspace");