Close hostile recovery and concurrency gaps

crates/lazaret-client/src/lib.rs

@@ -212,17 +212,18 @@ pub fn run<B: ClientBackend>(
     let endpoint = match backend.observe().map_err(ClientError::Backend)? {
         DomainStatus::Ready(endpoint) => endpoint,
         DomainStatus::Starting => backend.wait_until_ready()?,
-        DomainStatus::Stale => return Err(ClientError::StaleCoordinator),
-        DomainStatus::Absent => match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
-            BootstrapAcquisition::Leader => match backend.bootstrap() {
-                Ok(endpoint) => endpoint,
-                Err(ClientError::BootstrapOwnerDisconnected) => {
-                    return Err(ClientError::BootstrapOwnerDisconnected);
-                }
-                Err(error) => return Err(error),
-            },
-            BootstrapAcquisition::Waiter => backend.wait_until_ready()?,
-        },
+        DomainStatus::Stale | DomainStatus::Absent => {
+            match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
+                BootstrapAcquisition::Leader => match backend.bootstrap() {
+                    Ok(endpoint) => endpoint,
+                    Err(ClientError::BootstrapOwnerDisconnected) => {
+                        return Err(ClientError::BootstrapOwnerDisconnected);
+                    }
+                    Err(error) => return Err(error),
+                },
+                BootstrapAcquisition::Waiter => backend.wait_until_ready()?,
+            }
+        }
     };
     backend.execute(endpoint, &command.argv, stdout, stderr)
 }

crates/lazaret-client/src/production.rs

@@ -84,6 +84,9 @@ impl ProductionClientBackend {
             Ok((authority, socket_path, publication))
                 if publication_matches(configuration, &publication) =>
             {
+                if !control_socket_is_live(&socket_path) {
+                    return Ok(Discovery::Stale);
+                }
                 Ok(Discovery::Ready(Box::new(ControlEndpoint::confined(
                     socket_path,
                     publication,
@@ -177,8 +180,7 @@ impl ProductionClientBackend {
         loop {
             match self.discover().map_err(ClientError::Backend)? {
                 Discovery::Ready(endpoint) => return Ok(*endpoint),
-                Discovery::Stale => return Err(ClientError::StaleCoordinator),
-                Discovery::Absent => {}
+                Discovery::Stale | Discovery::Absent => {}
             }
             if self
                 .coordinator
@@ -231,11 +233,7 @@ impl ClientBackend for ProductionClientBackend {
             Discovery::Stale => return Ok(DomainStatus::Stale),
             Discovery::Absent => {}
         }
-        if self.domain_lock_is_available()? {
-            Ok(DomainStatus::Absent)
-        } else {
-            Ok(DomainStatus::Starting)
-        }
+        Ok(DomainStatus::Absent)
     }
 
     fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
@@ -273,8 +271,7 @@ impl ClientBackend for ProductionClientBackend {
         loop {
             match self.discover().map_err(ClientError::Backend)? {
                 Discovery::Ready(endpoint) => return Ok(*endpoint),
-                Discovery::Stale => return Err(ClientError::StaleCoordinator),
-                Discovery::Absent => {}
+                Discovery::Stale | Discovery::Absent => {}
             }
             match std::os::unix::net::UnixStream::connect(&socket) {
                 Ok(mut stream) => {
@@ -407,6 +404,10 @@ pub(crate) fn open_private_domain_lock(root: &impl AsFd) -> Result<File, Product
     Ok(file)
 }
 
+fn control_socket_is_live(socket_path: &std::path::Path) -> bool {
+    std::os::unix::net::UnixStream::connect(socket_path).is_ok()
+}
+
 fn publication_matches(
     configuration: &ResolvedDomainConfiguration,
     publication: &lazaret_core::records::ProtocolPublicationIdentity,
@@ -587,7 +588,10 @@ mod tests {
         let (server, _) = listener.accept().unwrap();
 
         assert_eq!(observed, publication);
-        drop((authority, client, server));
+        assert!(control_socket_is_live(&runtime_path.join("c.sock")));
+        let (probe, _) = listener.accept().unwrap();
+        drop((authority, client, server, probe, listener));
+        assert!(!control_socket_is_live(&runtime_path.join("c.sock")));
     }
 
     #[test]

crates/lazaret-client/tests/cli_contract.rs

@@ -177,12 +177,27 @@ fn nonzero_guest_exit_is_the_cli_status() {
 }
 
 #[test]
-fn stale_and_lost_coordinators_are_actionable_failures() {
-    let mut stale = FakeBackend {
+fn stale_domain_enters_lock_arbitration_while_live_stale_and_lost_faces_refuse() {
+    let mut unowned = FakeBackend {
         observations: [DomainStatus::Stale].into(),
+        acquisition: Some(BootstrapAcquisition::Leader),
+        bootstrap: Some(Ok(Endpoint::Published)),
         ..FakeBackend::default()
     };
-    assert!(invoke(&mut stale).unwrap_err().contains("stale"));
+    assert_eq!(invoke(&mut unowned).unwrap().0, 0);
+    assert_eq!(
+        unowned.actions,
+        ["validate", "observe", "acquire", "bootstrap", "execute"]
+    );
+
+    let mut live = FakeBackend {
+        observations: [DomainStatus::Stale].into(),
+        acquisition: Some(BootstrapAcquisition::Waiter),
+        waited: Some(Err(ClientError::StaleCoordinator)),
+        ..FakeBackend::default()
+    };
+    assert!(invoke(&mut live).unwrap_err().contains("stale"));
+    assert_eq!(live.actions, ["validate", "observe", "acquire", "wait"]);
 
     let mut lost = FakeBackend {
         observations: [DomainStatus::Ready(Endpoint::Existing)].into(),