Harden session terminal release authority

crates/lazaret-host-coordinator/src/sandbox_init.rs

@@ -57,15 +57,69 @@ fn run_session_exec(arguments: &[OsString]) -> io::Result<()> {
 }
 
 fn report_session_identity(arguments: &[OsString]) -> io::Result<()> {
-    let pid = match arguments {
+    match arguments {
-        [pid] => pid
+        [pid] => report_process_identity(parse_positive_pid(pid)?),
+        [mode, group] if mode == "--group" => report_group_presence(parse_positive_pid(group)?),
+        _ => Err(io::Error::other("session identity query is invalid")),
+    }
+}
+
+fn parse_positive_pid(value: &OsStr) -> io::Result<u32> {
+    value
         .to_str()
         .and_then(|value| value.parse::<u32>().ok())
-            .filter(|pid| *pid > 0)
+        .filter(|value| *value > 0)
-            .ok_or_else(|| io::Error::other("session identity PID is invalid"))?,
+        .ok_or_else(|| io::Error::other("session identity number is invalid"))
-        _ => return Err(io::Error::other("session identity requires one PID")),
+}
+
+fn report_process_identity(pid: u32) -> io::Result<()> {
+    let Some((observed_pid, start_ticks, group)) = read_process_identity(pid)? else {
+        println!("absent");
+        return Ok(());
+    };
+    println!("{observed_pid} {start_ticks} {group}");
+    Ok(())
+}
+
+fn report_group_presence(group: u32) -> io::Result<()> {
+    println!(
+        "{}",
+        if group_is_present_except(group, std::process::id())? {
+            "present"
+        } else {
+            "absent"
+        }
+    );
+    Ok(())
+}
+
+fn group_is_present_except(group: u32, observer: u32) -> io::Result<bool> {
+    for entry in std::fs::read_dir("/proc")? {
+        let entry = entry?;
+        let Some(pid) = entry
+            .file_name()
+            .to_str()
+            .and_then(|value| value.parse::<u32>().ok())
+        else {
+            continue;
+        };
+        if pid == observer {
+            continue;
+        }
+        if read_process_identity(pid)?.is_some_and(|(_, _, observed_group)| observed_group == group)
+        {
+            return Ok(true);
+        }
+    }
+    Ok(false)
+}
+
+fn read_process_identity(pid: u32) -> io::Result<Option<(u32, u64, u32)>> {
+    let stat = match std::fs::read_to_string(format!("/proc/{pid}/stat")) {
+        Ok(stat) => stat,
+        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
+        Err(error) => return Err(error),
     };
-    let stat = std::fs::read_to_string(format!("/proc/{pid}/stat"))?;
     let close = stat
         .rfind(')')
         .ok_or_else(|| io::Error::other("guest process stat is malformed"))?;
@@ -82,8 +136,7 @@ fn report_session_identity(arguments: &[OsString]) -> io::Result<()> {
         .get(19)
         .and_then(|value| value.parse::<u64>().ok())
         .ok_or_else(|| io::Error::other("guest process stat start is malformed"))?;
-    println!("{observed_pid} {start_ticks} {group}");
+    Ok(Some((observed_pid, start_ticks, group)))
-    Ok(())
 }
 
 #[cfg(test)]
@@ -125,4 +178,11 @@ mod tests {
             .is_err()
         );
     }
+
+    #[test]
+    fn group_presence_is_independent_of_a_designated_leader() {
+        let (_, _, group) = read_process_identity(std::process::id()).unwrap().unwrap();
+        assert!(group_is_present_except(group, u32::MAX).unwrap());
+        assert!(!group_is_present_except(u32::MAX, u32::MAX).unwrap());
+    }
 }

crates/lazaret-host-coordinator/src/terminal.rs

@@ -14,9 +14,9 @@ use lazaret_core::identity::{
     PidfsProcessIdentityV1, SessionId,
 };
 use lazaret_core::records::{
-    GuestProcessIdentity, ParticipantOwnership, ParticipantRole, ParticipantState,
+    DomainStore, DurableDecision, GuestProcessIdentity, ParticipantOwnership, ParticipantRole,
-    PersistedHostProcessOwnership, ProcessOwnershipCommitAck, PtyIdentity, SessionExit,
+    ParticipantState, PersistedHostProcessOwnership, ProcessOwnershipCommitAck, PtyIdentity,
-    SessionPhase, SessionTerminal, SessionTerminalEvent, SessionTerminalFields,
+    SessionExit, SessionPhase, SessionTerminal, SessionTerminalEvent, SessionTerminalFields,
     SessionTerminalMessage, SessionTerminalState, StagedProcessOwnership, TerminalOutput,
     TerminalResize, TerminalSignalKind,
 };
@@ -72,6 +72,10 @@ pub enum TerminalError {
     GuestGroupStillPresent,
     InvalidExit,
     AuthorityMismatch,
+    AcquisitionCleanup {
+        original: Box<TerminalError>,
+        cleanup: Option<Box<TerminalError>>,
+    },
 }
 
 impl std::fmt::Display for TerminalError {
@@ -100,11 +104,37 @@ impl std::fmt::Display for TerminalError {
             Self::AuthorityMismatch => {
                 formatter.write_str("Session Terminal runtime authority does not match")
             }
+            Self::AcquisitionCleanup { original, cleanup } => {
+                write!(formatter, "Session Terminal acquisition failed: {original}")?;
+                if let Some(cleanup) = cleanup {
+                    write!(formatter, "; partial terminal cleanup failed: {cleanup}")?;
+                }
+                Ok(())
+            }
         }
     }
 }
 
-impl std::error::Error for TerminalError {}
+impl std::error::Error for TerminalError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Self::Io(error) => Some(error),
+            Self::Store(error) => Some(error),
+            Self::AcquisitionCleanup { original, .. } => Some(original.as_ref()),
+            Self::UnknownSession
+            | Self::DuplicateSession
+            | Self::StaleGeneration
+            | Self::InactiveRoute
+            | Self::Releasing
+            | Self::OversizedInput
+            | Self::Backpressure
+            | Self::GuestIdentityMismatch
+            | Self::GuestGroupStillPresent
+            | Self::InvalidExit
+            | Self::AuthorityMismatch => None,
+        }
+    }
+}
 
 impl From<io::Error> for TerminalError {
     fn from(error: io::Error) -> Self {
@@ -168,7 +198,7 @@ pub struct TerminalAcquisition {
 }
 
 pub struct HostCoordinatorTerminalRecorder {
-    store: Arc<HostCoordinatorDomainStore>,
+    store: Arc<dyn TerminalDomainStore>,
 }
 
 impl HostCoordinatorTerminalRecorder {
@@ -177,22 +207,57 @@ impl HostCoordinatorTerminalRecorder {
         Self { store }
     }
 
+    #[cfg(test)]
+    fn with_store(store: Arc<dyn TerminalDomainStore>) -> Self {
+        Self { store }
+    }
+
     fn put(&self, terminal: SessionTerminal) -> Result<(), TerminalError> {
         self.store
             .apply(&[DomainStoreMutation::PutSessionTerminal(terminal)])?;
         Ok(())
     }
 
-    fn require_authority(&self, request: &TerminalAcquisition) -> Result<(), TerminalError> {
+    fn require_authority(
+        &self,
+        request: &TerminalAcquisition,
+    ) -> Result<TerminalRoute, TerminalError> {
         let snapshot = self.store.snapshot()?;
+        let generation = snapshot
+            .generations
+            .iter()
+            .find(|generation| generation.generation_id == request.generation_id);
+        let current = snapshot
+            .generations
+            .iter()
+            .all(|candidate| candidate.generation_id <= request.generation_id);
         if snapshot.domain.domain_id != request.domain_id
-            || !snapshot.generations.iter().any(|generation| {
+            || !current
-                generation.generation_id == request.generation_id
+            || generation.is_none_or(|generation| {
-                    && generation.coordinator_id == request.coordinator_id
+                generation.coordinator_id != request.coordinator_id
+                    || generation.durable_decision != DurableDecision::Published
             })
         {
             return Err(TerminalError::AuthorityMismatch);
         }
+        let route = snapshot
+            .sessions
+            .iter()
+            .find(|session| {
+                session.generation_id == request.generation_id
+                    && session.session_id == request.session_id
+            })
+            .map(|session| match session.session_phase {
+                SessionPhase::Attached => TerminalRoute::Sandbox,
+                SessionPhase::Prompting => TerminalRoute::Prompt,
+                SessionPhase::HostExecuting => TerminalRoute::HostProcess,
+                SessionPhase::New
+                | SessionPhase::Waiting
+                | SessionPhase::Registering
+                | SessionPhase::Closed
+                | SessionPhase::Failed => TerminalRoute::Detached,
+            })
+            .ok_or(TerminalError::AuthorityMismatch)?;
         let sandbox = snapshot
             .participants
             .iter()
@@ -211,7 +276,7 @@ impl HostCoordinatorTerminalRecorder {
         {
             return Err(TerminalError::AuthorityMismatch);
         }
-        Ok(())
+        Ok(route)
     }
 
     fn begin_release(&self, terminal: SessionTerminal) -> Result<(), TerminalError> {
@@ -224,11 +289,16 @@ impl HostCoordinatorTerminalRecorder {
                     && session.session_id == terminal.session_id()
             })
             .ok_or(TerminalError::UnknownSession)?;
+        let mut mutations = Vec::with_capacity(2);
+        if !matches!(
+            session.session_phase,
+            SessionPhase::Closed | SessionPhase::Failed
+        ) {
             session.session_phase = SessionPhase::Closed;
-        self.store.apply(&[
+            mutations.push(DomainStoreMutation::PutSession(session));
-            DomainStoreMutation::PutSession(session),
+        }
-            DomainStoreMutation::PutSessionTerminal(terminal),
+        mutations.push(DomainStoreMutation::PutSessionTerminal(terminal));
-        ])?;
+        self.store.apply(&mutations)?;
         Ok(())
     }
 
@@ -241,8 +311,24 @@ impl HostCoordinatorTerminalRecorder {
     }
 }
 
+trait TerminalDomainStore: Send + Sync {
+    fn snapshot(&self) -> Result<DomainStore, DomainStoreError>;
+    fn apply(&self, mutations: &[DomainStoreMutation]) -> Result<(), DomainStoreError>;
+}
+
+impl TerminalDomainStore for HostCoordinatorDomainStore {
+    fn snapshot(&self) -> Result<DomainStore, DomainStoreError> {
+        self.snapshot()
+    }
+
+    fn apply(&self, mutations: &[DomainStoreMutation]) -> Result<(), DomainStoreError> {
+        self.apply(mutations)
+    }
+}
+
 struct ActiveTerminal {
     generation_id: GenerationId,
+    request: TerminalAcquisition,
     route: TerminalRoute,
     record: SessionTerminal,
     process: Box<dyn SessionTerminalProcess>,
@@ -278,7 +364,9 @@ impl SessionTerminalEngine {
         if self.active.contains_key(&request.session_id) {
             return Err(TerminalError::DuplicateSession);
         }
-        self.recorder.require_authority(request)?;
+        if self.recorder.require_authority(request)? != TerminalRoute::Sandbox {
+            return Err(TerminalError::AuthorityMismatch);
+        }
         let intent = terminal_snapshot(
             request,
             SessionTerminalState::Intent,
@@ -293,6 +381,7 @@ impl SessionTerminalEngine {
             request.session_id,
             ActiveTerminal {
                 generation_id: request.generation_id,
+                request: request.clone(),
                 route: TerminalRoute::Sandbox,
                 record,
                 process,
@@ -311,6 +400,10 @@ impl SessionTerminalEngine {
         generation_id: GenerationId,
         route: TerminalRoute,
     ) -> Result<(), TerminalError> {
+        let authoritative = self.refresh_route(session_id, generation_id)?;
+        if authoritative != route {
+            return Err(TerminalError::InactiveRoute);
+        }
         let terminal = self.active_mut(session_id, generation_id)?;
         if terminal.releasing || route == TerminalRoute::Detached {
             return Err(TerminalError::Releasing);
@@ -335,6 +428,10 @@ impl SessionTerminalEngine {
         if matches!(event, SessionTerminalEvent::Close(_)) {
             return self.close(session_id, generation_id, None, true);
         }
+        let authoritative = self.refresh_route(session_id, generation_id)?;
+        if authoritative != TerminalRoute::Sandbox {
+            return Err(TerminalError::InactiveRoute);
+        }
         let terminal = self.active_mut(session_id, generation_id)?;
         if terminal.releasing {
             return Err(TerminalError::Releasing);
@@ -378,11 +475,12 @@ impl SessionTerminalEngine {
         session_id: SessionId,
         generation_id: GenerationId,
     ) -> Result<Vec<SessionTerminalMessage>, TerminalError> {
+        let authoritative = self.refresh_route(session_id, generation_id)?;
         let terminal = self.active_mut(session_id, generation_id)?;
         if terminal.process.exec_exited()? {
             return self.close(session_id, generation_id, None, true);
         }
-        if terminal.route != TerminalRoute::Sandbox || terminal.releasing {
+        if authoritative != TerminalRoute::Sandbox || terminal.releasing {
             return Ok(Vec::new());
         }
         flush_input(terminal)?;
@@ -430,6 +528,33 @@ impl SessionTerminalEngine {
         self.active.contains_key(&session_id)
     }
 
+    fn refresh_route(
+        &mut self,
+        session_id: SessionId,
+        generation_id: GenerationId,
+    ) -> Result<TerminalRoute, TerminalError> {
+        let request = {
+            let terminal = self.active_mut(session_id, generation_id)?;
+            terminal.request.clone()
+        };
+        match self.recorder.require_authority(&request) {
+            Ok(TerminalRoute::Detached) | Err(TerminalError::AuthorityMismatch) => {
+                let cleanup = self.close(
+                    session_id,
+                    generation_id,
+                    Some(TerminalLoss::ExecClientLost),
+                    false,
+                );
+                Err(with_cleanup(TerminalError::AuthorityMismatch, cleanup))
+            }
+            Err(error) => Err(error),
+            Ok(route) => {
+                self.active_mut(session_id, generation_id)?.route = route;
+                Ok(route)
+            }
+        }
+    }
+
     fn active_mut(
         &mut self,
         session_id: SessionId,
@@ -469,26 +594,80 @@ impl SessionTerminalEngine {
             terminal.record = release;
             self.cancellation.cancel_for_close(session_id)?;
 
-            let mut output = Vec::new();
+            let (mut output, exit) = release_physical(
-            if !matches!(loss, Some(TerminalLoss::HostCoordinatorLost)) {
+                terminal.process.as_mut(),
+                session_id,
+                connected,
+                !matches!(loss, Some(TerminalLoss::HostCoordinatorLost)),
+            )?;
+            let clean = clean_terminal(generation_id, session_id)?;
+            self.recorder.clean(clean)?;
+            if connected {
+                output.push(SessionTerminalMessage::Exit(exit_message(
+                    session_id, loss, exit,
+                )?));
+            }
+            Ok(output)
+        })();
+        if result.is_err() {
+            self.active.insert(session_id, terminal);
+        }
+        result
+    }
+}
+
+fn release_physical(
+    process: &mut dyn SessionTerminalProcess,
+    session_id: SessionId,
+    connected: bool,
+    signal_guest: bool,
+) -> Result<(Vec<SessionTerminalMessage>, ProcessExit), TerminalError> {
+    let mut failure = None;
+    if signal_guest {
         for signal in ["HUP", "TERM", "KILL"] {
-                    if terminal.process.group_absent()? {
+            match process.group_absent() {
+                Ok(true) => break,
+                Ok(false) => {}
+                Err(error) => {
+                    failure.get_or_insert(error);
                     break;
                 }
-                    terminal.process.signal_group(signal)?;
+            }
-                    wait_until(SIGNAL_GRACE, || terminal.process.group_absent())?;
+            if let Err(error) = process.signal_group(signal) {
+                failure.get_or_insert(error);
+                break;
+            }
+            if let Err(error) = wait_until(SIGNAL_GRACE, || process.group_absent()) {
+                failure.get_or_insert(error);
+                break;
             }
         }
-            if !terminal.process.exec_exited()? {
-                terminal.process.terminate_exec()?;
     }
-            let exit = terminal.process.reap_exec()?;
+    match process.exec_exited() {
+        Ok(false) => {
+            if let Err(error) = process.terminate_exec() {
+                failure.get_or_insert(error.into());
+            }
+        }
+        Ok(true) => {}
+        Err(error) => {
+            failure.get_or_insert(error.into());
+        }
+    }
+    let exit = match process.reap_exec() {
+        Ok(exit) => Some(exit),
+        Err(error) => {
+            failure.get_or_insert(error.into());
+            None
+        }
+    };
+    let mut output = Vec::new();
     if connected {
         let deadline = Instant::now() + DRAIN_DEADLINE;
         let mut bytes = [0_u8; MAX_OUTPUT_BYTES];
         let mut drained = 0;
         while Instant::now() < deadline && drained < MAX_DRAIN_BYTES {
-                    match terminal.process.read_output(&mut bytes) {
+            match process.read_output(&mut bytes) {
                 Ok(0) => break,
                 Ok(length) => {
                     let length = length.min(MAX_DRAIN_BYTES - drained);
@@ -505,23 +684,32 @@ impl SessionTerminalEngine {
             }
         }
     }
-            terminal.process.close_pty();
+    process.close_pty();
-            if !terminal.process.group_absent()? {
+    match process.group_absent() {
-                return Err(TerminalError::GuestGroupStillPresent);
+        Ok(true) => {}
+        Ok(false) => {
+            failure.get_or_insert(TerminalError::GuestGroupStillPresent);
         }
-            let clean = clean_terminal(generation_id, session_id)?;
+        Err(error) => {
-            self.recorder.clean(clean)?;
+            failure.get_or_insert(error);
-            if connected {
-                output.push(SessionTerminalMessage::Exit(exit_message(
-                    session_id, loss, exit,
-                )?));
         }
-            Ok(output)
-        })();
-        if result.is_err() {
-            self.active.insert(session_id, terminal);
     }
-        result
+    if let Some(error) = failure {
+        return Err(error);
+    }
+    Ok((
+        output,
+        exit.ok_or_else(|| io::Error::other("exec client exit is unavailable"))?,
+    ))
+}
+
+fn with_cleanup<T>(original: TerminalError, cleanup: Result<T, TerminalError>) -> TerminalError {
+    match cleanup {
+        Ok(_) => original,
+        Err(cleanup) => TerminalError::AcquisitionCleanup {
+            original: Box::new(original),
+            cleanup: Some(Box::new(cleanup)),
+        },
     }
 }
 
@@ -700,19 +888,89 @@ impl SessionTerminalFactory for RunscSessionTerminalFactory {
             PersistedHostProcessOwnership::committed(process.identity()),
             Some(guest),
         )?;
-        recorder.put(running.clone())?;
+        let mut terminal_process = RunscSessionTerminal {
-        Ok((
-            running,
-            Box::new(RunscSessionTerminal {
             request: request.clone(),
             master: Some(master),
             exec: Some(process),
+            exit: None,
             guest,
-            }),
+        };
-        ))
+        let exec_client = terminal_process
+            .exec
+            .as_ref()
+            .expect("new terminal retains its exec client")
+            .identity();
+        commit_running_or_cleanup(
+            recorder,
+            request,
+            running.clone(),
+            pty,
+            exec_client,
+            guest,
+            &mut terminal_process,
+        )?;
+        Ok((running, Box::new(terminal_process)))
     }
 }
 
+fn commit_running_or_cleanup(
+    recorder: &HostCoordinatorTerminalRecorder,
+    request: &TerminalAcquisition,
+    running: SessionTerminal,
+    pty: PtyIdentity,
+    exec_client: PidfsProcessIdentityV1,
+    guest: GuestProcessIdentity,
+    process: &mut dyn SessionTerminalProcess,
+) -> Result<(), TerminalError> {
+    if let Err(original) = recorder.put(running) {
+        let cleanup =
+            cleanup_failed_running_commit(recorder, request, pty, exec_client, guest, process)
+                .err()
+                .map(Box::new);
+        return Err(TerminalError::AcquisitionCleanup {
+            original: Box::new(original),
+            cleanup,
+        });
+    }
+    Ok(())
+}
+
+fn cleanup_failed_running_commit(
+    recorder: &HostCoordinatorTerminalRecorder,
+    request: &TerminalAcquisition,
+    pty: PtyIdentity,
+    exec_client: PidfsProcessIdentityV1,
+    guest: GuestProcessIdentity,
+    process: &mut dyn SessionTerminalProcess,
+) -> Result<(), TerminalError> {
+    let observed = terminal_snapshot(
+        request,
+        SessionTerminalState::Intent,
+        PersistedHostProcessOwnership::committed(request.host_coordinator),
+        Some(pty),
+        PersistedHostProcessOwnership::committed(exec_client),
+        Some(guest),
+    )?;
+    let durable = recorder.put(observed.clone()).and_then(|()| {
+        recorder.put(copy_terminal_state(
+            &observed,
+            SessionTerminalState::ReleaseIntent,
+        )?)
+    });
+    let physical = release_physical(process, request.session_id, false, true).map(|_| ());
+    match (durable, physical) {
+        (Err(durable), Err(physical)) => {
+            return Err(TerminalError::AcquisitionCleanup {
+                original: Box::new(durable),
+                cleanup: Some(Box::new(physical)),
+            });
+        }
+        (Err(error), Ok(())) | (Ok(()), Err(error)) => return Err(error),
+        (Ok(()), Ok(())) => {}
+    }
+    recorder.clean(clean_terminal(request.generation_id, request.session_id)?)
+}
+
 fn clean_failed_blocked_acquisition(
     recorder: &HostCoordinatorTerminalRecorder,
     request: &TerminalAcquisition,
@@ -744,6 +1002,7 @@ struct RunscSessionTerminal {
     request: TerminalAcquisition,
     master: Option<File>,
     exec: Option<RetainedProcess>,
+    exit: Option<ProcessExit>,
     guest: GuestProcessIdentity,
 }
 
@@ -782,8 +1041,7 @@ impl SessionTerminalProcess for RunscSessionTerminal {
     fn exec_exited(&self) -> io::Result<bool> {
         self.exec
             .as_ref()
-            .ok_or_else(|| io::Error::other("exec client was already reaped"))?
+            .map_or(Ok(true), RetainedProcess::has_exited)
-            .has_exited()
     }
 
     fn signal_group(&self, signal: &str) -> Result<(), TerminalError> {
@@ -803,22 +1061,32 @@ impl SessionTerminalProcess for RunscSessionTerminal {
     }
 
     fn group_absent(&self) -> Result<bool, TerminalError> {
-        Ok(observe_guest_identity(&self.request, self.guest.leader_pid.get())?.is_none())
+        if observe_guest_identity(&self.request, self.guest.leader_pid.get())?
+            .is_some_and(|observed| observed != self.guest)
+        {
+            return Err(TerminalError::GuestIdentityMismatch);
+        }
+        observe_group_absent(&self.request, self.guest.process_group_id.get())
     }
 
     fn terminate_exec(&self) -> io::Result<()> {
-        self.exec
+        match self.exec.as_ref() {
-            .as_ref()
+            Some(exec) => exec.send_signal(rustix::process::Signal::KILL),
-            .ok_or_else(|| io::Error::other("exec client was already reaped"))?
+            None => Ok(()),
-            .send_signal(rustix::process::Signal::KILL)
+        }
     }
 
     fn reap_exec(&mut self) -> io::Result<ProcessExit> {
-        child_broker().reap_status(
+        if let Some(exit) = self.exit {
+            return Ok(exit);
+        }
+        let exit = child_broker().reap_status(
             self.exec
                 .take()
-                .ok_or_else(|| io::Error::other("exec client was already reaped"))?,
+                .ok_or_else(|| io::Error::other("exec client exit is unavailable"))?,
-        )
+        )?;
+        self.exit = Some(exit);
+        Ok(exit)
     }
 
     fn close_pty(&mut self) {
@@ -952,31 +1220,8 @@ fn observe_guest_identity(
     request: &TerminalAcquisition,
     pid: u32,
 ) -> Result<Option<GuestProcessIdentity>, TerminalError> {
-    let mut target = runsc_target(request);
+    let stdout = run_guest_identity_query(request, &[OsString::from(pid.to_string())])?;
-    target = target
+    if stdout == b"absent\n" {
-        .argument("exec")
-        .argument(hex_lower(request.container_id.as_bytes()))
-        .argument(SANDBOX_INIT_PATH)
-        .argument(PRIVATE_SESSION_IDENTITY_ROLE)
-        .argument(pid.to_string());
-    let mut child = child_broker().run_transient_captured(&target)?;
-    let deadline = Instant::now() + IDENTITY_DEADLINE;
-    let mut stdout = Vec::new();
-    let mut stderr = Vec::new();
-    while Instant::now() < deadline && !child.process().has_exited()? {
-        read_available(child.streams().0, &mut stdout)?;
-        read_available(child.streams().1, &mut stderr)?;
-        std::thread::sleep(POLL_INTERVAL);
-    }
-    if !child.process().has_exited()? {
-        child.process().send_signal(rustix::process::Signal::KILL)?;
-        child_broker().reap_status(child.into_process())?;
-        return Err(io::Error::other("guest identity observation deadline expired").into());
-    }
-    read_available(child.streams().0, &mut stdout)?;
-    read_available(child.streams().1, &mut stderr)?;
-    let exit = child_broker().reap_status(child.into_process())?;
-    if exit.exit_code != Some(0) || exit.terminating_signal.is_some() {
         return Ok(None);
     }
     let fields = std::str::from_utf8(&stdout)
@@ -1007,6 +1252,55 @@ fn observe_guest_identity(
     }))
 }
 
+fn observe_group_absent(request: &TerminalAcquisition, group: u32) -> Result<bool, TerminalError> {
+    let stdout = run_guest_identity_query(
+        request,
+        &[OsString::from("--group"), OsString::from(group.to_string())],
+    )?;
+    match stdout.as_slice() {
+        b"absent\n" => Ok(true),
+        b"present\n" => Ok(false),
+        _ => Err(io::Error::other("guest group response is malformed").into()),
+    }
+}
+
+fn run_guest_identity_query(
+    request: &TerminalAcquisition,
+    arguments: &[OsString],
+) -> Result<Vec<u8>, TerminalError> {
+    let mut target = runsc_target(request);
+    target = target
+        .argument("exec")
+        .argument(hex_lower(request.container_id.as_bytes()))
+        .argument(SANDBOX_INIT_PATH)
+        .argument(PRIVATE_SESSION_IDENTITY_ROLE);
+    for argument in arguments {
+        target = target.argument(argument.clone());
+    }
+    let mut child = child_broker().run_transient_captured(&target)?;
+    let deadline = Instant::now() + IDENTITY_DEADLINE;
+    let mut stdout = Vec::new();
+    let mut stderr = Vec::new();
+    while Instant::now() < deadline && !child.process().has_exited()? {
+        read_available(child.streams().0, &mut stdout)?;
+        read_available(child.streams().1, &mut stderr)?;
+        std::thread::sleep(POLL_INTERVAL);
+    }
+    if !child.process().has_exited()? {
+        child.process().send_signal(rustix::process::Signal::KILL)?;
+        child_broker().reap_status(child.into_process())?;
+        return Err(io::Error::other("guest identity observation deadline expired").into());
+    }
+    read_available(child.streams().0, &mut stdout)?;
+    read_available(child.streams().1, &mut stderr)?;
+    let exit = child_broker().reap_status(child.into_process())?;
+    if exit.exit_code != Some(0) || exit.terminating_signal.is_some() {
+        let stderr = String::from_utf8_lossy(&stderr);
+        return Err(io::Error::other(format!("guest identity query failed: {stderr}")).into());
+    }
+    Ok(stdout)
+}
+
 fn runsc_target(request: &TerminalAcquisition) -> TargetCommand {
     request.runsc_global_arguments.iter().cloned().fold(
         TargetCommand::new(request.runsc_program.clone()),

crates/lazaret-host-coordinator/src/terminal/tests.rs

@@ -62,8 +62,7 @@ impl Fixture {
             HostCoordinatorDomainStore::open(lock, &directory.path().join("domain.redb"), &domain)
                 .unwrap(),
         );
-        store
+        let mut generation = Generation {
-            .apply(&[DomainStoreMutation::PutGeneration(Generation {
             domain_id: domain.domain_id,
             generation_id: GenerationId::new(7),
             configuration_fingerprint: ConfigurationFingerprint::new(bytes(2)),
@@ -71,7 +70,17 @@ impl Fixture {
             network_sidecar_id: NetworkSidecarId::new(bytes(4)),
             sandbox_id: SandboxId::new(bytes(5)),
             durable_decision: DurableDecision::None,
-            })])
+        };
+        store
+            .apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
+            .unwrap();
+        generation.durable_decision = DurableDecision::Preparing;
+        store
+            .apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
+            .unwrap();
+        generation.durable_decision = DurableDecision::Published;
+        store
+            .apply(&[DomainStoreMutation::PutGeneration(generation)])
             .unwrap();
         store
             .apply(&[DomainStoreMutation::PutParticipant(
@@ -123,6 +132,78 @@ impl Fixture {
                 sandbox,
             ))])
             .unwrap();
+        insert_attached_sessions(&store, session_ids);
+        Self {
+            _directory: directory,
+            store,
+        }
+    }
+
+    fn insert_session(&self, session_id: SessionId, phase: SessionPhase) {
+        self.store
+            .apply(&[DomainStoreMutation::PutSession(Session {
+                generation_id: GenerationId::new(7),
+                session_id,
+                cli_id: ClientId::new(bytes(20)),
+                session_anchor_id: SessionAnchorId::new(bytes(30)),
+                session_phase: phase,
+            })])
+            .unwrap();
+    }
+
+    fn set_session_phase(&self, session_id: SessionId, phase: SessionPhase) {
+        let mut session = self
+            .store
+            .snapshot()
+            .unwrap()
+            .sessions
+            .into_iter()
+            .find(|session| session.session_id == session_id)
+            .unwrap();
+        session.session_phase = phase;
+        self.store
+            .apply(&[DomainStoreMutation::PutSession(session)])
+            .unwrap();
+    }
+
+    fn release_sandbox(&self) {
+        let sandbox = self
+            .store
+            .snapshot()
+            .unwrap()
+            .participants
+            .into_iter()
+            .find(|participant| participant.role() == ParticipantRole::Sandbox)
+            .unwrap();
+        self.store
+            .apply(&[DomainStoreMutation::PutParticipant(
+                Participant::snapshot(
+                    sandbox.generation_id(),
+                    sandbox.role(),
+                    ParticipantState::ReleaseIntent,
+                    sandbox.ownership().cloned(),
+                )
+                .unwrap(),
+            )])
+            .unwrap();
+    }
+
+    fn add_newer_generation(&self) {
+        self.store
+            .apply(&[DomainStoreMutation::PutGeneration(Generation {
+                domain_id: DomainId::new(bytes(1)),
+                generation_id: GenerationId::new(8),
+                configuration_fingerprint: ConfigurationFingerprint::new(bytes(42)),
+                coordinator_id: CoordinatorId::new(bytes(43)),
+                network_sidecar_id: NetworkSidecarId::new(bytes(44)),
+                sandbox_id: SandboxId::new(bytes(45)),
+                durable_decision: DurableDecision::None,
+            })])
+            .unwrap();
+    }
+}
+
+fn insert_attached_sessions(store: &HostCoordinatorDomainStore, session_ids: &[SessionId]) {
     for (index, session_id) in session_ids.iter().enumerate() {
         let mut session = Session {
             generation_id: GenerationId::new(7),
@@ -139,26 +220,87 @@ impl Fixture {
             .apply(&[DomainStoreMutation::PutSession(session)])
             .unwrap();
     }
-        Self {
-            _directory: directory,
-            store,
-        }
-    }
 }
 
 #[derive(Debug)]
 struct MockState {
     output: VecDeque<Vec<u8>>,
     log: Arc<Mutex<Vec<String>>>,
+    guest: MockGuestState,
+    resources: MockResourceState,
+}
+
+#[derive(Debug)]
+struct MockGuestState {
     group_present: bool,
+    leader_present: bool,
     identity_mismatch: bool,
+}
+
+#[derive(Debug)]
+struct MockResourceState {
     exec_exited: bool,
+    exec_reaped: bool,
+    pty_open: bool,
 }
 
 struct MockProcess {
     state: Arc<Mutex<MockState>>,
 }
 
+#[derive(Clone, Copy)]
+enum StoreFailurePoint {
+    Snapshot,
+    Write,
+    Commit,
+}
+
+struct FailingRunningStore {
+    inner: Arc<HostCoordinatorDomainStore>,
+    failure: Mutex<Option<(StoreFailurePoint, bool)>>,
+}
+
+impl FailingRunningStore {
+    fn new(inner: Arc<HostCoordinatorDomainStore>) -> Self {
+        Self {
+            inner,
+            failure: Mutex::new(None),
+        }
+    }
+
+    fn arm(&self, point: StoreFailurePoint, persistent: bool) {
+        *self.failure.lock().unwrap() = Some((point, persistent));
+    }
+}
+
+impl TerminalDomainStore for FailingRunningStore {
+    fn snapshot(&self) -> Result<lazaret_core::records::DomainStore, DomainStoreError> {
+        self.inner.snapshot()
+    }
+
+    fn apply(&self, mutations: &[DomainStoreMutation]) -> Result<(), DomainStoreError> {
+        let mut failure = self.failure.lock().unwrap();
+        if let Some((point, persistent)) = *failure {
+            let running = mutations.iter().any(|mutation| {
+                matches!(mutation, DomainStoreMutation::PutSessionTerminal(terminal) if terminal.state() == SessionTerminalState::Running)
+            });
+            if running || persistent {
+                if !persistent {
+                    *failure = None;
+                }
+                let point = match point {
+                    StoreFailurePoint::Snapshot => "injected snapshot failure",
+                    StoreFailurePoint::Write => "injected write failure",
+                    StoreFailurePoint::Commit => "injected commit failure",
+                };
+                return Err(DomainStoreError::Io(io::Error::other(point)));
+            }
+        }
+        drop(failure);
+        self.inner.apply(mutations)
+    }
+}
+
 impl SessionTerminalProcess for MockProcess {
     fn write_input(&mut self, bytes: &[u8]) -> io::Result<usize> {
         self.state
@@ -194,44 +336,42 @@ impl SessionTerminalProcess for MockProcess {
     }
 
     fn exec_exited(&self) -> io::Result<bool> {
-        Ok(self.state.lock().unwrap().exec_exited)
+        Ok(self.state.lock().unwrap().resources.exec_exited)
     }
 
     fn signal_group(&self, signal: &str) -> Result<(), TerminalError> {
         let mut state = self.state.lock().unwrap();
-        if state.identity_mismatch {
+        if !state.guest.leader_present || state.guest.identity_mismatch {
             return Err(TerminalError::GuestIdentityMismatch);
         }
         state.log.lock().unwrap().push(format!("signal:{signal}"));
         if signal == "HUP" {
-            state.group_present = false;
+            state.guest.group_present = false;
         }
         Ok(())
     }
 
     fn group_absent(&self) -> Result<bool, TerminalError> {
         let state = self.state.lock().unwrap();
-        if state.identity_mismatch {
+        if state.guest.identity_mismatch {
             return Err(TerminalError::GuestIdentityMismatch);
         }
-        Ok(!state.group_present)
+        Ok(!state.guest.group_present)
     }
 
     fn terminate_exec(&self) -> io::Result<()> {
         let mut state = self.state.lock().unwrap();
         state.log.lock().unwrap().push("terminate-exec".into());
-        state.exec_exited = true;
+        state.resources.exec_exited = true;
         Ok(())
     }
 
     fn reap_exec(&mut self) -> io::Result<ProcessExit> {
-        self.state
+        let mut state = self.state.lock().unwrap();
-            .lock()
+        if !state.resources.exec_reaped {
-            .unwrap()
+            state.log.lock().unwrap().push("reap-exec".into());
-            .log
+            state.resources.exec_reaped = true;
-            .lock()
+        }
-            .unwrap()
-            .push("reap-exec".into());
         Ok(ProcessExit {
             exit_code: Some(17),
             terminating_signal: None,
@@ -239,8 +379,9 @@ impl SessionTerminalProcess for MockProcess {
     }
 
     fn close_pty(&mut self) {
-        let state = self.state.lock().unwrap();
+        let mut state = self.state.lock().unwrap();
         state.log.lock().unwrap().push("close-pty".into());
+        state.resources.pty_open = false;
     }
 }
 
@@ -301,9 +442,16 @@ impl SessionTerminalFactory for MockFactory {
         let state = Arc::new(Mutex::new(MockState {
             output: VecDeque::new(),
             log: Arc::clone(&self.log),
+            guest: MockGuestState {
                 group_present: true,
+                leader_present: true,
                 identity_mismatch: false,
+            },
+            resources: MockResourceState {
                 exec_exited: false,
+                exec_reaped: false,
+                pty_open: true,
+            },
         }));
         self.states.lock().unwrap().push(Arc::clone(&state));
         Ok((running, Box::new(MockProcess { state })))
@@ -336,13 +484,18 @@ fn acquisition(session_id: SessionId) -> TerminalAcquisition {
 
 fn setup(ids: &[SessionId]) -> (Fixture, Arc<MockFactory>, SessionTerminalEngine) {
     let fixture = Fixture::new(ids);
+    let (factory, engine) = engine_for(&fixture);
+    (fixture, factory, engine)
+}
+
+fn engine_for(fixture: &Fixture) -> (Arc<MockFactory>, SessionTerminalEngine) {
     let factory = Arc::new(MockFactory::default());
     let engine = SessionTerminalEngine::new(
         Arc::clone(&fixture.store),
         factory.clone(),
         Arc::new(Cancellation(Arc::clone(&factory.log))),
     );
-    (fixture, factory, engine)
+    (factory, engine)
 }
 
 fn id(value: u8) -> SessionId {
@@ -369,6 +522,88 @@ fn acquisition_refuses_authority_outside_the_committed_sandbox() {
     );
 }
 
+#[test]
+fn acquisition_requires_attached_current_session() {
+    for phase in [
+        SessionPhase::New,
+        SessionPhase::Waiting,
+        SessionPhase::Registering,
+    ] {
+        let session = id(6);
+        let fixture = Fixture::new(&[]);
+        fixture.insert_session(session, phase);
+        let (_factory, mut engine) = engine_for(&fixture);
+        assert!(matches!(
+            engine.acquire(&acquisition(session)),
+            Err(TerminalError::AuthorityMismatch)
+        ));
+        assert!(
+            fixture
+                .store
+                .snapshot()
+                .unwrap()
+                .session_terminals
+                .is_empty()
+        );
+    }
+}
+
+#[test]
+fn durable_detach_failure_and_sandbox_release_revoke_active_terminal() {
+    for revoke in ["closed", "failed", "sandbox-release"] {
+        let session = id(6);
+        let (fixture, _factory, mut engine) = setup(&[session]);
+        engine.acquire(&acquisition(session)).unwrap();
+        match revoke {
+            "closed" => fixture.set_session_phase(session, SessionPhase::Closed),
+            "failed" => fixture.set_session_phase(session, SessionPhase::Failed),
+            "sandbox-release" => fixture.release_sandbox(),
+            _ => unreachable!(),
+        }
+        assert!(matches!(
+            engine.handle(
+                GenerationId::new(7),
+                SessionTerminalEvent::Input(TerminalInput {
+                    session_id: session,
+                    bytes: b"refused".to_vec(),
+                }),
+            ),
+            Err(TerminalError::AuthorityMismatch)
+        ));
+        assert!(!engine.contains(session));
+        assert_eq!(
+            fixture.store.snapshot().unwrap().session_terminals[0].state(),
+            SessionTerminalState::Clean
+        );
+    }
+}
+
+#[test]
+fn stale_caller_and_superseded_generation_refuse_without_cross_generation_action() {
+    let session = id(6);
+    let (fixture, factory, mut engine) = setup(&[session]);
+    engine.acquire(&acquisition(session)).unwrap();
+    assert!(matches!(
+        engine.handle(
+            GenerationId::new(8),
+            SessionTerminalEvent::Signal(TerminalSignal {
+                session_id: session,
+                signal: TerminalSignalKind::Interrupt,
+            }),
+        ),
+        Err(TerminalError::StaleGeneration)
+    ));
+    assert!(factory.log.lock().unwrap().is_empty());
+    assert!(engine.contains(session));
+
+    fixture.add_newer_generation();
+    assert!(matches!(
+        engine.poll(session, GenerationId::new(7)),
+        Err(TerminalError::AuthorityMismatch)
+    ));
+    assert!(!engine.contains(session));
+}
+
 #[test]
 fn resize_and_terminal_signals_use_the_exact_group_map() {
     let session = id(6);
@@ -411,9 +646,9 @@ fn resize_and_terminal_signals_use_the_exact_group_map() {
 #[test]
 fn changed_guest_identity_refuses_numeric_group_action() {
     let session = id(6);
-    let (_fixture, factory, mut engine) = setup(&[session]);
+    let (fixture, factory, mut engine) = setup(&[session]);
     engine.acquire(&acquisition(session)).unwrap();
-    factory.state(0).lock().unwrap().identity_mismatch = true;
+    factory.state(0).lock().unwrap().guest.identity_mismatch = true;
     assert!(matches!(
         engine.handle(
             GenerationId::new(7),
@@ -425,13 +660,186 @@ fn changed_guest_identity_refuses_numeric_group_action() {
         Err(TerminalError::GuestIdentityMismatch)
     ));
     assert!(factory.log.lock().unwrap().is_empty());
+    assert!(matches!(
+        engine.handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Close(lazaret_core::records::SessionClose {
+                session_id: session,
+            }),
+        ),
+        Err(TerminalError::GuestIdentityMismatch)
+    ));
+    assert_eq!(
+        fixture.store.snapshot().unwrap().session_terminals[0].state(),
+        SessionTerminalState::ReleaseIntent
+    );
+}
+
+#[test]
+fn absent_leader_with_live_group_retains_release_until_group_is_empty() {
+    let session = id(6);
+    let (fixture, factory, mut engine) = setup(&[session]);
+    engine.acquire(&acquisition(session)).unwrap();
+    {
+        let state = factory.state(0);
+        let mut state = state.lock().unwrap();
+        state.guest.leader_present = false;
+        state.guest.group_present = true;
+    }
+    assert!(matches!(
+        engine.handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Close(lazaret_core::records::SessionClose {
+                session_id: session,
+            }),
+        ),
+        Err(TerminalError::GuestIdentityMismatch)
+    ));
+    let state = factory.state(0);
+    assert!(state.lock().unwrap().resources.exec_reaped);
+    assert!(!state.lock().unwrap().resources.pty_open);
+    assert_eq!(
+        fixture.store.snapshot().unwrap().session_terminals[0].state(),
+        SessionTerminalState::ReleaseIntent
+    );
+
+    state.lock().unwrap().guest.group_present = false;
+    engine
+        .handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Close(lazaret_core::records::SessionClose {
+                session_id: session,
+            }),
+        )
+        .unwrap();
+    assert_eq!(
+        fixture.store.snapshot().unwrap().session_terminals[0].state(),
+        SessionTerminalState::Clean
+    );
+}
+
+#[test]
+#[allow(clippy::too_many_lines)]
+fn running_store_failures_release_every_live_partial_handle() {
+    for point in [
+        StoreFailurePoint::Snapshot,
+        StoreFailurePoint::Write,
+        StoreFailurePoint::Commit,
+    ] {
+        for persistent in [false, true] {
+            let session = id(6);
+            let fixture = Fixture::new(&[session]);
+            let store = Arc::new(FailingRunningStore::new(Arc::clone(&fixture.store)));
+            let recorder = HostCoordinatorTerminalRecorder::with_store(store.clone());
+            let request = acquisition(session);
+            recorder
+                .put(
+                    terminal_snapshot(
+                        &request,
+                        SessionTerminalState::Intent,
+                        PersistedHostProcessOwnership::unassigned(),
+                        None,
+                        PersistedHostProcessOwnership::unassigned(),
+                        None,
+                    )
+                    .unwrap(),
+                )
+                .unwrap();
+            let pty = PtyIdentity {
+                master_fd: FileDescriptor::new(9),
+                device: DeviceId::new(10),
+                inode: InodeId::new(11),
+            };
+            recorder
+                .put(
+                    staged_terminal(
+                        &request,
+                        pty,
+                        StagedProcessOwnership::v1(
+                            request.domain_id,
+                            request.generation_id,
+                            request.coordinator_id,
+                            lazaret_core::records::HostProcessOwnershipSlot::SessionTerminalExecClient(
+                                request.session_id,
+                            ),
+                            process(90),
+                        ),
+                    )
+                    .unwrap(),
+                )
+                .unwrap();
+            let guest = GuestProcessIdentity {
+                leader_pid: GuestProcessId::new(41),
+                leader_start_time: GuestProcessStartTimeTicks::new(101),
+                process_group_id: GuestProcessGroupId::new(41),
+            };
+            let running = terminal_snapshot(
+                &request,
+                SessionTerminalState::Running,
+                PersistedHostProcessOwnership::committed(request.host_coordinator),
+                Some(pty),
+                PersistedHostProcessOwnership::committed(process(90)),
+                Some(guest),
+            )
+            .unwrap();
+            let state = Arc::new(Mutex::new(MockState {
+                output: VecDeque::new(),
+                log: Arc::new(Mutex::new(Vec::new())),
+                guest: MockGuestState {
+                    group_present: true,
+                    leader_present: true,
+                    identity_mismatch: false,
+                },
+                resources: MockResourceState {
+                    exec_exited: false,
+                    exec_reaped: false,
+                    pty_open: true,
+                },
+            }));
+            let mut process_handle = MockProcess {
+                state: Arc::clone(&state),
+            };
+            store.arm(point, persistent);
+            let error = commit_running_or_cleanup(
+                &recorder,
+                &request,
+                running,
+                pty,
+                process(90),
+                guest,
+                &mut process_handle,
+            )
+            .unwrap_err();
+            match error {
+                TerminalError::AcquisitionCleanup { cleanup: None, .. } if !persistent => {}
+                TerminalError::AcquisitionCleanup {
+                    cleanup: Some(_), ..
+                } if persistent => {}
+                error => panic!("unexpected cleanup result: {error}"),
+            }
+            let state = state.lock().unwrap();
+            assert!(!state.guest.group_present);
+            assert!(state.resources.exec_exited);
+            assert!(state.resources.exec_reaped);
+            assert!(!state.resources.pty_open);
+            drop(state);
+            assert_eq!(
+                fixture.store.snapshot().unwrap().session_terminals[0].state(),
+                if persistent {
+                    SessionTerminalState::Intent
+                } else {
+                    SessionTerminalState::Clean
+                }
+            );
+        }
+    }
 }
 
 #[test]
 fn merged_output_is_ordered_and_prompt_route_isolated_by_session() {
     let first = id(6);
     let second = id(7);
-    let (_fixture, factory, mut engine) = setup(&[first, second]);
+    let (fixture, factory, mut engine) = setup(&[first, second]);
     engine.acquire(&acquisition(first)).unwrap();
     engine.acquire(&acquisition(second)).unwrap();
     factory
@@ -446,6 +854,7 @@ fn merged_output_is_ordered_and_prompt_route_isolated_by_session() {
         .unwrap()
         .output
         .push_back(b"other".to_vec());
+    fixture.set_session_phase(first, SessionPhase::Prompting);
     engine
         .set_route(first, GenerationId::new(7), TerminalRoute::Prompt)
         .unwrap();
@@ -457,6 +866,7 @@ fn merged_output_is_ordered_and_prompt_route_isolated_by_session() {
             bytes: b"other".to_vec(),
         })]
     );
+    fixture.set_session_phase(first, SessionPhase::Attached);
     engine
         .set_route(first, GenerationId::new(7), TerminalRoute::Sandbox)
         .unwrap();