Recover pre-identity session terminals safely

crates/lazaret-host-coordinator/src/recovery/reconciliation.rs

@@ -9,7 +9,8 @@ use lazaret_core::configuration::{ConfinedPathType, ResolvedDomainConfiguration}
 use lazaret_core::identity::{GenerationId, HostPath, MountIdentity, PidfsProcessIdentityV1};
 use lazaret_core::records::{
     ArtifactLease, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
-    ParticipantState, SandboxOwnership, SessionPhase,
+    ParticipantState, PersistedHostProcessOwnership, SandboxOwnership, SessionPhase,
+    SessionTerminal, SessionTerminalFields, SessionTerminalState,
 };
 use lazaret_linux::filesystem::{current_mount_identity, mount_identity};
 use lazaret_linux::process::{
@@ -148,6 +149,7 @@ pub(crate) enum RecoveryBoundary {
     GitRouting,
     StateAnchor,
     PreparedLower,
+    SessionTerminals,
     PrepareOperations,
     ArtifactGc,
     FinishGeneration,
@@ -265,15 +267,11 @@ where
     let generation_id = generation.generation_id;
     let snapshot = store.snapshot().map_err(|error| error.to_string())?;
     if snapshot
-        .session_terminals
-        .iter()
-        .any(|terminal| terminal.generation_id() == generation_id)
-        || snapshot
         .mailboxes
         .iter()
         .any(|mailbox| mailbox.generation_id == generation_id)
     {
-        return Err("unfinished terminal or mailbox state cannot be recovered safely".into());
+        return Err("unfinished mailbox state cannot be recovered safely".into());
     }
     let session_updates = snapshot
         .sessions
@@ -300,9 +298,20 @@ where
         .snapshot()
         .map_err(|error| format!("Participant snapshot failed: {error}"))?
         .participants;
+    let sandbox_container = participants
+        .iter()
+        .find(|participant| {
+            participant.generation_id() == generation_id
+                && participant.role() == ParticipantRole::Sandbox
+        })
+        .and_then(Participant::ownership)
+        .and_then(|ownership| match ownership {
+            ParticipantOwnership::Sandbox(sandbox) => sandbox.container_id,
+            _ => None,
+        });
     let ownership = runtime_recovery_ownership(&participants, generation)
         .map_err(|error| format!("runtime ownership inventory failed: {error}"))?;
-    let mut owners = seal_runtime_owners(&ownership, open)
+    let mut owners = seal_runtime_owners(&ownership, &mut *open)
         .map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
 
     failpoint(RecoveryBoundary::Publication, generation_id)?;
@@ -341,6 +350,36 @@ where
     recover_partial_prepared_lower(configuration, store, catalog, generation_id)
         .map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
 
+    finish_runtime_and_terminal_recovery(
+        store,
+        generation_id,
+        &ownership,
+        sandbox_container,
+        &mut owners,
+        &mut *open,
+        failpoint,
+    )?;
+    failpoint(RecoveryBoundary::FinishGeneration, generation_id)?;
+    let mut clean = generation.clone();
+    clean.durable_decision = DurableDecision::Clean;
+    store
+        .apply(&[DomainStoreMutation::PutGeneration(clean)])
+        .map_err(|error| error.to_string())
+}
+
+fn finish_runtime_and_terminal_recovery<Open, Failpoint>(
+    store: &HostCoordinatorDomainStore,
+    generation_id: GenerationId,
+    runtime: &RuntimeRecoveryOwnership,
+    sandbox_container: Option<lazaret_core::identity::ContainerId>,
+    owners: &mut SealedRuntimeOwners,
+    open: &mut Open,
+    failpoint: &mut Failpoint,
+) -> Result<(), String>
+where
+    Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
+    Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
+{
     let snapshot = store.snapshot().map_err(|error| error.to_string())?;
     if snapshot.participants.iter().any(|participant| {
         participant.generation_id() == generation_id
@@ -348,12 +387,161 @@ where
     }) {
         return Err("generation retains non-Clean participants after recovery".into());
     }
-    failpoint(RecoveryBoundary::FinishGeneration, generation_id)?;
+    owners
-    let mut clean = generation.clone();
+        .prove_absent()
-    clean.durable_decision = DurableDecision::Clean;
+        .map_err(|error| format!("runtime absence proof failed: {error}"))?;
+    failpoint(RecoveryBoundary::SessionTerminals, generation_id)?;
+    recover_session_terminals_after_runtime_absence(
+        store,
+        generation_id,
+        runtime,
+        sandbox_container,
+        open,
+    )
+}
+
+fn recover_session_terminals_after_runtime_absence<Open>(
+    store: &HostCoordinatorDomainStore,
+    generation_id: GenerationId,
+    runtime: &RuntimeRecoveryOwnership,
+    sandbox_container: Option<lazaret_core::identity::ContainerId>,
+    open: &mut Open,
+) -> Result<(), String>
+where
+    Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
+{
+    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
+    for terminal in snapshot
+        .session_terminals
+        .iter()
+        .filter(|terminal| terminal.generation_id() == generation_id)
+    {
+        if terminal.state() == SessionTerminalState::Clean {
+            continue;
+        }
+        if !snapshot.sessions.iter().any(|session| {
+            session.generation_id == generation_id
+                && session.session_id == terminal.session_id()
+                && matches!(
+                    session.session_phase,
+                    SessionPhase::Closed | SessionPhase::Failed
+                )
+        }) {
+            return Err("Session Terminal has no closed or failed Session".into());
+        }
+        require_terminal_runtime_binding(terminal, runtime, sandbox_container)?;
+        require_terminal_exec_absent(terminal.exec_client(), open)?;
+        let bare_intent = terminal.state() == SessionTerminalState::Intent
+            && terminal.container_id().is_none()
+            && terminal.host_coordinator().is_unassigned()
+            && terminal.pty_master().is_none()
+            && terminal.exec_client().is_unassigned()
+            && terminal.guest_process().is_none();
+        let current = if terminal.state() == SessionTerminalState::ReleaseIntent || bare_intent {
+            terminal.clone()
+        } else {
+            let release = terminal_with_state(terminal, SessionTerminalState::ReleaseIntent)?;
             store
-        .apply(&[DomainStoreMutation::PutGeneration(clean)])
+                .apply(&[DomainStoreMutation::PutSessionTerminal(release.clone())])
-        .map_err(|error| error.to_string())
+                .map_err(|error| format!("Session Terminal release failed: {error}"))?;
+            release
+        };
+        let clean = SessionTerminal::snapshot(SessionTerminalFields {
+            generation_id,
+            session_id: terminal.session_id(),
+            state: SessionTerminalState::Clean,
+            container_id: None,
+            host_coordinator: PersistedHostProcessOwnership::unassigned(),
+            pty_master: None,
+            exec_client: PersistedHostProcessOwnership::unassigned(),
+            guest_process: None,
+        })
+        .map_err(|_| "Session Terminal Clean snapshot is invalid")?;
+        current
+            .validate_transition_to_clean_after_absence(&clean)
+            .map_err(|error| format!("Session Terminal cleanup is invalid: {error}"))?;
+        store
+            .apply(&[DomainStoreMutation::CleanSessionTerminalAfterAbsence(clean)])
+            .map_err(|error| format!("Session Terminal cleanup failed: {error}"))?;
+    }
+    Ok(())
+}
+
+fn require_terminal_runtime_binding(
+    terminal: &SessionTerminal,
+    runtime: &RuntimeRecoveryOwnership,
+    sandbox_container: Option<lazaret_core::identity::ContainerId>,
+) -> Result<(), String> {
+    if let Some(container_id) = terminal.container_id()
+        && Some(container_id) != sandbox_container
+    {
+        return Err("Session Terminal container differs from runtime authority".into());
+    }
+    if terminal.guest_process().is_some() && terminal.container_id().is_none() {
+        return Err("Session Terminal guest identity has no container authority".into());
+    }
+    match terminal.host_coordinator().authoritative_pidfs() {
+        Ok(identity) if runtime.process(RuntimeProcessRole::HostCoordinator) == Some(*identity) => {
+        }
+        Ok(_) => {
+            return Err("Session Terminal Host Coordinator differs from runtime authority".into());
+        }
+        Err(_) if terminal.pty_master().is_none() => {}
+        Err(_) => return Err("Session Terminal PTY has no Host Coordinator authority".into()),
+    }
+    if let Ok(identity) = terminal.exec_client().authoritative_pidfs()
+        && runtime
+            .processes
+            .iter()
+            .any(|owner| owner.identity == *identity)
+    {
+        return Err("Session Terminal exec identity aliases runtime authority".into());
+    }
+    Ok(())
+}
+
+fn require_terminal_exec_absent<Open>(
+    ownership: &PersistedHostProcessOwnership,
+    open: &mut Open,
+) -> Result<(), String>
+where
+    Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
+{
+    let Ok(identity) = ownership.authoritative_pidfs() else {
+        return Ok(());
+    };
+    match open(*identity) {
+        Ok(PidfsRecoveryMatch::BootMismatch) => Ok(()),
+        Ok(PidfsRecoveryMatch::Matched(process))
+            if process.has_exited().map_err(|error| error.to_string())? =>
+        {
+            Ok(())
+        }
+        Ok(PidfsRecoveryMatch::Matched(_)) => {
+            Err("Session Terminal exec client remains present".into())
+        }
+        Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
+            Ok(())
+        }
+        Err(error) => Err(format!("Session Terminal exec identity refused: {error}")),
+    }
+}
+
+fn terminal_with_state(
+    terminal: &SessionTerminal,
+    state: SessionTerminalState,
+) -> Result<SessionTerminal, String> {
+    SessionTerminal::snapshot(SessionTerminalFields {
+        generation_id: terminal.generation_id(),
+        session_id: terminal.session_id(),
+        state,
+        container_id: terminal.container_id(),
+        host_coordinator: *terminal.host_coordinator(),
+        pty_master: terminal.pty_master(),
+        exec_client: *terminal.exec_client(),
+        guest_process: terminal.guest_process(),
+    })
+    .map_err(|_| "Session Terminal release snapshot is invalid".into())
 }
 
 pub(super) fn recover_publication(

crates/lazaret-host-coordinator/src/recovery/tests.rs

@@ -11,18 +11,22 @@ use lazaret_core::configuration::{
     ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
 };
 use lazaret_core::identity::{
-    BootId, BundleDigest, ContainerId, CoordinatorId, DeviceId, HostPidLocator, IdMapDigest,
+    BootId, BundleDigest, ClientId, ContainerId, CoordinatorId, DeviceId, FileDescriptor,
-    InodeId, MountId, NetworkSidecarId, PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1,
+    HostPidLocator, IdMapDigest, InodeId, MountId, NetworkSidecarId, PidfsDeviceId, PidfsInodeId,
-    PolicyId, PrepareOperationId, PreparedArtifactId, ProcessStartTimeTicks, RunscBuildId,
+    PidfsProcessAuthorityV1, PolicyId, PrepareOperationId, PreparedArtifactId,
-    RunscDigest, SandboxId, SelectedManifestDigest, SourceIdentity,
+    ProcessStartTimeTicks, RunscBuildId, RunscDigest, SandboxId, SelectedManifestDigest,
+    SessionAnchorId, SessionId, SourceIdentity,
 };
 use lazaret_core::records::{
     ArtifactLease, ArtifactState, DurableDecision, GitRoutingOwnership, HelperOwnership,
-    PrepareOperation, PrepareStageState, PreparedArtifact, PreparedLowerOwnership,
+    PrepareOperation, PrepareStageState, PreparedArtifact, PreparedLowerOwnership, PtyIdentity,
-    PublicationOwnership, SandboxOwnership, StateAnchorOwnership,
+    PublicationOwnership, SandboxOwnership, Session, SessionPhase, SessionTerminal,
+    SessionTerminalFields, SessionTerminalState, StateAnchorOwnership,
 };
 use lazaret_linux::filesystem::{current_mount_identity, try_exclusive_flock};
-use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
+use lazaret_linux::process::{
+    PidfsRecoveryMatch, current_process_identity, match_recovery_process,
+};
 use lazaret_prepared_lower::PreparedLowerStore;
 use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
 use lazaret_state_anchor::StateAnchorLayout;
@@ -361,6 +365,7 @@ fn recovery_preflight_retries_every_recovery_boundary() {
         RecoveryBoundary::GitRouting,
         RecoveryBoundary::StateAnchor,
         RecoveryBoundary::PreparedLower,
+        RecoveryBoundary::SessionTerminals,
         RecoveryBoundary::PrepareOperations,
         RecoveryBoundary::ArtifactGc,
         RecoveryBoundary::FinishGeneration,
@@ -411,6 +416,52 @@ fn recovery_preflight_opens_every_exact_owner_once() {
     assert!(opened.values().all(|count| *count == 1));
 }
 
+#[test]
+fn recovery_cleans_pre_identity_terminal_after_runtime_absence() {
+    let fixture = RecoveryFixture::new();
+    fixture.add_released_terminal(process(7), process(1));
+
+    fixture.recover_with(|_, _| Ok(())).unwrap();
+
+    let snapshot = fixture.store.snapshot().unwrap();
+    assert_eq!(
+        snapshot.generations[0].durable_decision,
+        DurableDecision::Clean
+    );
+    assert_eq!(
+        snapshot.session_terminals[0].state(),
+        SessionTerminalState::Clean
+    );
+}
+
+#[test]
+fn recovery_retains_pre_identity_terminal_when_exec_client_is_live() {
+    let fixture = RecoveryFixture::new();
+    fixture.add_released_terminal(current_process_identity().unwrap(), process(1));
+
+    let error = fixture.recover_with(|_, _| Ok(())).unwrap_err();
+
+    assert!(error.contains("Session Terminal exec client remains present"));
+    assert_eq!(
+        fixture.store.snapshot().unwrap().session_terminals[0].state(),
+        SessionTerminalState::ReleaseIntent
+    );
+}
+
+#[test]
+fn recovery_retains_terminal_on_runtime_authority_mismatch() {
+    let fixture = RecoveryFixture::new();
+    fixture.add_released_terminal(process(7), process(8));
+
+    let error = fixture.recover_with(|_, _| Ok(())).unwrap_err();
+
+    assert!(error.contains("Host Coordinator differs from runtime authority"));
+    assert_eq!(
+        fixture.store.snapshot().unwrap().session_terminals[0].state(),
+        SessionTerminalState::ReleaseIntent
+    );
+}
+
 #[test]
 fn publication_replacement_and_runsc_root_replacement_refuse_untouched() {
     let fixture = RecoveryFixture::new();
@@ -544,6 +595,77 @@ impl RecoveryFixture {
         )
     }
 
+    fn add_released_terminal(
+        &self,
+        exec_client: PidfsProcessIdentityV1,
+        host_coordinator: PidfsProcessIdentityV1,
+    ) {
+        let session = Session {
+            generation_id: GenerationId::new(1),
+            session_id: SessionId::new([31; 32]),
+            cli_id: ClientId::new([32; 32]),
+            session_anchor_id: SessionAnchorId::new([33; 32]),
+            session_phase: SessionPhase::New,
+        };
+        let mut registering = session.clone();
+        registering.session_phase = SessionPhase::Registering;
+        let mut attached = registering.clone();
+        attached.session_phase = SessionPhase::Attached;
+        let mut closed = attached.clone();
+        closed.session_phase = SessionPhase::Closed;
+        let bare = SessionTerminal::snapshot(SessionTerminalFields {
+            generation_id: session.generation_id,
+            session_id: session.session_id,
+            state: SessionTerminalState::Intent,
+            container_id: None,
+            host_coordinator: PersistedHostProcessOwnership::unassigned(),
+            pty_master: None,
+            exec_client: PersistedHostProcessOwnership::unassigned(),
+            guest_process: None,
+        })
+        .unwrap();
+        let partial = SessionTerminal::snapshot(SessionTerminalFields {
+            generation_id: session.generation_id,
+            session_id: session.session_id,
+            state: SessionTerminalState::Intent,
+            container_id: Some(ContainerId::new([6; 32])),
+            host_coordinator: PersistedHostProcessOwnership::committed(host_coordinator),
+            pty_master: Some(PtyIdentity {
+                master_fd: FileDescriptor::new(17),
+                device: DeviceId::new(18),
+                inode: InodeId::new(19),
+            }),
+            exec_client: PersistedHostProcessOwnership::staged(exec_client),
+            guest_process: None,
+        })
+        .unwrap();
+        let release = SessionTerminal::snapshot(SessionTerminalFields {
+            state: SessionTerminalState::ReleaseIntent,
+            ..SessionTerminalFields {
+                generation_id: partial.generation_id(),
+                session_id: partial.session_id(),
+                state: partial.state(),
+                container_id: partial.container_id(),
+                host_coordinator: *partial.host_coordinator(),
+                pty_master: partial.pty_master(),
+                exec_client: *partial.exec_client(),
+                guest_process: partial.guest_process(),
+            }
+        })
+        .unwrap();
+        self.store
+            .apply(&[
+                DomainStoreMutation::PutSession(session),
+                DomainStoreMutation::PutSession(registering),
+                DomainStoreMutation::PutSession(attached),
+                DomainStoreMutation::PutSession(closed),
+                DomainStoreMutation::PutSessionTerminal(bare),
+                DomainStoreMutation::PutSessionTerminal(partial),
+                DomainStoreMutation::PutSessionTerminal(release),
+            ])
+            .unwrap();
+    }
+
     #[allow(clippy::too_many_lines)]
     fn new() -> Self {
         let tree = tempfile::tempdir().unwrap();