Add coordinator-owned interactive session terminals

crates/lazaret-host-coordinator/src/terminal/tests.rs

@@ -0,0 +1,561 @@
+use std::collections::VecDeque;
+use std::fs::{File, OpenOptions};
+use std::io;
+use std::num::NonZeroU16;
+use std::sync::{Arc, Mutex};
+
+use lazaret_core::identity::{
+    BootId, BundleDigest, ClientId, ConfigurationFingerprint, HostPath, HostPidLocator,
+    NetworkSidecarId, PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks,
+    SandboxId, SessionAnchorId,
+};
+use lazaret_core::records::{
+    Domain, DurableDecision, Generation, Participant, SandboxOwnership, Session, TerminalInput,
+    TerminalSignal,
+};
+use lazaret_linux::filesystem::try_exclusive_flock;
+use tempfile::TempDir;
+
+use super::*;
+
+fn bytes(value: u8) -> [u8; 32] {
+    [value; 32]
+}
+
+fn process(value: u64) -> PidfsProcessIdentityV1 {
+    PidfsProcessIdentityV1::new(
+        PidfsProcessAuthorityV1 {
+            boot_id: BootId::new([1; 16]),
+            pidfs_st_dev: PidfsDeviceId::new(value),
+            pidfs_st_ino: PidfsInodeId::new(value + 1),
+        },
+        HostPidLocator::new(u32::try_from(value).unwrap()),
+        ProcessStartTimeTicks::new(value + 2),
+    )
+}
+
+fn lock_file(path: &std::path::Path) -> File {
+    OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .truncate(false)
+        .open(path)
+        .unwrap()
+}
+
+struct Fixture {
+    _directory: TempDir,
+    store: Arc<HostCoordinatorDomainStore>,
+}
+
+impl Fixture {
+    fn new(session_ids: &[SessionId]) -> Self {
+        let directory = tempfile::tempdir().unwrap();
+        let domain = Domain {
+            domain_id: DomainId::new(bytes(1)),
+            profile_name: "trusted".into(),
+            canonical_workspace: HostPath::new(b"/workspace/project".to_vec()),
+        };
+        let lock = try_exclusive_flock(lock_file(&directory.path().join("domain.lock"))).unwrap();
+        let store = Arc::new(
+            HostCoordinatorDomainStore::open(lock, &directory.path().join("domain.redb"), &domain)
+                .unwrap(),
+        );
+        store
+            .apply(&[DomainStoreMutation::PutGeneration(Generation {
+                domain_id: domain.domain_id,
+                generation_id: GenerationId::new(7),
+                configuration_fingerprint: ConfigurationFingerprint::new(bytes(2)),
+                coordinator_id: CoordinatorId::new(bytes(3)),
+                network_sidecar_id: NetworkSidecarId::new(bytes(4)),
+                sandbox_id: SandboxId::new(bytes(5)),
+                durable_decision: DurableDecision::None,
+            })])
+            .unwrap();
+        store
+            .apply(&[DomainStoreMutation::PutParticipant(
+                Participant::bare_intent(GenerationId::new(7), ParticipantRole::Sandbox),
+            )])
+            .unwrap();
+        let mut sandbox = SandboxOwnership {
+            container_id: Some(ContainerId::new(bytes(9))),
+            runtime_root: Some(HostPath::new(b"runtime".to_vec())),
+            bundle_digest: Some(BundleDigest::new(bytes(10))),
+            host_coordinator: PersistedHostProcessOwnership::committed(process(70)),
+            runtime_supervisor: PersistedHostProcessOwnership::unassigned(),
+            attached_runsc: PersistedHostProcessOwnership::unassigned(),
+        };
+        let sandbox_snapshot = |state, ownership| {
+            Participant::snapshot(
+                GenerationId::new(7),
+                ParticipantRole::Sandbox,
+                state,
+                Some(ParticipantOwnership::Sandbox(ownership)),
+            )
+            .unwrap()
+        };
+        store
+            .apply(&[DomainStoreMutation::PutParticipant(sandbox_snapshot(
+                ParticipantState::Intent,
+                sandbox.clone(),
+            ))])
+            .unwrap();
+        sandbox.runtime_supervisor = PersistedHostProcessOwnership::staged(process(71));
+        store
+            .apply(&[DomainStoreMutation::PutParticipant(sandbox_snapshot(
+                ParticipantState::Intent,
+                sandbox.clone(),
+            ))])
+            .unwrap();
+        sandbox.runtime_supervisor = PersistedHostProcessOwnership::committed(process(71));
+        sandbox.attached_runsc = PersistedHostProcessOwnership::staged(process(72));
+        store
+            .apply(&[DomainStoreMutation::PutParticipant(sandbox_snapshot(
+                ParticipantState::Intent,
+                sandbox.clone(),
+            ))])
+            .unwrap();
+        sandbox.attached_runsc = PersistedHostProcessOwnership::committed(process(72));
+        store
+            .apply(&[DomainStoreMutation::PutParticipant(sandbox_snapshot(
+                ParticipantState::Committed,
+                sandbox,
+            ))])
+            .unwrap();
+        for (index, session_id) in session_ids.iter().enumerate() {
+            let mut session = Session {
+                generation_id: GenerationId::new(7),
+                session_id: *session_id,
+                cli_id: ClientId::new(bytes(20 + u8::try_from(index).unwrap())),
+                session_anchor_id: SessionAnchorId::new(bytes(30 + u8::try_from(index).unwrap())),
+                session_phase: SessionPhase::Registering,
+            };
+            store
+                .apply(&[DomainStoreMutation::PutSession(session.clone())])
+                .unwrap();
+            session.session_phase = SessionPhase::Attached;
+            store
+                .apply(&[DomainStoreMutation::PutSession(session)])
+                .unwrap();
+        }
+        Self {
+            _directory: directory,
+            store,
+        }
+    }
+}
+
+#[derive(Debug)]
+struct MockState {
+    output: VecDeque<Vec<u8>>,
+    log: Arc<Mutex<Vec<String>>>,
+    group_present: bool,
+    identity_mismatch: bool,
+    exec_exited: bool,
+}
+
+struct MockProcess {
+    state: Arc<Mutex<MockState>>,
+}
+
+impl SessionTerminalProcess for MockProcess {
+    fn write_input(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        self.state
+            .lock()
+            .unwrap()
+            .log
+            .lock()
+            .unwrap()
+            .push(format!("input:{}", bytes.len()));
+        Ok(bytes.len())
+    }
+
+    fn resize(&self, resize: TerminalResize) -> io::Result<()> {
+        self.state
+            .lock()
+            .unwrap()
+            .log
+            .lock()
+            .unwrap()
+            .push(format!("resize:{}x{}", resize.rows, resize.columns));
+        Ok(())
+    }
+
+    fn read_output(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
+        let mut state = self.state.lock().unwrap();
+        let Some(output) = state.output.pop_front() else {
+            return Err(io::ErrorKind::WouldBlock.into());
+        };
+        let length = output.len().min(bytes.len());
+        bytes[..length].copy_from_slice(&output[..length]);
+        state.log.lock().unwrap().push(format!("output:{length}"));
+        Ok(length)
+    }
+
+    fn exec_exited(&self) -> io::Result<bool> {
+        Ok(self.state.lock().unwrap().exec_exited)
+    }
+
+    fn signal_group(&self, signal: &str) -> Result<(), TerminalError> {
+        let mut state = self.state.lock().unwrap();
+        if state.identity_mismatch {
+            return Err(TerminalError::GuestIdentityMismatch);
+        }
+        state.log.lock().unwrap().push(format!("signal:{signal}"));
+        if signal == "HUP" {
+            state.group_present = false;
+        }
+        Ok(())
+    }
+
+    fn group_absent(&self) -> Result<bool, TerminalError> {
+        let state = self.state.lock().unwrap();
+        if state.identity_mismatch {
+            return Err(TerminalError::GuestIdentityMismatch);
+        }
+        Ok(!state.group_present)
+    }
+
+    fn terminate_exec(&self) -> io::Result<()> {
+        let mut state = self.state.lock().unwrap();
+        state.log.lock().unwrap().push("terminate-exec".into());
+        state.exec_exited = true;
+        Ok(())
+    }
+
+    fn reap_exec(&mut self) -> io::Result<ProcessExit> {
+        self.state
+            .lock()
+            .unwrap()
+            .log
+            .lock()
+            .unwrap()
+            .push("reap-exec".into());
+        Ok(ProcessExit {
+            exit_code: Some(17),
+            terminating_signal: None,
+        })
+    }
+
+    fn close_pty(&mut self) {
+        let state = self.state.lock().unwrap();
+        state.log.lock().unwrap().push("close-pty".into());
+    }
+}
+
+#[derive(Default)]
+struct MockFactory {
+    states: Mutex<Vec<Arc<Mutex<MockState>>>>,
+    log: Arc<Mutex<Vec<String>>>,
+}
+
+impl MockFactory {
+    fn state(&self, index: usize) -> Arc<Mutex<MockState>> {
+        Arc::clone(&self.states.lock().unwrap()[index])
+    }
+}
+
+impl SessionTerminalFactory for MockFactory {
+    fn acquire(
+        &self,
+        request: &TerminalAcquisition,
+        recorder: &HostCoordinatorTerminalRecorder,
+    ) -> Result<(SessionTerminal, Box<dyn SessionTerminalProcess>), TerminalError> {
+        let staged = staged_terminal(
+            request,
+            PtyIdentity {
+                master_fd: FileDescriptor::new(9),
+                device: DeviceId::new(10),
+                inode: InodeId::new(11),
+            },
+            StagedProcessOwnership::v1(
+                request.domain_id,
+                request.generation_id,
+                request.coordinator_id,
+                lazaret_core::records::HostProcessOwnershipSlot::SessionTerminalExecClient(
+                    request.session_id,
+                ),
+                process(90),
+            ),
+        )?;
+        recorder.put(staged)?;
+        let guest = GuestProcessIdentity {
+            leader_pid: GuestProcessId::new(41),
+            leader_start_time: GuestProcessStartTimeTicks::new(101),
+            process_group_id: GuestProcessGroupId::new(41),
+        };
+        let running = terminal_snapshot(
+            request,
+            SessionTerminalState::Running,
+            PersistedHostProcessOwnership::committed(request.host_coordinator),
+            Some(PtyIdentity {
+                master_fd: FileDescriptor::new(9),
+                device: DeviceId::new(10),
+                inode: InodeId::new(11),
+            }),
+            PersistedHostProcessOwnership::committed(process(90)),
+            Some(guest),
+        )?;
+        recorder.put(running.clone())?;
+        let state = Arc::new(Mutex::new(MockState {
+            output: VecDeque::new(),
+            log: Arc::clone(&self.log),
+            group_present: true,
+            identity_mismatch: false,
+            exec_exited: false,
+        }));
+        self.states.lock().unwrap().push(Arc::clone(&state));
+        Ok((running, Box::new(MockProcess { state })))
+    }
+}
+
+struct Cancellation(Arc<Mutex<Vec<String>>>);
+
+impl PromptRouteCancellation for Cancellation {
+    fn cancel_for_close(&self, _session_id: SessionId) -> Result<(), TerminalError> {
+        self.0.lock().unwrap().push("cancel-route".into());
+        Ok(())
+    }
+}
+
+fn acquisition(session_id: SessionId) -> TerminalAcquisition {
+    TerminalAcquisition {
+        domain_id: DomainId::new(bytes(1)),
+        generation_id: GenerationId::new(7),
+        coordinator_id: CoordinatorId::new(bytes(3)),
+        session_id,
+        container_id: ContainerId::new(bytes(9)),
+        host_coordinator: process(70),
+        runsc_program: "runsc".into(),
+        runsc_global_arguments: Vec::new(),
+        internal_pid_file: "/tmp/not-used-by-mock".into(),
+        argv: vec![b"/bin/agent".to_vec()],
+    }
+}
+
+fn setup(ids: &[SessionId]) -> (Fixture, Arc<MockFactory>, SessionTerminalEngine) {
+    let fixture = Fixture::new(ids);
+    let factory = Arc::new(MockFactory::default());
+    let engine = SessionTerminalEngine::new(
+        Arc::clone(&fixture.store),
+        factory.clone(),
+        Arc::new(Cancellation(Arc::clone(&factory.log))),
+    );
+    (fixture, factory, engine)
+}
+
+fn id(value: u8) -> SessionId {
+    SessionId::new(bytes(value))
+}
+
+#[test]
+fn acquisition_refuses_authority_outside_the_committed_sandbox() {
+    let session = id(6);
+    let (fixture, _factory, mut engine) = setup(&[session]);
+    let mut request = acquisition(session);
+    request.host_coordinator = process(700);
+    assert!(matches!(
+        engine.acquire(&request),
+        Err(TerminalError::AuthorityMismatch)
+    ));
+    assert!(
+        fixture
+            .store
+            .snapshot()
+            .unwrap()
+            .session_terminals
+            .is_empty()
+    );
+}
+
+#[test]
+fn resize_and_terminal_signals_use_the_exact_group_map() {
+    let session = id(6);
+    let (_fixture, factory, mut engine) = setup(&[session]);
+    engine.acquire(&acquisition(session)).unwrap();
+    engine
+        .handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Resize(TerminalResize {
+                session_id: session,
+                rows: NonZeroU16::new(40).unwrap(),
+                columns: NonZeroU16::new(120).unwrap(),
+                pixel_width: 0,
+                pixel_height: 0,
+            }),
+        )
+        .unwrap();
+    for (kind, expected) in [
+        (TerminalSignalKind::Interrupt, "signal:INT"),
+        (TerminalSignalKind::Quit, "signal:QUIT"),
+        (TerminalSignalKind::Suspend, "signal:TSTP"),
+    ] {
+        engine
+            .handle(
+                GenerationId::new(7),
+                SessionTerminalEvent::Signal(TerminalSignal {
+                    session_id: session,
+                    signal: kind,
+                }),
+            )
+            .unwrap();
+        assert_eq!(factory.log.lock().unwrap().last().unwrap(), expected);
+    }
+    assert_eq!(
+        &factory.log.lock().unwrap()[..2],
+        &["resize:40x120", "signal:WINCH"]
+    );
+}
+
+#[test]
+fn changed_guest_identity_refuses_numeric_group_action() {
+    let session = id(6);
+    let (_fixture, factory, mut engine) = setup(&[session]);
+    engine.acquire(&acquisition(session)).unwrap();
+    factory.state(0).lock().unwrap().identity_mismatch = true;
+    assert!(matches!(
+        engine.handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Signal(TerminalSignal {
+                session_id: session,
+                signal: TerminalSignalKind::Interrupt,
+            }),
+        ),
+        Err(TerminalError::GuestIdentityMismatch)
+    ));
+    assert!(factory.log.lock().unwrap().is_empty());
+}
+
+#[test]
+fn merged_output_is_ordered_and_prompt_route_isolated_by_session() {
+    let first = id(6);
+    let second = id(7);
+    let (_fixture, factory, mut engine) = setup(&[first, second]);
+    engine.acquire(&acquisition(first)).unwrap();
+    engine.acquire(&acquisition(second)).unwrap();
+    factory
+        .state(0)
+        .lock()
+        .unwrap()
+        .output
+        .extend([b"one".to_vec(), b"two".to_vec()]);
+    factory
+        .state(1)
+        .lock()
+        .unwrap()
+        .output
+        .push_back(b"other".to_vec());
+    engine
+        .set_route(first, GenerationId::new(7), TerminalRoute::Prompt)
+        .unwrap();
+    assert!(engine.poll(first, GenerationId::new(7)).unwrap().is_empty());
+    assert_eq!(
+        engine.poll(second, GenerationId::new(7)).unwrap(),
+        vec![SessionTerminalMessage::Output(TerminalOutput {
+            session_id: second,
+            bytes: b"other".to_vec(),
+        })]
+    );
+    engine
+        .set_route(first, GenerationId::new(7), TerminalRoute::Sandbox)
+        .unwrap();
+    let mut merged = Vec::new();
+    for _ in 0..2 {
+        let messages = engine.poll(first, GenerationId::new(7)).unwrap();
+        let [SessionTerminalMessage::Output(output)] = messages.as_slice() else {
+            panic!("expected one merged output message");
+        };
+        merged.extend_from_slice(&output.bytes);
+    }
+    assert_eq!(merged, b"onetwo");
+}
+
+#[test]
+fn close_cancels_route_then_bounds_drain_and_emits_one_final_exit() {
+    let session = id(6);
+    let (fixture, factory, mut engine) = setup(&[session]);
+    engine.acquire(&acquisition(session)).unwrap();
+    factory
+        .state(0)
+        .lock()
+        .unwrap()
+        .output
+        .extend((0..10).map(|_| vec![b'x'; MAX_OUTPUT_BYTES]));
+    let messages = engine
+        .handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Close(lazaret_core::records::SessionClose {
+                session_id: session,
+            }),
+        )
+        .unwrap();
+    let bytes = messages
+        .iter()
+        .filter_map(|message| match message {
+            SessionTerminalMessage::Output(output) => Some(output.bytes.len()),
+            SessionTerminalMessage::Exit(_) => None,
+        })
+        .sum::<usize>();
+    assert_eq!(bytes, MAX_DRAIN_BYTES);
+    assert!(
+        matches!(messages.last(), Some(SessionTerminalMessage::Exit(exit)) if exit.exit_code() == Some(17))
+    );
+    assert_eq!(
+        &factory.log.lock().unwrap()[..2],
+        &["cancel-route", "signal:HUP"]
+    );
+    assert!(!engine.contains(session));
+    assert!(matches!(
+        engine.handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Input(TerminalInput {
+                session_id: session,
+                bytes: b"reattach".to_vec(),
+            }),
+        ),
+        Err(TerminalError::UnknownSession)
+    ));
+    assert_eq!(
+        fixture.store.snapshot().unwrap().session_terminals[0].state(),
+        SessionTerminalState::Clean
+    );
+}
+
+#[test]
+fn disconnect_component_loss_and_hard_hc_loss_never_reattach() {
+    for loss in [
+        TerminalLoss::ClientDisconnected,
+        TerminalLoss::PtyLost,
+        TerminalLoss::ExecClientLost,
+    ] {
+        let session = id(6);
+        let (_fixture, _factory, mut engine) = setup(&[session]);
+        engine.acquire(&acquisition(session)).unwrap();
+        let messages = engine.lose(session, GenerationId::new(7), loss).unwrap();
+        assert_eq!(
+            messages.is_empty(),
+            loss == TerminalLoss::ClientDisconnected
+        );
+        assert!(!engine.contains(session));
+    }
+
+    let session = id(6);
+    let (fixture, _factory, mut engine) = setup(&[session]);
+    engine.acquire(&acquisition(session)).unwrap();
+    assert!(
+        engine
+            .lose(
+                session,
+                GenerationId::new(7),
+                TerminalLoss::HostCoordinatorLost,
+            )
+            .unwrap()
+            .is_empty()
+    );
+    assert!(!engine.contains(session));
+    assert_eq!(
+        fixture.store.snapshot().unwrap().session_terminals[0].state(),
+        SessionTerminalState::Running
+    );
+}