Add coordinator-owned interactive session terminals

crates/lazaret-host-coordinator/tests/runtime-composition.rs

@@ -7,6 +7,7 @@ use std::os::unix::fs::symlink;
 use std::os::unix::net::{SocketAddr, UnixListener, UnixStream};
 use std::path::{Path, PathBuf};
 use std::process::ExitCode;
+use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 
@@ -14,18 +15,24 @@ use lazaret_core::configuration::{
     ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
 };
 use lazaret_core::identity::{
-    BundleDigest, ContainerId, CoordinatorId, GenerationId, HostPath, NetworkSidecarId, SandboxId,
+    BundleDigest, ClientId, ContainerId, CoordinatorId, GenerationId, HostPath, NetworkSidecarId,
+    SandboxId, SessionAnchorId, SessionId,
 };
 use lazaret_core::network::{DnsQueryExpectation, ReachabilityExpectation, derive_ipv4_policy};
 use lazaret_core::records::{
     Domain, DurableDecision, Generation, HostProcessOwnershipSlot, IpAddress, ParticipantRole,
-    ParticipantState, ProcessOwnershipCommitAck,
+    ParticipantState, ProcessOwnershipCommitAck, Session, SessionPhase, SessionTerminalEvent,
+    SessionTerminalMessage, TerminalResize, TerminalSignal, TerminalSignalKind,
 };
 use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
 use lazaret_host_coordinator::network::NetworkCoordinator;
 use lazaret_host_coordinator::runtime::{
     AwaitingAttachedRunsc, CommittedSandboxRuntime, SandboxCoordinator,
 };
+use lazaret_host_coordinator::terminal::{
+    NoActivePrompt, RunscSessionTerminalFactory, SessionTerminalEngine, TerminalAcquisition,
+    TerminalRoute,
+};
 use lazaret_linux::filesystem::try_exclusive_flock;
 use lazaret_linux::platform::{PlatformObservation, check_platform};
 use lazaret_linux::process::{
@@ -48,6 +55,7 @@ const HOST_COORDINATOR_DRIVER: &str = "--runtime-composition-host-coordinator";
 const SANDBOX_NETWORK_PROBE: &str = "--sandbox-network-probe";
 const NETWORK_MATRIX: &str = "--network-matrix";
 const HANDOFF_MATRIX: &str = "--handoff-matrix";
+const SESSION_TERMINAL_MATRIX: &str = "--session-terminal-matrix";
 const BENCHMARK_DRIVER: &str = "--runtime-composition-benchmark-command";
 const BENCHMARK_GUEST: &str = "--runtime-composition-benchmark-guest";
 const BENCHMARK_COMPLETE: &str = "benchmark-complete";
@@ -61,6 +69,11 @@ fn main() -> ExitCode {
     if let Some(result) = dispatch_private_runtime_role(&arguments) {
         return exit(result);
     }
+    if let Some(result) =
+        lazaret_host_coordinator::sandbox_init::dispatch_private_sandbox_init(&arguments)
+    {
+        return exit(result);
+    }
     if arguments.first().and_then(|argument| argument.to_str()) == Some(BENCHMARK_GUEST) {
         return exit(run_benchmark_guest(&arguments[1..]));
     }
@@ -105,6 +118,22 @@ fn main() -> ExitCode {
             }
         };
     }
+    if arguments.first().and_then(|argument| argument.to_str()) == Some(SESSION_TERMINAL_MATRIX) {
+        return match run_session_terminal_matrix() {
+            Ok(()) => {
+                println!("PASS production Session Terminal matrix");
+                ExitCode::SUCCESS
+            }
+            Err(MatrixError::Skip(reason)) => {
+                println!("SKIP production Session Terminal matrix: {reason}");
+                ExitCode::from(SKIP)
+            }
+            Err(MatrixError::Failure(error)) => {
+                eprintln!("production Session Terminal matrix failed: {error}");
+                ExitCode::FAILURE
+            }
+        };
+    }
     match run_matrix() {
         Ok(()) => {
             println!("PASS foreground runtime composition and causal-loss matrix");
@@ -241,6 +270,202 @@ fn run_network_matrix() -> Result<(), MatrixError> {
     run_network_cases()
 }
 
+#[allow(clippy::too_many_lines)]
+fn run_session_terminal_matrix() -> Result<(), MatrixError> {
+    require_matrix_prerequisites()?;
+    let fixture = Fixture::new()?;
+    let runtime_domain = host_path(
+        fixture
+            .configuration
+            .runtime_domain_root()
+            .canonical_label(),
+    );
+    fs::copy(
+        env!("CARGO_BIN_EXE_lazaret-host-coordinator"),
+        runtime_domain
+            .join("sandbox-control")
+            .join(hex(ContainerId::new([6; 32]).as_bytes()))
+            .join("bin/lazaret-sandbox-init"),
+    )?;
+    let runtime = fixture.launch_committed()?;
+    let session_id = SessionId::new([31; 32]);
+    let mut session = Session {
+        generation_id: GenerationId::new(7),
+        session_id,
+        cli_id: ClientId::new([32; 32]),
+        session_anchor_id: SessionAnchorId::new([33; 32]),
+        session_phase: SessionPhase::Registering,
+    };
+    fixture
+        .store
+        .apply(&[DomainStoreMutation::PutSession(session.clone())])
+        .map_err(failure)?;
+    session.session_phase = SessionPhase::Attached;
+    fixture
+        .store
+        .apply(&[DomainStoreMutation::PutSession(session)])
+        .map_err(failure)?;
+    let command = fixture.bundle.command();
+    let run = command
+        .arguments()
+        .iter()
+        .position(|argument| argument == "run")
+        .ok_or_else(|| failure("production runsc command has no run action"))?;
+    let workspace = host_path(fixture.configuration.workspace().canonical_label());
+    let count = workspace.join("session-count");
+    let finish = workspace.join("session-finish");
+    let signals = workspace.join("session-signals");
+    let line = "x".repeat(1024);
+    let script = format!(
+        "trap 'echo INT >> /workspace/session-signals' INT; \
+         trap 'echo QUIT >> /workspace/session-signals' QUIT; \
+         trap 'echo TSTP >> /workspace/session-signals' TSTP; \
+         trap 'echo WINCH >> /workspace/session-signals' WINCH; \
+         n=0; while test ! -e /workspace/session-finish; do \
+         echo '{line}'; n=$((n+1)); echo $n > /workspace/session-count; done; exit 23"
+    );
+    let request = TerminalAcquisition {
+        domain_id: fixture.configuration.domain_id(),
+        generation_id: GenerationId::new(7),
+        coordinator_id: CoordinatorId::new([3; 32]),
+        session_id,
+        container_id: ContainerId::new([6; 32]),
+        host_coordinator: current_process_identity()?,
+        runsc_program: command.program().to_owned(),
+        runsc_global_arguments: command.arguments()[..run].to_vec(),
+        internal_pid_file: runtime_domain.join("session-internal.pid"),
+        argv: vec![b"/bin/sh".to_vec(), b"-c".to_vec(), script.into_bytes()],
+    };
+    let mut engine = SessionTerminalEngine::new(
+        Arc::clone(&fixture.store),
+        Arc::new(RunscSessionTerminalFactory),
+        Arc::new(NoActivePrompt),
+    );
+    engine.acquire(&request).map_err(failure)?;
+    wait_for_path(&count)?;
+    let running = fixture
+        .store
+        .snapshot()
+        .map_err(failure)?
+        .session_terminals
+        .into_iter()
+        .find(|terminal| terminal.session_id() == session_id)
+        .ok_or_else(|| failure("running Session Terminal record is absent"))?;
+    let guest = running
+        .guest_process()
+        .ok_or_else(|| failure("running Session Terminal guest identity is absent"))?;
+    if guest.leader_pid.get() != guest.process_group_id.get() {
+        return Err(failure("guest leader does not own its process group"));
+    }
+    engine
+        .handle(
+            GenerationId::new(7),
+            SessionTerminalEvent::Resize(TerminalResize {
+                session_id,
+                rows: std::num::NonZeroU16::new(41).unwrap(),
+                columns: std::num::NonZeroU16::new(121).unwrap(),
+                pixel_width: 0,
+                pixel_height: 0,
+            }),
+        )
+        .map_err(failure)?;
+    for signal in [
+        TerminalSignalKind::Interrupt,
+        TerminalSignalKind::Quit,
+        TerminalSignalKind::Suspend,
+    ] {
+        engine
+            .handle(
+                GenerationId::new(7),
+                SessionTerminalEvent::Signal(TerminalSignal { session_id, signal }),
+            )
+            .map_err(failure)?;
+    }
+    wait_for_signal_lines(&signals, 4)?;
+    engine
+        .set_route(session_id, GenerationId::new(7), TerminalRoute::Prompt)
+        .map_err(failure)?;
+    thread::sleep(Duration::from_millis(300));
+    let before = fs::read_to_string(&count)?;
+    thread::sleep(Duration::from_millis(300));
+    let after = fs::read_to_string(&count)?;
+    if before != after {
+        return Err(failure("prompt route did not apply PTY backpressure"));
+    }
+    engine
+        .set_route(session_id, GenerationId::new(7), TerminalRoute::Sandbox)
+        .map_err(failure)?;
+    let mut output = 0_usize;
+    for _ in 0..2_000 {
+        for message in engine
+            .poll(session_id, GenerationId::new(7))
+            .map_err(failure)?
+        {
+            if let SessionTerminalMessage::Output(chunk) = message {
+                output += chunk.bytes.len();
+            }
+        }
+        if fs::read_to_string(&count).ok().as_deref() != Some(after.as_str()) {
+            break;
+        }
+        thread::sleep(Duration::from_millis(5));
+    }
+    if output == 0 || fs::read_to_string(&count).ok().as_deref() == Some(after.as_str()) {
+        return Err(failure("sandbox route did not resume ordered PTY output"));
+    }
+    fs::write(&finish, b"finish")?;
+    let mut exit = None;
+    for _ in 0..2_000 {
+        for message in engine
+            .poll(session_id, GenerationId::new(7))
+            .map_err(failure)?
+        {
+            if let SessionTerminalMessage::Exit(outcome) = message {
+                exit = Some(outcome);
+            }
+        }
+        if exit.is_some() {
+            break;
+        }
+        thread::sleep(Duration::from_millis(5));
+    }
+    if exit.and_then(lazaret_core::records::SessionExit::exit_code) != Some(23)
+        || engine.contains(session_id)
+    {
+        return Err(failure(
+            "Session Terminal exit or no-reattachment cleanup failed",
+        ));
+    }
+    runtime
+        .stop(&SandboxCoordinator::new(&fixture.store))
+        .map_err(failure)
+}
+
+fn wait_for_path(path: &Path) -> Result<(), MatrixError> {
+    for _ in 0..2_000 {
+        if path.is_file() {
+            return Ok(());
+        }
+        thread::sleep(Duration::from_millis(5));
+    }
+    Err(failure("Session Terminal guest did not become ready"))
+}
+
+fn wait_for_signal_lines(path: &Path, expected: usize) -> Result<(), MatrixError> {
+    for _ in 0..2_000 {
+        if fs::read_to_string(path)
+            .ok()
+            .is_some_and(|value| value.lines().count() >= expected)
+        {
+            return Ok(());
+        }
+        thread::sleep(Duration::from_millis(5));
+    }
+    Err(failure(
+        "Session Terminal guest did not observe exact signals",
+    ))
+}
+
 fn run_handoff_matrix() -> Result<(), MatrixError> {
     require_matrix_prerequisites()?;
     replaced_handoff_authority_refuses_target_work()?;
@@ -575,7 +800,7 @@ fn assert_activity_stable(path: &Path) -> Result<(), MatrixError> {
 struct Fixture {
     _tree: Option<TempDir>,
     configuration: ResolvedDomainConfiguration,
-    store: HostCoordinatorDomainStore,
+    store: Arc<HostCoordinatorDomainStore>,
     bundle: SandboxBundle,
     activity: PathBuf,
     benchmark_results: Option<PathBuf>,
@@ -778,9 +1003,10 @@ impl Fixture {
         };
         let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label());
         let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?;
-        let store =
+        let store = Arc::new(
             HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain)
-                .map_err(failure)?;
+                .map_err(failure)?,
+        );
         store
             .apply(&[DomainStoreMutation::PutGeneration(Generation {
                 domain_id: configuration.domain_id(),