Run non-interactive sessions through production coordination

crates/lazaret-client/tests/cli_contract.rs

@@ -0,0 +1,255 @@
+use std::collections::VecDeque;
+use std::convert::Infallible;
+use std::ffi::OsString;
+use std::io::Write;
+use std::path::PathBuf;
+
+use lazaret_client::{
+    BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, DomainStatus,
+    parse_command_line, run,
+};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum Endpoint {
+    Existing,
+    Started,
+    Published,
+}
+
+#[derive(Default)]
+struct FakeBackend {
+    observations: VecDeque<DomainStatus<Endpoint>>,
+    acquisition: Option<BootstrapAcquisition>,
+    bootstrap: Option<Result<Endpoint, ClientError<Infallible>>>,
+    waited: Option<Result<Endpoint, ClientError<Infallible>>>,
+    execution: Option<Result<i32, ClientError<Infallible>>>,
+    actions: Vec<&'static str>,
+    validation: Option<ClientError<Infallible>>,
+}
+
+impl ClientBackend for FakeBackend {
+    type Endpoint = Endpoint;
+    type Error = Infallible;
+
+    fn validate(&mut self, _command: &ClientCommand) -> Result<(), ClientError<Self::Error>> {
+        self.actions.push("validate");
+        self.validation.take().map_or(Ok(()), Err)
+    }
+
+    fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error> {
+        self.actions.push("observe");
+        Ok(self.observations.pop_front().unwrap())
+    }
+
+    fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
+        self.actions.push("acquire");
+        Ok(self.acquisition.unwrap())
+    }
+
+    fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
+        self.actions.push("bootstrap");
+        self.bootstrap.take().unwrap()
+    }
+
+    fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
+        self.actions.push("wait");
+        self.waited.take().unwrap()
+    }
+
+    fn execute(
+        &mut self,
+        _endpoint: Self::Endpoint,
+        _argv: &[Vec<u8>],
+        stdout: &mut dyn Write,
+        stderr: &mut dyn Write,
+    ) -> Result<i32, ClientError<Self::Error>> {
+        self.actions.push("execute");
+        stdout.write_all(b"out").unwrap();
+        stderr.write_all(b"err").unwrap();
+        self.execution.take().unwrap_or(Ok(0))
+    }
+}
+
+fn command() -> ClientCommand {
+    ClientCommand {
+        profile: "agent".into(),
+        workspace: PathBuf::from("/workspace"),
+        configuration: None,
+        argv: vec![b"sh".to_vec(), b"-c".to_vec(), b"exit 0".to_vec()],
+    }
+}
+
+fn invoke(backend: &mut FakeBackend) -> Result<(i32, Vec<u8>, Vec<u8>), String> {
+    let mut stdout = Vec::new();
+    let mut stderr = Vec::new();
+    run(backend, &command(), &mut stdout, &mut stderr)
+        .map(|status| (status, stdout, stderr))
+        .map_err(|error| error.to_string())
+}
+
+#[test]
+fn cold_start_leader_bootstraps_before_forwarding_streams() {
+    let mut backend = FakeBackend {
+        observations: [DomainStatus::Absent].into(),
+        acquisition: Some(BootstrapAcquisition::Leader),
+        bootstrap: Some(Ok(Endpoint::Started)),
+        ..FakeBackend::default()
+    };
+
+    assert_eq!(
+        invoke(&mut backend),
+        Ok((0, b"out".to_vec(), b"err".to_vec()))
+    );
+    assert_eq!(
+        backend.actions,
+        ["validate", "observe", "acquire", "bootstrap", "execute"]
+    );
+}
+
+#[test]
+fn ready_domain_is_joined_without_bootstrap() {
+    let mut backend = FakeBackend {
+        observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
+        ..FakeBackend::default()
+    };
+
+    assert_eq!(invoke(&mut backend).unwrap().0, 0);
+    assert_eq!(backend.actions, ["validate", "observe", "execute"]);
+}
+
+#[test]
+fn bootstrap_owner_disconnect_fails_that_closed_session() {
+    let mut backend = FakeBackend {
+        observations: [DomainStatus::Absent].into(),
+        acquisition: Some(BootstrapAcquisition::Leader),
+        bootstrap: Some(Err(ClientError::BootstrapOwnerDisconnected)),
+        ..FakeBackend::default()
+    };
+
+    assert!(
+        invoke(&mut backend)
+            .unwrap_err()
+            .contains("bootstrap owner disconnected")
+    );
+    assert_eq!(
+        backend.actions,
+        ["validate", "observe", "acquire", "bootstrap"]
+    );
+}
+
+#[test]
+fn distinct_waiter_joins_after_publication() {
+    let mut backend = FakeBackend {
+        observations: [DomainStatus::Starting].into(),
+        waited: Some(Ok(Endpoint::Published)),
+        ..FakeBackend::default()
+    };
+
+    assert_eq!(invoke(&mut backend).unwrap().0, 0);
+    assert_eq!(backend.actions, ["validate", "observe", "wait", "execute"]);
+}
+
+#[test]
+fn distinct_waiter_is_not_restarted_after_owner_loss() {
+    let mut backend = FakeBackend {
+        observations: [DomainStatus::Starting].into(),
+        waited: Some(Err(ClientError::WaitingDomainFailed)),
+        ..FakeBackend::default()
+    };
+
+    assert!(
+        invoke(&mut backend)
+            .unwrap_err()
+            .contains("failed while this Client was waiting")
+    );
+    assert_eq!(backend.actions, ["validate", "observe", "wait"]);
+}
+
+#[test]
+fn nonzero_guest_exit_is_the_cli_status() {
+    let mut backend = FakeBackend {
+        observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
+        execution: Some(Ok(23)),
+        ..FakeBackend::default()
+    };
+
+    assert_eq!(invoke(&mut backend).unwrap().0, 23);
+}
+
+#[test]
+fn stale_and_lost_coordinators_are_actionable_failures() {
+    let mut stale = FakeBackend {
+        observations: [DomainStatus::Stale].into(),
+        ..FakeBackend::default()
+    };
+    assert!(invoke(&mut stale).unwrap_err().contains("stale"));
+
+    let mut lost = FakeBackend {
+        observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
+        execution: Some(Err(ClientError::LostConnection)),
+        ..FakeBackend::default()
+    };
+    assert!(
+        invoke(&mut lost)
+            .unwrap_err()
+            .contains("connection was lost")
+    );
+}
+
+#[test]
+fn unsupported_host_and_configuration_errors_keep_their_cause() {
+    for (error, expected) in [
+        (
+            ClientError::UnsupportedHost("Linux kernel 6.8 is below the required 6.9 floor".into()),
+            "host is unsupported: Linux kernel 6.8 is below the required 6.9 floor",
+        ),
+        (
+            ClientError::Configuration("profile `agent` is not configured".into()),
+            "configuration is invalid: profile `agent` is not configured",
+        ),
+    ] {
+        let mut backend = FakeBackend {
+            validation: Some(error),
+            ..FakeBackend::default()
+        };
+        assert_eq!(invoke(&mut backend).unwrap_err(), expected);
+        assert_eq!(backend.actions, ["validate"]);
+    }
+}
+
+#[test]
+fn command_line_requires_an_explicit_profile_and_preserves_command_bytes() {
+    let parsed = parse_command_line(
+        [
+            "--profile",
+            "agent",
+            "--workspace",
+            "/project",
+            "--config",
+            "/host/config.toml",
+            "--",
+            "tool",
+            "argument",
+        ]
+        .map(OsString::from),
+        PathBuf::from("/current"),
+    )
+    .unwrap();
+    assert_eq!(parsed.profile, "agent");
+    assert_eq!(parsed.workspace, PathBuf::from("/project"));
+    assert_eq!(
+        parsed.configuration,
+        Some(PathBuf::from("/host/config.toml"))
+    );
+    assert_eq!(parsed.argv, [b"tool".to_vec(), b"argument".to_vec()]);
+
+    assert!(
+        parse_command_line(
+            ["--", "tool"].map(OsString::from),
+            PathBuf::from("/current")
+        )
+        .unwrap_err()
+        .to_string()
+        .contains("profile")
+    );
+}

crates/lazaret-client/tests/control_face.rs

@@ -0,0 +1,164 @@
+use std::fs::OpenOptions;
+use std::os::unix::fs::PermissionsExt;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use lazaret_client::{ControlEndpoint, execute_over_control_face};
+use lazaret_core::identity::{
+    ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, HostPath, NetworkSidecarId,
+    SandboxId,
+};
+use lazaret_core::records::{
+    Domain, DurableDecision, Generation, GenerationIdentity, NonInteractiveOutput,
+    NonInteractiveSessionMessage, NonInteractiveSessionRequest, SessionExit,
+};
+use lazaret_host_coordinator::control::{
+    ControlError, NonInteractiveSessionExecutor, PublishedControlFace, TrustAnchor,
+};
+use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
+use lazaret_linux::filesystem::try_exclusive_flock;
+
+struct TestDirectory(PathBuf);
+
+impl TestDirectory {
+    fn new() -> Self {
+        let path = std::env::temp_dir().join(format!(
+            "lazaret-client-control-{}-{}",
+            std::process::id(),
+            random_suffix()
+        ));
+        std::fs::create_dir(&path).unwrap();
+        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
+        Self(path)
+    }
+}
+
+impl Drop for TestDirectory {
+    fn drop(&mut self) {
+        std::fs::remove_dir_all(&self.0).unwrap();
+    }
+}
+
+struct Executor;
+
+impl NonInteractiveSessionExecutor for Executor {
+    fn execute(
+        &self,
+        request: &NonInteractiveSessionRequest,
+        emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
+    ) -> Result<SessionExit, ControlError> {
+        assert_eq!(request.argv, [b"tool".to_vec(), b"argument".to_vec()]);
+        emit(NonInteractiveSessionMessage::Stdout(NonInteractiveOutput {
+            session_id: request.session_id,
+            bytes: b"first".to_vec(),
+        }))?;
+        emit(NonInteractiveSessionMessage::Stderr(NonInteractiveOutput {
+            session_id: request.session_id,
+            bytes: b"second".to_vec(),
+        }))?;
+        Ok(SessionExit::exited(request.session_id, 19))
+    }
+}
+
+#[test]
+fn client_enrolls_registers_forwards_both_streams_and_returns_guest_status() {
+    let directory = TestDirectory::new();
+    let identity = generation_identity();
+    let store = open_store(&directory, &identity);
+    let face = Arc::new(
+        PublishedControlFace::publish_with_executor(
+            &directory.0,
+            rustix::process::getuid().as_raw(),
+            TrustAnchor::create(identity).unwrap(),
+            store,
+            Arc::new(Executor),
+        )
+        .unwrap(),
+    );
+    let endpoint = ControlEndpoint::new(
+        face.socket_path().to_path_buf(),
+        face.publication_identity().clone(),
+    );
+    let server = {
+        let face = Arc::clone(&face);
+        std::thread::spawn(move || {
+            face.serve_next().unwrap();
+            face.serve_next().unwrap();
+        })
+    };
+    let mut stdout = Vec::new();
+    let mut stderr = Vec::new();
+
+    let status = execute_over_control_face(
+        &endpoint,
+        &[b"tool".to_vec(), b"argument".to_vec()],
+        &mut stdout,
+        &mut stderr,
+    )
+    .unwrap();
+
+    server.join().unwrap();
+    assert_eq!(status, 19);
+    assert_eq!(stdout, b"first");
+    assert_eq!(stderr, b"second");
+}
+
+fn generation_identity() -> GenerationIdentity {
+    GenerationIdentity {
+        agent_profile: "trusted".into(),
+        domain_id: DomainId::new([1; 32]),
+        generation_id: GenerationId::new(1),
+        configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
+        coordinator_id: CoordinatorId::new([3; 32]),
+        sandbox_id: SandboxId::new([4; 32]),
+    }
+}
+
+fn open_store(
+    directory: &TestDirectory,
+    identity: &GenerationIdentity,
+) -> Arc<HostCoordinatorDomainStore> {
+    let lock = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create_new(true)
+        .open(directory.0.join("domain.lock"))
+        .unwrap();
+    let domain = Domain {
+        domain_id: identity.domain_id,
+        profile_name: identity.agent_profile.clone(),
+        canonical_workspace: HostPath::new(b"/workspace/project".to_vec()),
+    };
+    let store = Arc::new(
+        HostCoordinatorDomainStore::open(
+            try_exclusive_flock(lock).unwrap(),
+            &directory.0.join("domain.redb"),
+            &domain,
+        )
+        .unwrap(),
+    );
+    for durable_decision in [
+        DurableDecision::None,
+        DurableDecision::Preparing,
+        DurableDecision::Published,
+    ] {
+        store
+            .apply(&[DomainStoreMutation::PutGeneration(Generation {
+                domain_id: identity.domain_id,
+                generation_id: identity.generation_id,
+                configuration_fingerprint: identity.configuration_fingerprint,
+                coordinator_id: identity.coordinator_id,
+                network_sidecar_id: NetworkSidecarId::new([5; 32]),
+                sandbox_id: identity.sandbox_id,
+                durable_decision,
+            })])
+            .unwrap();
+    }
+    store
+}
+
+fn random_suffix() -> u64 {
+    let mut bytes = [0_u8; 8];
+    getrandom::fill(&mut bytes).unwrap();
+    u64::from_ne_bytes(bytes)
+}