Orchestrate normal domain lifecycle through adapters

crates/lazaret-host-coordinator/tests/lifecycle.rs

@@ -0,0 +1,347 @@
+use std::convert::Infallible;
+use std::fs::OpenOptions;
+use std::sync::Mutex;
+
+use lazaret_core::identity::{
+    ClientId, ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, HostPath,
+    NetworkSidecarId, SandboxId, SessionAnchorId, SessionId,
+};
+use lazaret_core::records::{Domain, DurableDecision, Generation, Session, SessionPhase};
+use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore;
+use lazaret_host_coordinator::lifecycle::{
+    DomainPhase, HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, LifecycleError,
+};
+use lazaret_linux::filesystem::try_exclusive_flock;
+
+#[derive(Default)]
+struct RecordingAdapter {
+    actions: Vec<LifecycleAction>,
+}
+
+impl LifecycleAdapter for RecordingAdapter {
+    type Error = Infallible;
+
+    fn execute(
+        &mut self,
+        action: LifecycleAction,
+        _generation: &Generation,
+    ) -> Result<(), Self::Error> {
+        self.actions.push(action);
+        Ok(())
+    }
+}
+
+fn bytes(value: u8) -> [u8; 32] {
+    [value; 32]
+}
+
+fn domain() -> Domain {
+    Domain {
+        domain_id: DomainId::new(bytes(1)),
+        profile_name: "trusted".into(),
+        canonical_workspace: HostPath::new(b"/workspace/project".to_vec()),
+    }
+}
+
+fn fingerprint(value: u8) -> ConfigurationFingerprint {
+    ConfigurationFingerprint::new(bytes(value))
+}
+
+fn generation(id: u64) -> Generation {
+    Generation {
+        domain_id: domain().domain_id,
+        generation_id: GenerationId::new(id),
+        configuration_fingerprint: fingerprint(2),
+        coordinator_id: CoordinatorId::new(bytes(u8::try_from(id * 3).unwrap())),
+        network_sidecar_id: NetworkSidecarId::new(bytes(u8::try_from(id * 3 + 1).unwrap())),
+        sandbox_id: SandboxId::new(bytes(u8::try_from(id * 3 + 2).unwrap())),
+        durable_decision: DurableDecision::None,
+    }
+}
+
+fn session(id: u8, generation_id: u64) -> Session {
+    Session {
+        generation_id: GenerationId::new(generation_id),
+        session_id: SessionId::new(bytes(id)),
+        cli_id: ClientId::new(bytes(id.wrapping_add(32))),
+        session_anchor_id: SessionAnchorId::new(bytes(id.wrapping_add(64))),
+        session_phase: SessionPhase::New,
+    }
+}
+
+fn open_store(directory: &tempfile::TempDir) -> HostCoordinatorDomainStore {
+    let lock_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .truncate(false)
+        .open(directory.path().join("domain.lock"))
+        .unwrap();
+    let lock = try_exclusive_flock(lock_file).unwrap();
+    HostCoordinatorDomainStore::open(lock, &directory.path().join("domain.redb"), &domain())
+        .unwrap()
+}
+
+fn complete_startup(lifecycle: &mut HostCoordinatorLifecycle<'_, RecordingAdapter>) {
+    lifecycle.create_trust_anchor().unwrap();
+    lifecycle.start_runtime_supervisor().unwrap();
+    lifecycle.lease_prepared_lower().unwrap();
+    lifecycle.mount_state_anchor().unwrap();
+    lifecycle.start_sidecar().unwrap();
+    lifecycle.start_connectivity_helper().unwrap();
+    lifecycle.mark_network_ready().unwrap();
+    lifecycle.arm_runtime_causal_edge().unwrap();
+    lifecycle.start_sandbox().unwrap();
+    lifecycle.publish_domain().unwrap();
+}
+
+#[test]
+fn exact_start_join_disconnect_and_teardown_actions_are_ordered() {
+    let directory = tempfile::tempdir().unwrap();
+    let store = open_store(&directory);
+    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
+    let owner = session(10, 1);
+    let waiter = session(11, 1);
+
+    lifecycle
+        .start_new_domain(generation(1), owner.clone())
+        .unwrap();
+    lifecycle.wait_for_domain(waiter.clone()).unwrap();
+    complete_startup(&mut lifecycle);
+    lifecycle
+        .advance_waiting_session(owner.session_id, fingerprint(2))
+        .unwrap();
+    lifecycle.register_session(owner.session_id).unwrap();
+    lifecycle
+        .advance_waiting_session(waiter.session_id, fingerprint(2))
+        .unwrap();
+    lifecycle.register_session(waiter.session_id).unwrap();
+    lifecycle.disconnect_session(owner.session_id).unwrap();
+    assert!(!lifecycle.is_teardown_ready());
+    lifecycle.disconnect_session(waiter.session_id).unwrap();
+    assert!(lifecycle.is_teardown_ready());
+    lifecycle.begin_teardown().unwrap();
+    lifecycle.stop_sandbox().unwrap();
+    lifecycle.stop_sidecar().unwrap();
+    lifecycle.unmount_state_anchor().unwrap();
+    lifecycle.release_prepared_lower_lease().unwrap();
+    lifecycle.graceful_supervisor_exit().unwrap();
+    lifecycle.finish_teardown().unwrap();
+
+    assert_eq!(lifecycle.phase(), DomainPhase::Absent);
+    assert!(!lifecycle.is_published());
+    assert_eq!(
+        lifecycle.into_adapter().actions,
+        vec![
+            LifecycleAction::StartNewDomain,
+            LifecycleAction::WaitForDomain,
+            LifecycleAction::CreateTrustAnchor,
+            LifecycleAction::StartRuntimeSupervisor,
+            LifecycleAction::LeasePreparedLower,
+            LifecycleAction::MountStateAnchor,
+            LifecycleAction::StartSidecar,
+            LifecycleAction::StartConnectivityHelper,
+            LifecycleAction::MarkNetworkReady,
+            LifecycleAction::ArmRuntimeCausalEdge,
+            LifecycleAction::StartSandbox,
+            LifecycleAction::PublishDomain,
+            LifecycleAction::AdvanceWaitingSession,
+            LifecycleAction::RegisterSession,
+            LifecycleAction::AdvanceWaitingSession,
+            LifecycleAction::RegisterSession,
+            LifecycleAction::DisconnectSession,
+            LifecycleAction::DisconnectSession,
+            LifecycleAction::BeginTeardown,
+            LifecycleAction::StopSandbox,
+            LifecycleAction::StopSidecar,
+            LifecycleAction::UnmountStateAnchor,
+            LifecycleAction::ReleasePreparedLowerLease,
+            LifecycleAction::GracefulSupervisorExit,
+            LifecycleAction::FinishTeardown,
+        ]
+    );
+    let snapshot = store.snapshot().unwrap();
+    assert_eq!(
+        snapshot.generations[0].durable_decision,
+        DurableDecision::Clean
+    );
+    assert!(
+        snapshot
+            .sessions
+            .iter()
+            .all(|row| row.session_phase == SessionPhase::Closed)
+    );
+}
+
+#[test]
+fn publication_and_invalid_transitions_fail_before_adapter_execution() {
+    let directory = tempfile::tempdir().unwrap();
+    let store = open_store(&directory);
+    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
+    lifecycle
+        .start_new_domain(generation(1), session(10, 1))
+        .unwrap();
+
+    assert!(matches!(
+        lifecycle.publish_domain(),
+        Err(LifecycleError::InvalidTransition {
+            action: LifecycleAction::PublishDomain,
+            ..
+        })
+    ));
+    assert!(matches!(
+        lifecycle.start_sandbox(),
+        Err(LifecycleError::InvalidTransition {
+            action: LifecycleAction::StartSandbox,
+            ..
+        })
+    ));
+    assert!(matches!(
+        lifecycle.mount_state_anchor(),
+        Err(LifecycleError::InvalidTransition {
+            action: LifecycleAction::MountStateAnchor,
+            ..
+        })
+    ));
+    assert_eq!(
+        lifecycle.into_adapter().actions,
+        vec![LifecycleAction::StartNewDomain]
+    );
+    assert_eq!(
+        store.snapshot().unwrap().generations[0].durable_decision,
+        DurableDecision::None
+    );
+}
+
+#[test]
+fn configuration_mismatch_is_refused_without_registration() {
+    let directory = tempfile::tempdir().unwrap();
+    let store = open_store(&directory);
+    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
+    let owner = session(10, 1);
+    lifecycle
+        .start_new_domain(generation(1), owner.clone())
+        .unwrap();
+    complete_startup(&mut lifecycle);
+    let mismatched = session(11, 1);
+
+    assert!(matches!(
+        lifecycle.start_ready_session(mismatched.clone(), fingerprint(9)),
+        Err(LifecycleError::InvalidTransition {
+            action: LifecycleAction::StartReadySession,
+            ..
+        })
+    ));
+    lifecycle
+        .reject_new_configuration_mismatch(mismatched.clone(), fingerprint(9))
+        .unwrap();
+    assert_eq!(
+        lifecycle.session_phase(mismatched.session_id),
+        Some(SessionPhase::Failed)
+    );
+    assert_eq!(lifecycle.registered_sessions(), 0);
+}
+
+#[test]
+fn waiting_session_restarts_after_normal_teardown() {
+    let directory = tempfile::tempdir().unwrap();
+    let store = open_store(&directory);
+    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
+    let owner = session(10, 1);
+    lifecycle
+        .start_new_domain(generation(1), owner.clone())
+        .unwrap();
+    complete_startup(&mut lifecycle);
+    lifecycle
+        .advance_waiting_session(owner.session_id, fingerprint(2))
+        .unwrap();
+    lifecycle.register_session(owner.session_id).unwrap();
+    lifecycle.disconnect_session(owner.session_id).unwrap();
+    lifecycle.begin_teardown().unwrap();
+    let waiter = session(11, 1);
+    lifecycle.wait_for_domain(waiter.clone()).unwrap();
+    lifecycle.stop_sandbox().unwrap();
+    lifecycle.stop_sidecar().unwrap();
+    lifecycle.unmount_state_anchor().unwrap();
+    lifecycle.release_prepared_lower_lease().unwrap();
+    lifecycle.graceful_supervisor_exit().unwrap();
+    lifecycle.finish_teardown().unwrap();
+    lifecycle
+        .restart_waiting_domain(generation(2), waiter.session_id)
+        .unwrap();
+
+    assert_eq!(lifecycle.phase(), DomainPhase::Starting);
+    let snapshot = store.snapshot().unwrap();
+    assert!(snapshot.sessions.iter().any(|row| {
+        row.generation_id == GenerationId::new(1)
+            && row.session_id == waiter.session_id
+            && row.session_phase == SessionPhase::Closed
+    }));
+    assert!(snapshot.sessions.iter().any(|row| {
+        row.generation_id == GenerationId::new(2)
+            && row.session_id == waiter.session_id
+            && row.session_phase == SessionPhase::Waiting
+    }));
+}
+
+#[test]
+fn serialized_concurrent_joins_register_distinct_sessions() {
+    let directory = tempfile::tempdir().unwrap();
+    let store = open_store(&directory);
+    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
+    let owner = session(10, 1);
+    lifecycle
+        .start_new_domain(generation(1), owner.clone())
+        .unwrap();
+    complete_startup(&mut lifecycle);
+    let lifecycle = Mutex::new(lifecycle);
+
+    std::thread::scope(|scope| {
+        for id in 11..=18 {
+            let lifecycle = &lifecycle;
+            scope.spawn(move || {
+                let candidate = session(id, 1);
+                let mut guard = lifecycle.lock().unwrap();
+                guard
+                    .start_ready_session(candidate.clone(), fingerprint(2))
+                    .unwrap();
+                guard.register_session(candidate.session_id).unwrap();
+            });
+        }
+    });
+
+    let lifecycle = lifecycle.into_inner().unwrap();
+    assert_eq!(lifecycle.registered_sessions(), 8);
+    assert_eq!(
+        store
+            .snapshot()
+            .unwrap()
+            .sessions
+            .iter()
+            .filter(|row| row.session_phase == SessionPhase::Attached)
+            .count(),
+        8
+    );
+}
+
+#[test]
+fn bootstrap_owner_disconnect_enters_failed_without_teardown_or_recovery() {
+    let directory = tempfile::tempdir().unwrap();
+    let store = open_store(&directory);
+    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
+    let owner = session(10, 1);
+    lifecycle
+        .start_new_domain(generation(1), owner.clone())
+        .unwrap();
+    lifecycle.disconnect_session(owner.session_id).unwrap();
+
+    assert_eq!(lifecycle.phase(), DomainPhase::Failed);
+    assert!(!lifecycle.is_published());
+    assert!(matches!(
+        lifecycle.begin_teardown(),
+        Err(LifecycleError::InvalidTransition {
+            action: LifecycleAction::BeginTeardown,
+            ..
+        })
+    ));
+}