lazaret/crates/lazaret-host-coordinator/tests/lifecycle.rs

use std::convert::Infallible;
use std::fs::OpenOptions;
use std::sync::Mutex;

use lazaret_core::identity::{
    ClientId, ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, HostPath,
    NetworkSidecarId, SandboxId, SessionAnchorId, SessionId,
};
use lazaret_core::records::{Domain, DurableDecision, Generation, Session, SessionPhase};
use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore;
use lazaret_host_coordinator::lifecycle::{
    DomainPhase, HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, LifecycleError,
};
use lazaret_linux::filesystem::try_exclusive_flock;

#[derive(Default)]
struct RecordingAdapter {
    actions: Vec<LifecycleAction>,
}

impl LifecycleAdapter for RecordingAdapter {
    type Error = Infallible;

    fn execute(
        &mut self,
        action: LifecycleAction,
        _generation: &Generation,
    ) -> Result<(), Self::Error> {
        self.actions.push(action);
        Ok(())
    }
}

fn bytes(value: u8) -> [u8; 32] {
    [value; 32]
}

fn domain() -> Domain {
    Domain {
        domain_id: DomainId::new(bytes(1)),
        profile_name: "trusted".into(),
        canonical_workspace: HostPath::new(b"/workspace/project".to_vec()),
    }
}

fn fingerprint(value: u8) -> ConfigurationFingerprint {
    ConfigurationFingerprint::new(bytes(value))
}

fn generation(id: u64) -> Generation {
    Generation {
        domain_id: domain().domain_id,
        generation_id: GenerationId::new(id),
        configuration_fingerprint: fingerprint(2),
        coordinator_id: CoordinatorId::new(bytes(u8::try_from(id * 3).unwrap())),
        network_sidecar_id: NetworkSidecarId::new(bytes(u8::try_from(id * 3 + 1).unwrap())),
        sandbox_id: SandboxId::new(bytes(u8::try_from(id * 3 + 2).unwrap())),
        durable_decision: DurableDecision::None,
    }
}

fn session(id: u8, generation_id: u64) -> Session {
    Session {
        generation_id: GenerationId::new(generation_id),
        session_id: SessionId::new(bytes(id)),
        cli_id: ClientId::new(bytes(id.wrapping_add(32))),
        session_anchor_id: SessionAnchorId::new(bytes(id.wrapping_add(64))),
        session_phase: SessionPhase::New,
    }
}

fn open_store(directory: &tempfile::TempDir) -> HostCoordinatorDomainStore {
    let lock_file = OpenOptions::new()
        .read(true)
        .write(true)
        .create(true)
        .truncate(false)
        .open(directory.path().join("domain.lock"))
        .unwrap();
    let lock = try_exclusive_flock(lock_file).unwrap();
    HostCoordinatorDomainStore::open(lock, &directory.path().join("domain.redb"), &domain())
        .unwrap()
}

fn complete_startup(lifecycle: &mut HostCoordinatorLifecycle<'_, RecordingAdapter>) {
    lifecycle.create_trust_anchor().unwrap();
    lifecycle.start_runtime_supervisor().unwrap();
    lifecycle.lease_prepared_lower().unwrap();
    lifecycle.mount_state_anchor().unwrap();
    lifecycle.start_sidecar().unwrap();
    lifecycle.start_connectivity_helper().unwrap();
    lifecycle.mark_network_ready().unwrap();
    lifecycle.arm_runtime_causal_edge().unwrap();
    lifecycle.start_sandbox().unwrap();
    lifecycle.publish_domain().unwrap();
}

#[test]
fn exact_start_join_disconnect_and_teardown_actions_are_ordered() {
    let directory = tempfile::tempdir().unwrap();
    let store = open_store(&directory);
    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
    let owner = session(10, 1);
    let waiter = session(11, 1);

    lifecycle
        .start_new_domain(generation(1), owner.clone())
        .unwrap();
    lifecycle.wait_for_domain(waiter.clone()).unwrap();
    complete_startup(&mut lifecycle);
    lifecycle
        .advance_waiting_session(owner.session_id, fingerprint(2))
        .unwrap();
    lifecycle.register_session(owner.session_id).unwrap();
    lifecycle
        .advance_waiting_session(waiter.session_id, fingerprint(2))
        .unwrap();
    lifecycle.register_session(waiter.session_id).unwrap();
    lifecycle.disconnect_session(owner.session_id).unwrap();
    assert!(!lifecycle.is_teardown_ready());
    lifecycle.disconnect_session(waiter.session_id).unwrap();
    assert!(lifecycle.is_teardown_ready());
    lifecycle.begin_teardown().unwrap();
    lifecycle.stop_sandbox().unwrap();
    lifecycle.stop_sidecar().unwrap();
    lifecycle.unmount_state_anchor().unwrap();
    lifecycle.release_prepared_lower_lease().unwrap();
    lifecycle.graceful_supervisor_exit().unwrap();
    lifecycle.finish_teardown().unwrap();

    assert_eq!(lifecycle.phase(), DomainPhase::Absent);
    assert!(!lifecycle.is_published());
    assert_eq!(
        lifecycle.into_adapter().actions,
        vec![
            LifecycleAction::StartNewDomain,
            LifecycleAction::WaitForDomain,
            LifecycleAction::CreateTrustAnchor,
            LifecycleAction::StartRuntimeSupervisor,
            LifecycleAction::LeasePreparedLower,
            LifecycleAction::MountStateAnchor,
            LifecycleAction::StartSidecar,
            LifecycleAction::StartConnectivityHelper,
            LifecycleAction::MarkNetworkReady,
            LifecycleAction::ArmRuntimeCausalEdge,
            LifecycleAction::StartSandbox,
            LifecycleAction::PublishDomain,
            LifecycleAction::AdvanceWaitingSession,
            LifecycleAction::RegisterSession,
            LifecycleAction::AdvanceWaitingSession,
            LifecycleAction::RegisterSession,
            LifecycleAction::DisconnectSession,
            LifecycleAction::DisconnectSession,
            LifecycleAction::BeginTeardown,
            LifecycleAction::StopSandbox,
            LifecycleAction::StopSidecar,
            LifecycleAction::UnmountStateAnchor,
            LifecycleAction::ReleasePreparedLowerLease,
            LifecycleAction::GracefulSupervisorExit,
            LifecycleAction::FinishTeardown,
        ]
    );
    let snapshot = store.snapshot().unwrap();
    assert_eq!(
        snapshot.generations[0].durable_decision,
        DurableDecision::Clean
    );
    assert!(
        snapshot
            .sessions
            .iter()
            .all(|row| row.session_phase == SessionPhase::Closed)
    );
}

#[test]
fn publication_and_invalid_transitions_fail_before_adapter_execution() {
    let directory = tempfile::tempdir().unwrap();
    let store = open_store(&directory);
    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
    lifecycle
        .start_new_domain(generation(1), session(10, 1))
        .unwrap();

    assert!(matches!(
        lifecycle.publish_domain(),
        Err(LifecycleError::InvalidTransition {
            action: LifecycleAction::PublishDomain,
            ..
        })
    ));
    assert!(matches!(
        lifecycle.start_sandbox(),
        Err(LifecycleError::InvalidTransition {
            action: LifecycleAction::StartSandbox,
            ..
        })
    ));
    assert!(matches!(
        lifecycle.mount_state_anchor(),
        Err(LifecycleError::InvalidTransition {
            action: LifecycleAction::MountStateAnchor,
            ..
        })
    ));
    assert_eq!(
        lifecycle.into_adapter().actions,
        vec![LifecycleAction::StartNewDomain]
    );
    assert_eq!(
        store.snapshot().unwrap().generations[0].durable_decision,
        DurableDecision::None
    );
}

#[test]
fn configuration_mismatch_is_refused_without_registration() {
    let directory = tempfile::tempdir().unwrap();
    let store = open_store(&directory);
    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
    let owner = session(10, 1);
    lifecycle
        .start_new_domain(generation(1), owner.clone())
        .unwrap();
    complete_startup(&mut lifecycle);
    let mismatched = session(11, 1);

    assert!(matches!(
        lifecycle.start_ready_session(mismatched.clone(), fingerprint(9)),
        Err(LifecycleError::InvalidTransition {
            action: LifecycleAction::StartReadySession,
            ..
        })
    ));
    lifecycle
        .reject_new_configuration_mismatch(mismatched.clone(), fingerprint(9))
        .unwrap();
    assert_eq!(
        lifecycle.session_phase(mismatched.session_id),
        Some(SessionPhase::Failed)
    );
    assert_eq!(lifecycle.registered_sessions(), 0);
}

#[test]
fn waiting_session_restarts_after_normal_teardown() {
    let directory = tempfile::tempdir().unwrap();
    let store = open_store(&directory);
    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
    let owner = session(10, 1);
    lifecycle
        .start_new_domain(generation(1), owner.clone())
        .unwrap();
    complete_startup(&mut lifecycle);
    lifecycle
        .advance_waiting_session(owner.session_id, fingerprint(2))
        .unwrap();
    lifecycle.register_session(owner.session_id).unwrap();
    lifecycle.disconnect_session(owner.session_id).unwrap();
    lifecycle.begin_teardown().unwrap();
    let waiter = session(11, 1);
    lifecycle.wait_for_domain(waiter.clone()).unwrap();
    lifecycle.stop_sandbox().unwrap();
    lifecycle.stop_sidecar().unwrap();
    lifecycle.unmount_state_anchor().unwrap();
    lifecycle.release_prepared_lower_lease().unwrap();
    lifecycle.graceful_supervisor_exit().unwrap();
    lifecycle.finish_teardown().unwrap();
    lifecycle
        .restart_waiting_domain(generation(2), waiter.session_id)
        .unwrap();

    assert_eq!(lifecycle.phase(), DomainPhase::Starting);
    let snapshot = store.snapshot().unwrap();
    assert!(snapshot.sessions.iter().any(|row| {
        row.generation_id == GenerationId::new(1)
            && row.session_id == waiter.session_id
            && row.session_phase == SessionPhase::Closed
    }));
    assert!(snapshot.sessions.iter().any(|row| {
        row.generation_id == GenerationId::new(2)
            && row.session_id == waiter.session_id
            && row.session_phase == SessionPhase::Waiting
    }));
}

#[test]
fn serialized_concurrent_joins_register_distinct_sessions() {
    let directory = tempfile::tempdir().unwrap();
    let store = open_store(&directory);
    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
    let owner = session(10, 1);
    lifecycle
        .start_new_domain(generation(1), owner.clone())
        .unwrap();
    complete_startup(&mut lifecycle);
    let lifecycle = Mutex::new(lifecycle);

    std::thread::scope(|scope| {
        for id in 11..=18 {
            let lifecycle = &lifecycle;
            scope.spawn(move || {
                let candidate = session(id, 1);
                let mut guard = lifecycle.lock().unwrap();
                guard
                    .start_ready_session(candidate.clone(), fingerprint(2))
                    .unwrap();
                guard.register_session(candidate.session_id).unwrap();
            });
        }
    });

    let lifecycle = lifecycle.into_inner().unwrap();
    assert_eq!(lifecycle.registered_sessions(), 8);
    assert_eq!(
        store
            .snapshot()
            .unwrap()
            .sessions
            .iter()
            .filter(|row| row.session_phase == SessionPhase::Attached)
            .count(),
        8
    );
}

#[test]
fn bootstrap_owner_disconnect_enters_failed_without_teardown_or_recovery() {
    let directory = tempfile::tempdir().unwrap();
    let store = open_store(&directory);
    let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
    let owner = session(10, 1);
    lifecycle
        .start_new_domain(generation(1), owner.clone())
        .unwrap();
    lifecycle.disconnect_session(owner.session_id).unwrap();

    assert_eq!(lifecycle.phase(), DomainPhase::Failed);
    assert!(!lifecycle.is_published());
    assert!(matches!(
        lifecycle.begin_teardown(),
        Err(LifecycleError::InvalidTransition {
            action: LifecycleAction::BeginTeardown,
            ..
        })
    ));
}