use std::convert::Infallible; use std::fs::OpenOptions; use std::sync::Mutex; use lazaret_core::identity::{ ClientId, ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, HostPath, NetworkSidecarId, SandboxId, SessionAnchorId, SessionId, }; use lazaret_core::records::{Domain, DurableDecision, Generation, Session, SessionPhase}; use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore; use lazaret_host_coordinator::lifecycle::{ DomainPhase, HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, LifecycleError, }; use lazaret_linux::filesystem::try_exclusive_flock; #[derive(Default)] struct RecordingAdapter { actions: Vec, } impl LifecycleAdapter for RecordingAdapter { type Error = Infallible; fn execute( &mut self, action: LifecycleAction, _generation: &Generation, ) -> Result<(), Self::Error> { self.actions.push(action); Ok(()) } } fn bytes(value: u8) -> [u8; 32] { [value; 32] } fn domain() -> Domain { Domain { domain_id: DomainId::new(bytes(1)), profile_name: "trusted".into(), canonical_workspace: HostPath::new(b"/workspace/project".to_vec()), } } fn fingerprint(value: u8) -> ConfigurationFingerprint { ConfigurationFingerprint::new(bytes(value)) } fn generation(id: u64) -> Generation { Generation { domain_id: domain().domain_id, generation_id: GenerationId::new(id), configuration_fingerprint: fingerprint(2), coordinator_id: CoordinatorId::new(bytes(u8::try_from(id * 3).unwrap())), network_sidecar_id: NetworkSidecarId::new(bytes(u8::try_from(id * 3 + 1).unwrap())), sandbox_id: SandboxId::new(bytes(u8::try_from(id * 3 + 2).unwrap())), durable_decision: DurableDecision::None, } } fn session(id: u8, generation_id: u64) -> Session { Session { generation_id: GenerationId::new(generation_id), session_id: SessionId::new(bytes(id)), cli_id: ClientId::new(bytes(id.wrapping_add(32))), session_anchor_id: SessionAnchorId::new(bytes(id.wrapping_add(64))), session_phase: SessionPhase::New, } } fn open_store(directory: &tempfile::TempDir) -> HostCoordinatorDomainStore { let lock_file = OpenOptions::new() .read(true) .write(true) .create(true) .truncate(false) .open(directory.path().join("domain.lock")) .unwrap(); let lock = try_exclusive_flock(lock_file).unwrap(); HostCoordinatorDomainStore::open(lock, &directory.path().join("domain.redb"), &domain()) .unwrap() } fn complete_startup(lifecycle: &mut HostCoordinatorLifecycle<'_, RecordingAdapter>) { lifecycle.create_trust_anchor().unwrap(); lifecycle.start_runtime_supervisor().unwrap(); lifecycle.lease_prepared_lower().unwrap(); lifecycle.mount_state_anchor().unwrap(); lifecycle.start_sidecar().unwrap(); lifecycle.start_connectivity_helper().unwrap(); lifecycle.mark_network_ready().unwrap(); lifecycle.arm_runtime_causal_edge().unwrap(); lifecycle.start_sandbox().unwrap(); lifecycle.publish_domain().unwrap(); } #[test] fn exact_start_join_disconnect_and_teardown_actions_are_ordered() { let directory = tempfile::tempdir().unwrap(); let store = open_store(&directory); let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default()); let owner = session(10, 1); let waiter = session(11, 1); lifecycle .start_new_domain(generation(1), owner.clone()) .unwrap(); lifecycle.wait_for_domain(waiter.clone()).unwrap(); complete_startup(&mut lifecycle); lifecycle .advance_waiting_session(owner.session_id, fingerprint(2)) .unwrap(); lifecycle.register_session(owner.session_id).unwrap(); lifecycle .advance_waiting_session(waiter.session_id, fingerprint(2)) .unwrap(); lifecycle.register_session(waiter.session_id).unwrap(); lifecycle.disconnect_session(owner.session_id).unwrap(); assert!(!lifecycle.is_teardown_ready()); lifecycle.disconnect_session(waiter.session_id).unwrap(); assert!(lifecycle.is_teardown_ready()); lifecycle.begin_teardown().unwrap(); lifecycle.stop_sandbox().unwrap(); lifecycle.stop_sidecar().unwrap(); lifecycle.unmount_state_anchor().unwrap(); lifecycle.release_prepared_lower_lease().unwrap(); lifecycle.graceful_supervisor_exit().unwrap(); lifecycle.finish_teardown().unwrap(); assert_eq!(lifecycle.phase(), DomainPhase::Absent); assert!(!lifecycle.is_published()); assert_eq!( lifecycle.into_adapter().actions, vec![ LifecycleAction::StartNewDomain, LifecycleAction::WaitForDomain, LifecycleAction::CreateTrustAnchor, LifecycleAction::StartRuntimeSupervisor, LifecycleAction::LeasePreparedLower, LifecycleAction::MountStateAnchor, LifecycleAction::StartSidecar, LifecycleAction::StartConnectivityHelper, LifecycleAction::MarkNetworkReady, LifecycleAction::ArmRuntimeCausalEdge, LifecycleAction::StartSandbox, LifecycleAction::PublishDomain, LifecycleAction::AdvanceWaitingSession, LifecycleAction::RegisterSession, LifecycleAction::AdvanceWaitingSession, LifecycleAction::RegisterSession, LifecycleAction::DisconnectSession, LifecycleAction::DisconnectSession, LifecycleAction::BeginTeardown, LifecycleAction::StopSandbox, LifecycleAction::StopSidecar, LifecycleAction::UnmountStateAnchor, LifecycleAction::ReleasePreparedLowerLease, LifecycleAction::GracefulSupervisorExit, LifecycleAction::FinishTeardown, ] ); let snapshot = store.snapshot().unwrap(); assert_eq!( snapshot.generations[0].durable_decision, DurableDecision::Clean ); assert!( snapshot .sessions .iter() .all(|row| row.session_phase == SessionPhase::Closed) ); } #[test] fn publication_and_invalid_transitions_fail_before_adapter_execution() { let directory = tempfile::tempdir().unwrap(); let store = open_store(&directory); let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default()); lifecycle .start_new_domain(generation(1), session(10, 1)) .unwrap(); assert!(matches!( lifecycle.publish_domain(), Err(LifecycleError::InvalidTransition { action: LifecycleAction::PublishDomain, .. }) )); assert!(matches!( lifecycle.start_sandbox(), Err(LifecycleError::InvalidTransition { action: LifecycleAction::StartSandbox, .. }) )); assert!(matches!( lifecycle.mount_state_anchor(), Err(LifecycleError::InvalidTransition { action: LifecycleAction::MountStateAnchor, .. }) )); assert_eq!( lifecycle.into_adapter().actions, vec![LifecycleAction::StartNewDomain] ); assert_eq!( store.snapshot().unwrap().generations[0].durable_decision, DurableDecision::None ); } #[test] fn configuration_mismatch_is_refused_without_registration() { let directory = tempfile::tempdir().unwrap(); let store = open_store(&directory); let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default()); let owner = session(10, 1); lifecycle .start_new_domain(generation(1), owner.clone()) .unwrap(); complete_startup(&mut lifecycle); let mismatched = session(11, 1); assert!(matches!( lifecycle.start_ready_session(mismatched.clone(), fingerprint(9)), Err(LifecycleError::InvalidTransition { action: LifecycleAction::StartReadySession, .. }) )); lifecycle .reject_new_configuration_mismatch(mismatched.clone(), fingerprint(9)) .unwrap(); assert_eq!( lifecycle.session_phase(mismatched.session_id), Some(SessionPhase::Failed) ); assert_eq!(lifecycle.registered_sessions(), 0); } #[test] fn waiting_session_restarts_after_normal_teardown() { let directory = tempfile::tempdir().unwrap(); let store = open_store(&directory); let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default()); let owner = session(10, 1); lifecycle .start_new_domain(generation(1), owner.clone()) .unwrap(); complete_startup(&mut lifecycle); lifecycle .advance_waiting_session(owner.session_id, fingerprint(2)) .unwrap(); lifecycle.register_session(owner.session_id).unwrap(); lifecycle.disconnect_session(owner.session_id).unwrap(); lifecycle.begin_teardown().unwrap(); let waiter = session(11, 1); lifecycle.wait_for_domain(waiter.clone()).unwrap(); lifecycle.stop_sandbox().unwrap(); lifecycle.stop_sidecar().unwrap(); lifecycle.unmount_state_anchor().unwrap(); lifecycle.release_prepared_lower_lease().unwrap(); lifecycle.graceful_supervisor_exit().unwrap(); lifecycle.finish_teardown().unwrap(); lifecycle .restart_waiting_domain(generation(2), waiter.session_id) .unwrap(); assert_eq!(lifecycle.phase(), DomainPhase::Starting); let snapshot = store.snapshot().unwrap(); assert!(snapshot.sessions.iter().any(|row| { row.generation_id == GenerationId::new(1) && row.session_id == waiter.session_id && row.session_phase == SessionPhase::Closed })); assert!(snapshot.sessions.iter().any(|row| { row.generation_id == GenerationId::new(2) && row.session_id == waiter.session_id && row.session_phase == SessionPhase::Waiting })); } #[test] fn serialized_concurrent_joins_register_distinct_sessions() { let directory = tempfile::tempdir().unwrap(); let store = open_store(&directory); let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default()); let owner = session(10, 1); lifecycle .start_new_domain(generation(1), owner.clone()) .unwrap(); complete_startup(&mut lifecycle); let lifecycle = Mutex::new(lifecycle); std::thread::scope(|scope| { for id in 11..=18 { let lifecycle = &lifecycle; scope.spawn(move || { let candidate = session(id, 1); let mut guard = lifecycle.lock().unwrap(); guard .start_ready_session(candidate.clone(), fingerprint(2)) .unwrap(); guard.register_session(candidate.session_id).unwrap(); }); } }); let lifecycle = lifecycle.into_inner().unwrap(); assert_eq!(lifecycle.registered_sessions(), 8); assert_eq!( store .snapshot() .unwrap() .sessions .iter() .filter(|row| row.session_phase == SessionPhase::Attached) .count(), 8 ); } #[test] fn bootstrap_owner_disconnect_enters_failed_without_teardown_or_recovery() { let directory = tempfile::tempdir().unwrap(); let store = open_store(&directory); let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default()); let owner = session(10, 1); lifecycle .start_new_domain(generation(1), owner.clone()) .unwrap(); lifecycle.disconnect_session(owner.session_id).unwrap(); assert_eq!(lifecycle.phase(), DomainPhase::Failed); assert!(!lifecycle.is_published()); assert!(matches!( lifecycle.begin_teardown(), Err(LifecycleError::InvalidTransition { action: LifecycleAction::BeginTeardown, .. }) )); }