Orchestrate normal domain lifecycle through adapters
This commit is contained in:
parent
49d5def1a9
commit
671473106c
3 changed files with 1266 additions and 0 deletions
|
|
@ -3,6 +3,7 @@
|
|||
pub mod control;
|
||||
pub mod domain_store;
|
||||
pub mod escalation;
|
||||
pub mod lifecycle;
|
||||
pub mod network;
|
||||
pub mod runtime;
|
||||
pub mod state_anchor;
|
||||
|
|
|
|||
918
crates/lazaret-host-coordinator/src/lifecycle.rs
Normal file
918
crates/lazaret-host-coordinator/src/lifecycle.rs
Normal file
|
|
@ -0,0 +1,918 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt;
|
||||
|
||||
use lazaret_core::identity::{ConfigurationFingerprint, SessionId};
|
||||
use lazaret_core::records::{DurableDecision, Generation, Session, SessionPhase};
|
||||
|
||||
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum LifecycleAction {
|
||||
StartNewDomain,
|
||||
StartReadySession,
|
||||
WaitForDomain,
|
||||
RestartWaitingDomain,
|
||||
CreateTrustAnchor,
|
||||
StartRuntimeSupervisor,
|
||||
LeasePreparedLower,
|
||||
MountStateAnchor,
|
||||
StartSidecar,
|
||||
StartConnectivityHelper,
|
||||
MarkNetworkReady,
|
||||
ArmRuntimeCausalEdge,
|
||||
StartSandbox,
|
||||
PublishDomain,
|
||||
AdvanceWaitingSession,
|
||||
RejectConfigurationMismatch,
|
||||
RegisterSession,
|
||||
DisconnectSession,
|
||||
BeginTeardown,
|
||||
StopSandbox,
|
||||
StopSidecar,
|
||||
UnmountStateAnchor,
|
||||
ReleasePreparedLowerLease,
|
||||
GracefulSupervisorExit,
|
||||
FinishTeardown,
|
||||
}
|
||||
|
||||
pub trait LifecycleAdapter {
|
||||
type Error;
|
||||
|
||||
/// # Errors
|
||||
/// Returns the adapter's physical or durable action failure.
|
||||
fn execute(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
generation: &Generation,
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum DomainPhase {
|
||||
Absent,
|
||||
Starting,
|
||||
Ready,
|
||||
Stopping,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum LifecycleError<E> {
|
||||
Adapter(E),
|
||||
Store(DomainStoreError),
|
||||
InvalidTransition {
|
||||
action: LifecycleAction,
|
||||
reason: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl<E: fmt::Display> fmt::Display for LifecycleError<E> {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Adapter(error) => write!(formatter, "lifecycle adapter failed: {error}"),
|
||||
Self::Store(error) => {
|
||||
write!(formatter, "Domain Store lifecycle update failed: {error}")
|
||||
}
|
||||
Self::InvalidTransition { action, reason } => {
|
||||
write!(formatter, "{action:?} refused: {reason}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: std::error::Error + 'static> std::error::Error for LifecycleError<E> {}
|
||||
|
||||
impl<E> From<DomainStoreError> for LifecycleError<E> {
|
||||
fn from(error: DomainStoreError) -> Self {
|
||||
Self::Store(error)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
enum RuntimeMilestone {
|
||||
TrustAnchor,
|
||||
RuntimeSupervisor,
|
||||
PreparedLowerLease,
|
||||
StateAnchor,
|
||||
Sidecar,
|
||||
ConnectivityHelper,
|
||||
NetworkReady,
|
||||
CausalEdge,
|
||||
Sandbox,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
struct RuntimeProgress {
|
||||
completed: HashSet<RuntimeMilestone>,
|
||||
}
|
||||
|
||||
impl RuntimeProgress {
|
||||
fn has(&self, milestone: RuntimeMilestone) -> bool {
|
||||
self.completed.contains(&milestone)
|
||||
}
|
||||
|
||||
fn mark(&mut self, milestone: RuntimeMilestone) {
|
||||
self.completed.insert(milestone);
|
||||
}
|
||||
|
||||
fn clear(&mut self, milestone: RuntimeMilestone) {
|
||||
self.completed.remove(&milestone);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HostCoordinatorLifecycle<'store, A> {
|
||||
store: &'store HostCoordinatorDomainStore,
|
||||
adapter: A,
|
||||
phase: DomainPhase,
|
||||
generation: Option<Generation>,
|
||||
bootstrap_owner: Option<SessionId>,
|
||||
progress: RuntimeProgress,
|
||||
published: bool,
|
||||
sessions: HashMap<SessionId, Session>,
|
||||
connected: HashSet<SessionId>,
|
||||
registered: HashSet<SessionId>,
|
||||
}
|
||||
|
||||
impl<'store, A: LifecycleAdapter> HostCoordinatorLifecycle<'store, A> {
|
||||
#[must_use]
|
||||
pub fn new(store: &'store HostCoordinatorDomainStore, adapter: A) -> Self {
|
||||
Self {
|
||||
store,
|
||||
adapter,
|
||||
phase: DomainPhase::Absent,
|
||||
generation: None,
|
||||
bootstrap_owner: None,
|
||||
progress: RuntimeProgress::default(),
|
||||
published: false,
|
||||
sessions: HashMap::new(),
|
||||
connected: HashSet::new(),
|
||||
registered: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn phase(&self) -> DomainPhase {
|
||||
self.phase
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_published(&self) -> bool {
|
||||
self.published
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn session_phase(&self, session_id: SessionId) -> Option<SessionPhase> {
|
||||
self.sessions
|
||||
.get(&session_id)
|
||||
.map(|session| session.session_phase)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn registered_sessions(&self) -> usize {
|
||||
self.registered.len()
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a non-absent Domain, invalid initial records, or adapter/store failure.
|
||||
pub fn start_new_domain(
|
||||
&mut self,
|
||||
generation: Generation,
|
||||
mut session: Session,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
self.phase == DomainPhase::Absent,
|
||||
"Domain is not absent",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
generation.durable_decision == DurableDecision::None,
|
||||
"new generation decision is not None",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
session.generation_id == generation.generation_id,
|
||||
"session generation differs",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
session.session_phase == SessionPhase::New,
|
||||
"session is not New",
|
||||
)?;
|
||||
self.require_new_generation(LifecycleAction::StartNewDomain, &generation, &session)?;
|
||||
self.execute(LifecycleAction::StartNewDomain, &generation)?;
|
||||
session.session_phase = SessionPhase::Waiting;
|
||||
self.store.apply(&[
|
||||
DomainStoreMutation::PutGeneration(generation.clone()),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.phase = DomainPhase::Starting;
|
||||
self.bootstrap_owner = Some(session.session_id);
|
||||
self.sessions.insert(session.session_id, session);
|
||||
self.generation = Some(generation);
|
||||
self.progress = RuntimeProgress::default();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an unpublished, stale, mismatched, or duplicate session.
|
||||
pub fn start_ready_session(
|
||||
&mut self,
|
||||
session: Session,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.require_ready_join(LifecycleAction::StartReadySession, &session, fingerprint)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartReadySession,
|
||||
session.session_phase == SessionPhase::New
|
||||
&& !self.sessions.contains_key(&session.session_id),
|
||||
"session is not a distinct New session",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::StartReadySession)?;
|
||||
self.put_new_session(
|
||||
LifecycleAction::StartReadySession,
|
||||
session,
|
||||
SessionPhase::Registering,
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a session unless the Domain is starting or stopping.
|
||||
pub fn wait_for_domain(&mut self, session: Session) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::WaitForDomain,
|
||||
matches!(self.phase, DomainPhase::Starting | DomainPhase::Stopping),
|
||||
"Domain is neither starting nor stopping",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::WaitForDomain,
|
||||
session.session_phase == SessionPhase::New,
|
||||
"session is not New",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::WaitForDomain,
|
||||
session.generation_id
|
||||
== self
|
||||
.current_generation(LifecycleAction::WaitForDomain)?
|
||||
.generation_id
|
||||
&& !self.sessions.contains_key(&session.session_id),
|
||||
"session generation differs or Session ID is already known",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::WaitForDomain)?;
|
||||
self.put_new_session(
|
||||
LifecycleAction::WaitForDomain,
|
||||
session,
|
||||
SessionPhase::Waiting,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a restart unless the Domain is absent and the session is waiting.
|
||||
pub fn restart_waiting_domain(
|
||||
&mut self,
|
||||
generation: Generation,
|
||||
session_id: SessionId,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
self.phase == DomainPhase::Absent,
|
||||
"Domain is not absent",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
generation.durable_decision == DurableDecision::None,
|
||||
"new generation decision is not None",
|
||||
)?;
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::RestartWaitingDomain)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
session.session_phase == SessionPhase::Waiting,
|
||||
"session is not Waiting",
|
||||
)?;
|
||||
let successor_session = Session {
|
||||
generation_id: generation.generation_id,
|
||||
..session.clone()
|
||||
};
|
||||
self.require_new_generation(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
&generation,
|
||||
&successor_session,
|
||||
)?;
|
||||
self.execute(LifecycleAction::RestartWaitingDomain, &generation)?;
|
||||
let previous = Session {
|
||||
session_phase: SessionPhase::Closed,
|
||||
..session.clone()
|
||||
};
|
||||
session.generation_id = generation.generation_id;
|
||||
self.store.apply(&[
|
||||
DomainStoreMutation::PutSession(previous),
|
||||
DomainStoreMutation::PutGeneration(generation.clone()),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.phase = DomainPhase::Starting;
|
||||
self.bootstrap_owner = Some(session_id);
|
||||
self.generation = Some(generation);
|
||||
self.progress = RuntimeProgress::default();
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn create_trust_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::CreateTrustAnchor,
|
||||
!self.progress.has(RuntimeMilestone::TrustAnchor),
|
||||
|p| p.mark(RuntimeMilestone::TrustAnchor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn start_runtime_supervisor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartRuntimeSupervisor,
|
||||
self.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
|p| p.mark(RuntimeMilestone::RuntimeSupervisor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn lease_prepared_lower(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::LeasePreparedLower,
|
||||
self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||||
|p| p.mark(RuntimeMilestone::PreparedLowerLease),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn mount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::MountStateAnchor,
|
||||
self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.progress.has(RuntimeMilestone::StateAnchor),
|
||||
|p| p.mark(RuntimeMilestone::StateAnchor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn start_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartSidecar,
|
||||
self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::Sidecar),
|
||||
|p| p.mark(RuntimeMilestone::Sidecar),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn start_connectivity_helper(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartConnectivityHelper,
|
||||
self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self.progress.has(RuntimeMilestone::ConnectivityHelper),
|
||||
|p| p.mark(RuntimeMilestone::ConnectivityHelper),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn mark_network_ready(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::MarkNetworkReady,
|
||||
self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& !self.progress.has(RuntimeMilestone::NetworkReady),
|
||||
|p| p.mark(RuntimeMilestone::NetworkReady),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn arm_runtime_causal_edge(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::ArmRuntimeCausalEdge,
|
||||
self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.progress.has(RuntimeMilestone::CausalEdge),
|
||||
|p| p.mark(RuntimeMilestone::CausalEdge),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses launch before network readiness and the causal edge are established.
|
||||
pub fn start_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::StartSandbox,
|
||||
self.phase == DomainPhase::Starting && allowed,
|
||||
"runtime prerequisites are incomplete",
|
||||
)?;
|
||||
let mut generation = self
|
||||
.current_generation(LifecycleAction::StartSandbox)?
|
||||
.clone();
|
||||
generation.durable_decision = DurableDecision::Preparing;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation.clone());
|
||||
self.execute(LifecycleAction::StartSandbox, &generation)?;
|
||||
self.progress.mark(RuntimeMilestone::Sandbox);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses publication until every ordered startup prerequisite is complete.
|
||||
pub fn publish_domain(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.phase == DomainPhase::Starting
|
||||
&& self.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& self.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::PublishDomain,
|
||||
allowed,
|
||||
"publication prerequisites are incomplete",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::PublishDomain)?;
|
||||
let mut generation = self
|
||||
.current_generation(LifecycleAction::PublishDomain)?
|
||||
.clone();
|
||||
generation.durable_decision = DurableDecision::Published;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation);
|
||||
self.phase = DomainPhase::Ready;
|
||||
self.bootstrap_owner = None;
|
||||
self.published = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a non-waiting session, configuration mismatch, or unavailable Domain.
|
||||
pub fn advance_waiting_session(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.require_fingerprint(LifecycleAction::AdvanceWaitingSession, fingerprint)?;
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::AdvanceWaitingSession)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::AdvanceWaitingSession,
|
||||
session.session_phase == SessionPhase::Waiting && self.admission_ready(),
|
||||
"waiting session is not admissible",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::AdvanceWaitingSession)?;
|
||||
session.session_phase = SessionPhase::Registering;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.connected.insert(session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses unless a new or waiting session differs from the ready generation.
|
||||
pub fn reject_new_configuration_mismatch(
|
||||
&mut self,
|
||||
mut session: Session,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
self.phase == DomainPhase::Ready && session.session_phase == SessionPhase::New,
|
||||
"session is not eligible for mismatch refusal",
|
||||
)?;
|
||||
let generation = self.current_generation(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
session.generation_id == generation.generation_id
|
||||
&& fingerprint != generation.configuration_fingerprint,
|
||||
"configuration does not differ from the current generation",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
!self.sessions.contains_key(&session.session_id),
|
||||
"Session ID is already known",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
let new = session.clone();
|
||||
session.session_phase = SessionPhase::Failed;
|
||||
self.store.apply(&[
|
||||
DomainStoreMutation::PutSession(new),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.sessions.insert(session.session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses unless a new or waiting session differs from the ready generation.
|
||||
pub fn reject_configuration_mismatch(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::RejectConfigurationMismatch)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
self.phase == DomainPhase::Ready
|
||||
&& matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::New | SessionPhase::Waiting
|
||||
),
|
||||
"session is not eligible for mismatch refusal",
|
||||
)?;
|
||||
let generation = self.current_generation(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
fingerprint != generation.configuration_fingerprint,
|
||||
"configuration matches",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
session.session_phase = SessionPhase::Failed;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.connected.remove(&session_id);
|
||||
self.registered.remove(&session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a disconnected, stale, or non-registering session.
|
||||
pub fn register_session(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::RegisterSession)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::RegisterSession,
|
||||
self.admission_ready()
|
||||
&& session.session_phase == SessionPhase::Registering
|
||||
&& self.connected.contains(&session_id),
|
||||
"session is not admissible",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::RegisterSession)?;
|
||||
session.session_phase = SessionPhase::Attached;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.registered.insert(session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an absent or terminal session, or an adapter/store failure.
|
||||
pub fn disconnect_session(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::DisconnectSession)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::DisconnectSession,
|
||||
matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::Waiting
|
||||
| SessionPhase::Registering
|
||||
| SessionPhase::Attached
|
||||
| SessionPhase::Prompting
|
||||
| SessionPhase::HostExecuting
|
||||
),
|
||||
"session is not live",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::DisconnectSession)?;
|
||||
let leader_lost =
|
||||
self.phase == DomainPhase::Starting && self.bootstrap_owner == Some(session_id);
|
||||
session.session_phase = SessionPhase::Closed;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.connected.remove(&session_id);
|
||||
self.registered.remove(&session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
if leader_lost {
|
||||
self.phase = DomainPhase::Failed;
|
||||
self.bootstrap_owner = None;
|
||||
self.published = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_teardown_ready(&self) -> bool {
|
||||
self.phase == DomainPhase::Ready
|
||||
&& self.registered.is_empty()
|
||||
&& self.connected.is_empty()
|
||||
&& self.sessions.values().all(|session| {
|
||||
!matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::Waiting
|
||||
| SessionPhase::Registering
|
||||
| SessionPhase::Attached
|
||||
| SessionPhase::Prompting
|
||||
| SessionPhase::HostExecuting
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses while any session retains Domain interest.
|
||||
pub fn begin_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::BeginTeardown,
|
||||
self.is_teardown_ready(),
|
||||
"sessions still have interest",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::BeginTeardown)?;
|
||||
self.phase = DomainPhase::Stopping;
|
||||
self.published = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn stop_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::StopSandbox,
|
||||
self.progress.has(RuntimeMilestone::Sandbox),
|
||||
|p| {
|
||||
p.clear(RuntimeMilestone::Sandbox);
|
||||
p.clear(RuntimeMilestone::CausalEdge);
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn stop_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::StopSidecar,
|
||||
!self.progress.has(RuntimeMilestone::Sandbox)
|
||||
&& self.progress.has(RuntimeMilestone::Sidecar),
|
||||
|p| {
|
||||
p.clear(RuntimeMilestone::Sidecar);
|
||||
p.clear(RuntimeMilestone::ConnectivityHelper);
|
||||
p.clear(RuntimeMilestone::NetworkReady);
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn unmount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::UnmountStateAnchor,
|
||||
!self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self.progress.has(RuntimeMilestone::StateAnchor),
|
||||
|p| p.clear(RuntimeMilestone::StateAnchor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn release_prepared_lower_lease(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::ReleasePreparedLowerLease,
|
||||
!self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||||
|p| p.clear(RuntimeMilestone::PreparedLowerLease),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn graceful_supervisor_exit(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::GracefulSupervisorExit,
|
||||
!self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
|p| p.clear(RuntimeMilestone::RuntimeSupervisor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses while runtime resources remain or the final durable update fails.
|
||||
pub fn finish_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.phase == DomainPhase::Stopping
|
||||
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::FinishTeardown,
|
||||
allowed,
|
||||
"runtime resources remain",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::FinishTeardown)?;
|
||||
let mut generation = self
|
||||
.current_generation(LifecycleAction::FinishTeardown)?
|
||||
.clone();
|
||||
generation.durable_decision = DurableDecision::Clean;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation);
|
||||
self.phase = DomainPhase::Absent;
|
||||
self.progress = RuntimeProgress::default();
|
||||
self.bootstrap_owner = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn into_adapter(self) -> A {
|
||||
self.adapter
|
||||
}
|
||||
|
||||
fn startup_action(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
allowed: bool,
|
||||
update: impl FnOnce(&mut RuntimeProgress),
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.phase == DomainPhase::Starting && allowed,
|
||||
"startup ordering refused",
|
||||
)?;
|
||||
self.execute_current(action)?;
|
||||
update(&mut self.progress);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn teardown_action(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
allowed: bool,
|
||||
update: impl FnOnce(&mut RuntimeProgress),
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.phase == DomainPhase::Stopping && allowed,
|
||||
"teardown ordering refused",
|
||||
)?;
|
||||
self.execute_current(action)?;
|
||||
update(&mut self.progress);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn put_new_session(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
mut session: Session,
|
||||
phase: SessionPhase,
|
||||
connected: bool,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
session.session_phase == SessionPhase::New,
|
||||
"session is not New",
|
||||
)?;
|
||||
Self::require(
|
||||
action,
|
||||
!self.sessions.contains_key(&session.session_id),
|
||||
"Session ID is already known",
|
||||
)?;
|
||||
session.session_phase = phase;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
if connected {
|
||||
self.connected.insert(session.session_id);
|
||||
}
|
||||
self.sessions.insert(session.session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_ready_join(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
session: &Session,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.admission_ready(),
|
||||
"Domain is not published and ready",
|
||||
)?;
|
||||
Self::require(
|
||||
action,
|
||||
session.generation_id == self.current_generation(action)?.generation_id,
|
||||
"session generation differs",
|
||||
)?;
|
||||
Self::require(
|
||||
action,
|
||||
fingerprint == self.current_generation(action)?.configuration_fingerprint,
|
||||
"configuration differs",
|
||||
)
|
||||
}
|
||||
|
||||
fn require_new_generation(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
generation: &Generation,
|
||||
session: &Session,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let snapshot = self.store.snapshot()?;
|
||||
let latest = snapshot
|
||||
.generations
|
||||
.iter()
|
||||
.map(|candidate| candidate.generation_id)
|
||||
.max();
|
||||
Self::require(
|
||||
action,
|
||||
generation.domain_id == snapshot.domain.domain_id
|
||||
&& session.generation_id == generation.generation_id
|
||||
&& latest.is_none_or(|current| generation.generation_id > current),
|
||||
"generation is not a fresh successor for this Domain",
|
||||
)
|
||||
}
|
||||
|
||||
fn require_fingerprint(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
fingerprint == self.current_generation(action)?.configuration_fingerprint,
|
||||
"configuration differs",
|
||||
)
|
||||
}
|
||||
|
||||
fn admission_ready(&self) -> bool {
|
||||
self.phase == DomainPhase::Ready && self.published
|
||||
}
|
||||
|
||||
fn current_generation(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
) -> Result<&Generation, LifecycleError<A::Error>> {
|
||||
self.generation
|
||||
.as_ref()
|
||||
.ok_or(LifecycleError::InvalidTransition {
|
||||
action,
|
||||
reason: "generation is absent",
|
||||
})
|
||||
}
|
||||
|
||||
fn session(
|
||||
&self,
|
||||
session_id: SessionId,
|
||||
action: LifecycleAction,
|
||||
) -> Result<&Session, LifecycleError<A::Error>> {
|
||||
self.sessions
|
||||
.get(&session_id)
|
||||
.ok_or(LifecycleError::InvalidTransition {
|
||||
action,
|
||||
reason: "session is absent",
|
||||
})
|
||||
}
|
||||
|
||||
fn execute_current(&mut self, action: LifecycleAction) -> Result<(), LifecycleError<A::Error>> {
|
||||
let generation = self.current_generation(action)?.clone();
|
||||
self.execute(action, &generation)
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
generation: &Generation,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.adapter
|
||||
.execute(action, generation)
|
||||
.map_err(LifecycleError::Adapter)
|
||||
}
|
||||
|
||||
fn require(
|
||||
action: LifecycleAction,
|
||||
condition: bool,
|
||||
reason: &'static str,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
if condition {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(LifecycleError::InvalidTransition { action, reason })
|
||||
}
|
||||
}
|
||||
}
|
||||
347
crates/lazaret-host-coordinator/tests/lifecycle.rs
Normal file
347
crates/lazaret-host-coordinator/tests/lifecycle.rs
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
use std::convert::Infallible;
|
||||
use std::fs::OpenOptions;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use lazaret_core::identity::{
|
||||
ClientId, ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, HostPath,
|
||||
NetworkSidecarId, SandboxId, SessionAnchorId, SessionId,
|
||||
};
|
||||
use lazaret_core::records::{Domain, DurableDecision, Generation, Session, SessionPhase};
|
||||
use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore;
|
||||
use lazaret_host_coordinator::lifecycle::{
|
||||
DomainPhase, HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, LifecycleError,
|
||||
};
|
||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||
|
||||
#[derive(Default)]
|
||||
struct RecordingAdapter {
|
||||
actions: Vec<LifecycleAction>,
|
||||
}
|
||||
|
||||
impl LifecycleAdapter for RecordingAdapter {
|
||||
type Error = Infallible;
|
||||
|
||||
fn execute(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
_generation: &Generation,
|
||||
) -> Result<(), Self::Error> {
|
||||
self.actions.push(action);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn bytes(value: u8) -> [u8; 32] {
|
||||
[value; 32]
|
||||
}
|
||||
|
||||
fn domain() -> Domain {
|
||||
Domain {
|
||||
domain_id: DomainId::new(bytes(1)),
|
||||
profile_name: "trusted".into(),
|
||||
canonical_workspace: HostPath::new(b"/workspace/project".to_vec()),
|
||||
}
|
||||
}
|
||||
|
||||
fn fingerprint(value: u8) -> ConfigurationFingerprint {
|
||||
ConfigurationFingerprint::new(bytes(value))
|
||||
}
|
||||
|
||||
fn generation(id: u64) -> Generation {
|
||||
Generation {
|
||||
domain_id: domain().domain_id,
|
||||
generation_id: GenerationId::new(id),
|
||||
configuration_fingerprint: fingerprint(2),
|
||||
coordinator_id: CoordinatorId::new(bytes(u8::try_from(id * 3).unwrap())),
|
||||
network_sidecar_id: NetworkSidecarId::new(bytes(u8::try_from(id * 3 + 1).unwrap())),
|
||||
sandbox_id: SandboxId::new(bytes(u8::try_from(id * 3 + 2).unwrap())),
|
||||
durable_decision: DurableDecision::None,
|
||||
}
|
||||
}
|
||||
|
||||
fn session(id: u8, generation_id: u64) -> Session {
|
||||
Session {
|
||||
generation_id: GenerationId::new(generation_id),
|
||||
session_id: SessionId::new(bytes(id)),
|
||||
cli_id: ClientId::new(bytes(id.wrapping_add(32))),
|
||||
session_anchor_id: SessionAnchorId::new(bytes(id.wrapping_add(64))),
|
||||
session_phase: SessionPhase::New,
|
||||
}
|
||||
}
|
||||
|
||||
fn open_store(directory: &tempfile::TempDir) -> HostCoordinatorDomainStore {
|
||||
let lock_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(directory.path().join("domain.lock"))
|
||||
.unwrap();
|
||||
let lock = try_exclusive_flock(lock_file).unwrap();
|
||||
HostCoordinatorDomainStore::open(lock, &directory.path().join("domain.redb"), &domain())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn complete_startup(lifecycle: &mut HostCoordinatorLifecycle<'_, RecordingAdapter>) {
|
||||
lifecycle.create_trust_anchor().unwrap();
|
||||
lifecycle.start_runtime_supervisor().unwrap();
|
||||
lifecycle.lease_prepared_lower().unwrap();
|
||||
lifecycle.mount_state_anchor().unwrap();
|
||||
lifecycle.start_sidecar().unwrap();
|
||||
lifecycle.start_connectivity_helper().unwrap();
|
||||
lifecycle.mark_network_ready().unwrap();
|
||||
lifecycle.arm_runtime_causal_edge().unwrap();
|
||||
lifecycle.start_sandbox().unwrap();
|
||||
lifecycle.publish_domain().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_start_join_disconnect_and_teardown_actions_are_ordered() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = open_store(&directory);
|
||||
let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
|
||||
let owner = session(10, 1);
|
||||
let waiter = session(11, 1);
|
||||
|
||||
lifecycle
|
||||
.start_new_domain(generation(1), owner.clone())
|
||||
.unwrap();
|
||||
lifecycle.wait_for_domain(waiter.clone()).unwrap();
|
||||
complete_startup(&mut lifecycle);
|
||||
lifecycle
|
||||
.advance_waiting_session(owner.session_id, fingerprint(2))
|
||||
.unwrap();
|
||||
lifecycle.register_session(owner.session_id).unwrap();
|
||||
lifecycle
|
||||
.advance_waiting_session(waiter.session_id, fingerprint(2))
|
||||
.unwrap();
|
||||
lifecycle.register_session(waiter.session_id).unwrap();
|
||||
lifecycle.disconnect_session(owner.session_id).unwrap();
|
||||
assert!(!lifecycle.is_teardown_ready());
|
||||
lifecycle.disconnect_session(waiter.session_id).unwrap();
|
||||
assert!(lifecycle.is_teardown_ready());
|
||||
lifecycle.begin_teardown().unwrap();
|
||||
lifecycle.stop_sandbox().unwrap();
|
||||
lifecycle.stop_sidecar().unwrap();
|
||||
lifecycle.unmount_state_anchor().unwrap();
|
||||
lifecycle.release_prepared_lower_lease().unwrap();
|
||||
lifecycle.graceful_supervisor_exit().unwrap();
|
||||
lifecycle.finish_teardown().unwrap();
|
||||
|
||||
assert_eq!(lifecycle.phase(), DomainPhase::Absent);
|
||||
assert!(!lifecycle.is_published());
|
||||
assert_eq!(
|
||||
lifecycle.into_adapter().actions,
|
||||
vec![
|
||||
LifecycleAction::StartNewDomain,
|
||||
LifecycleAction::WaitForDomain,
|
||||
LifecycleAction::CreateTrustAnchor,
|
||||
LifecycleAction::StartRuntimeSupervisor,
|
||||
LifecycleAction::LeasePreparedLower,
|
||||
LifecycleAction::MountStateAnchor,
|
||||
LifecycleAction::StartSidecar,
|
||||
LifecycleAction::StartConnectivityHelper,
|
||||
LifecycleAction::MarkNetworkReady,
|
||||
LifecycleAction::ArmRuntimeCausalEdge,
|
||||
LifecycleAction::StartSandbox,
|
||||
LifecycleAction::PublishDomain,
|
||||
LifecycleAction::AdvanceWaitingSession,
|
||||
LifecycleAction::RegisterSession,
|
||||
LifecycleAction::AdvanceWaitingSession,
|
||||
LifecycleAction::RegisterSession,
|
||||
LifecycleAction::DisconnectSession,
|
||||
LifecycleAction::DisconnectSession,
|
||||
LifecycleAction::BeginTeardown,
|
||||
LifecycleAction::StopSandbox,
|
||||
LifecycleAction::StopSidecar,
|
||||
LifecycleAction::UnmountStateAnchor,
|
||||
LifecycleAction::ReleasePreparedLowerLease,
|
||||
LifecycleAction::GracefulSupervisorExit,
|
||||
LifecycleAction::FinishTeardown,
|
||||
]
|
||||
);
|
||||
let snapshot = store.snapshot().unwrap();
|
||||
assert_eq!(
|
||||
snapshot.generations[0].durable_decision,
|
||||
DurableDecision::Clean
|
||||
);
|
||||
assert!(
|
||||
snapshot
|
||||
.sessions
|
||||
.iter()
|
||||
.all(|row| row.session_phase == SessionPhase::Closed)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publication_and_invalid_transitions_fail_before_adapter_execution() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = open_store(&directory);
|
||||
let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
|
||||
lifecycle
|
||||
.start_new_domain(generation(1), session(10, 1))
|
||||
.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
lifecycle.publish_domain(),
|
||||
Err(LifecycleError::InvalidTransition {
|
||||
action: LifecycleAction::PublishDomain,
|
||||
..
|
||||
})
|
||||
));
|
||||
assert!(matches!(
|
||||
lifecycle.start_sandbox(),
|
||||
Err(LifecycleError::InvalidTransition {
|
||||
action: LifecycleAction::StartSandbox,
|
||||
..
|
||||
})
|
||||
));
|
||||
assert!(matches!(
|
||||
lifecycle.mount_state_anchor(),
|
||||
Err(LifecycleError::InvalidTransition {
|
||||
action: LifecycleAction::MountStateAnchor,
|
||||
..
|
||||
})
|
||||
));
|
||||
assert_eq!(
|
||||
lifecycle.into_adapter().actions,
|
||||
vec![LifecycleAction::StartNewDomain]
|
||||
);
|
||||
assert_eq!(
|
||||
store.snapshot().unwrap().generations[0].durable_decision,
|
||||
DurableDecision::None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configuration_mismatch_is_refused_without_registration() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = open_store(&directory);
|
||||
let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
|
||||
let owner = session(10, 1);
|
||||
lifecycle
|
||||
.start_new_domain(generation(1), owner.clone())
|
||||
.unwrap();
|
||||
complete_startup(&mut lifecycle);
|
||||
let mismatched = session(11, 1);
|
||||
|
||||
assert!(matches!(
|
||||
lifecycle.start_ready_session(mismatched.clone(), fingerprint(9)),
|
||||
Err(LifecycleError::InvalidTransition {
|
||||
action: LifecycleAction::StartReadySession,
|
||||
..
|
||||
})
|
||||
));
|
||||
lifecycle
|
||||
.reject_new_configuration_mismatch(mismatched.clone(), fingerprint(9))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
lifecycle.session_phase(mismatched.session_id),
|
||||
Some(SessionPhase::Failed)
|
||||
);
|
||||
assert_eq!(lifecycle.registered_sessions(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiting_session_restarts_after_normal_teardown() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = open_store(&directory);
|
||||
let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
|
||||
let owner = session(10, 1);
|
||||
lifecycle
|
||||
.start_new_domain(generation(1), owner.clone())
|
||||
.unwrap();
|
||||
complete_startup(&mut lifecycle);
|
||||
lifecycle
|
||||
.advance_waiting_session(owner.session_id, fingerprint(2))
|
||||
.unwrap();
|
||||
lifecycle.register_session(owner.session_id).unwrap();
|
||||
lifecycle.disconnect_session(owner.session_id).unwrap();
|
||||
lifecycle.begin_teardown().unwrap();
|
||||
let waiter = session(11, 1);
|
||||
lifecycle.wait_for_domain(waiter.clone()).unwrap();
|
||||
lifecycle.stop_sandbox().unwrap();
|
||||
lifecycle.stop_sidecar().unwrap();
|
||||
lifecycle.unmount_state_anchor().unwrap();
|
||||
lifecycle.release_prepared_lower_lease().unwrap();
|
||||
lifecycle.graceful_supervisor_exit().unwrap();
|
||||
lifecycle.finish_teardown().unwrap();
|
||||
lifecycle
|
||||
.restart_waiting_domain(generation(2), waiter.session_id)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(lifecycle.phase(), DomainPhase::Starting);
|
||||
let snapshot = store.snapshot().unwrap();
|
||||
assert!(snapshot.sessions.iter().any(|row| {
|
||||
row.generation_id == GenerationId::new(1)
|
||||
&& row.session_id == waiter.session_id
|
||||
&& row.session_phase == SessionPhase::Closed
|
||||
}));
|
||||
assert!(snapshot.sessions.iter().any(|row| {
|
||||
row.generation_id == GenerationId::new(2)
|
||||
&& row.session_id == waiter.session_id
|
||||
&& row.session_phase == SessionPhase::Waiting
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serialized_concurrent_joins_register_distinct_sessions() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = open_store(&directory);
|
||||
let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
|
||||
let owner = session(10, 1);
|
||||
lifecycle
|
||||
.start_new_domain(generation(1), owner.clone())
|
||||
.unwrap();
|
||||
complete_startup(&mut lifecycle);
|
||||
let lifecycle = Mutex::new(lifecycle);
|
||||
|
||||
std::thread::scope(|scope| {
|
||||
for id in 11..=18 {
|
||||
let lifecycle = &lifecycle;
|
||||
scope.spawn(move || {
|
||||
let candidate = session(id, 1);
|
||||
let mut guard = lifecycle.lock().unwrap();
|
||||
guard
|
||||
.start_ready_session(candidate.clone(), fingerprint(2))
|
||||
.unwrap();
|
||||
guard.register_session(candidate.session_id).unwrap();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
let lifecycle = lifecycle.into_inner().unwrap();
|
||||
assert_eq!(lifecycle.registered_sessions(), 8);
|
||||
assert_eq!(
|
||||
store
|
||||
.snapshot()
|
||||
.unwrap()
|
||||
.sessions
|
||||
.iter()
|
||||
.filter(|row| row.session_phase == SessionPhase::Attached)
|
||||
.count(),
|
||||
8
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_owner_disconnect_enters_failed_without_teardown_or_recovery() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let store = open_store(&directory);
|
||||
let mut lifecycle = HostCoordinatorLifecycle::new(&store, RecordingAdapter::default());
|
||||
let owner = session(10, 1);
|
||||
lifecycle
|
||||
.start_new_domain(generation(1), owner.clone())
|
||||
.unwrap();
|
||||
lifecycle.disconnect_session(owner.session_id).unwrap();
|
||||
|
||||
assert_eq!(lifecycle.phase(), DomainPhase::Failed);
|
||||
assert!(!lifecycle.is_published());
|
||||
assert!(matches!(
|
||||
lifecycle.begin_teardown(),
|
||||
Err(LifecycleError::InvalidTransition {
|
||||
action: LifecycleAction::BeginTeardown,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
Loading…
Reference in a new issue