Orchestrate normal domain lifecycle through adapters
This commit is contained in:
parent
49d5def1a9
commit
671473106c
1 changed files with 1266 additions and 0 deletions
918
crates/lazaret-host-coordinator/src/lifecycle.rs
Normal file
918
crates/lazaret-host-coordinator/src/lifecycle.rs
Normal file
|
|
@ -0,0 +1,918 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt;
|
||||
|
||||
use lazaret_core::identity::{ConfigurationFingerprint, SessionId};
|
||||
use lazaret_core::records::{DurableDecision, Generation, Session, SessionPhase};
|
||||
|
||||
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum LifecycleAction {
|
||||
StartNewDomain,
|
||||
StartReadySession,
|
||||
WaitForDomain,
|
||||
RestartWaitingDomain,
|
||||
CreateTrustAnchor,
|
||||
StartRuntimeSupervisor,
|
||||
LeasePreparedLower,
|
||||
MountStateAnchor,
|
||||
StartSidecar,
|
||||
StartConnectivityHelper,
|
||||
MarkNetworkReady,
|
||||
ArmRuntimeCausalEdge,
|
||||
StartSandbox,
|
||||
PublishDomain,
|
||||
AdvanceWaitingSession,
|
||||
RejectConfigurationMismatch,
|
||||
RegisterSession,
|
||||
DisconnectSession,
|
||||
BeginTeardown,
|
||||
StopSandbox,
|
||||
StopSidecar,
|
||||
UnmountStateAnchor,
|
||||
ReleasePreparedLowerLease,
|
||||
GracefulSupervisorExit,
|
||||
FinishTeardown,
|
||||
}
|
||||
|
||||
pub trait LifecycleAdapter {
|
||||
type Error;
|
||||
|
||||
/// # Errors
|
||||
/// Returns the adapter's physical or durable action failure.
|
||||
fn execute(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
generation: &Generation,
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum DomainPhase {
|
||||
Absent,
|
||||
Starting,
|
||||
Ready,
|
||||
Stopping,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum LifecycleError<E> {
|
||||
Adapter(E),
|
||||
Store(DomainStoreError),
|
||||
InvalidTransition {
|
||||
action: LifecycleAction,
|
||||
reason: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl<E: fmt::Display> fmt::Display for LifecycleError<E> {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Adapter(error) => write!(formatter, "lifecycle adapter failed: {error}"),
|
||||
Self::Store(error) => {
|
||||
write!(formatter, "Domain Store lifecycle update failed: {error}")
|
||||
}
|
||||
Self::InvalidTransition { action, reason } => {
|
||||
write!(formatter, "{action:?} refused: {reason}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: std::error::Error + 'static> std::error::Error for LifecycleError<E> {}
|
||||
|
||||
impl<E> From<DomainStoreError> for LifecycleError<E> {
|
||||
fn from(error: DomainStoreError) -> Self {
|
||||
Self::Store(error)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
enum RuntimeMilestone {
|
||||
TrustAnchor,
|
||||
RuntimeSupervisor,
|
||||
PreparedLowerLease,
|
||||
StateAnchor,
|
||||
Sidecar,
|
||||
ConnectivityHelper,
|
||||
NetworkReady,
|
||||
CausalEdge,
|
||||
Sandbox,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
struct RuntimeProgress {
|
||||
completed: HashSet<RuntimeMilestone>,
|
||||
}
|
||||
|
||||
impl RuntimeProgress {
|
||||
fn has(&self, milestone: RuntimeMilestone) -> bool {
|
||||
self.completed.contains(&milestone)
|
||||
}
|
||||
|
||||
fn mark(&mut self, milestone: RuntimeMilestone) {
|
||||
self.completed.insert(milestone);
|
||||
}
|
||||
|
||||
fn clear(&mut self, milestone: RuntimeMilestone) {
|
||||
self.completed.remove(&milestone);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct HostCoordinatorLifecycle<'store, A> {
|
||||
store: &'store HostCoordinatorDomainStore,
|
||||
adapter: A,
|
||||
phase: DomainPhase,
|
||||
generation: Option<Generation>,
|
||||
bootstrap_owner: Option<SessionId>,
|
||||
progress: RuntimeProgress,
|
||||
published: bool,
|
||||
sessions: HashMap<SessionId, Session>,
|
||||
connected: HashSet<SessionId>,
|
||||
registered: HashSet<SessionId>,
|
||||
}
|
||||
|
||||
impl<'store, A: LifecycleAdapter> HostCoordinatorLifecycle<'store, A> {
|
||||
#[must_use]
|
||||
pub fn new(store: &'store HostCoordinatorDomainStore, adapter: A) -> Self {
|
||||
Self {
|
||||
store,
|
||||
adapter,
|
||||
phase: DomainPhase::Absent,
|
||||
generation: None,
|
||||
bootstrap_owner: None,
|
||||
progress: RuntimeProgress::default(),
|
||||
published: false,
|
||||
sessions: HashMap::new(),
|
||||
connected: HashSet::new(),
|
||||
registered: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn phase(&self) -> DomainPhase {
|
||||
self.phase
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_published(&self) -> bool {
|
||||
self.published
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn session_phase(&self, session_id: SessionId) -> Option<SessionPhase> {
|
||||
self.sessions
|
||||
.get(&session_id)
|
||||
.map(|session| session.session_phase)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn registered_sessions(&self) -> usize {
|
||||
self.registered.len()
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a non-absent Domain, invalid initial records, or adapter/store failure.
|
||||
pub fn start_new_domain(
|
||||
&mut self,
|
||||
generation: Generation,
|
||||
mut session: Session,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
self.phase == DomainPhase::Absent,
|
||||
"Domain is not absent",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
generation.durable_decision == DurableDecision::None,
|
||||
"new generation decision is not None",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
session.generation_id == generation.generation_id,
|
||||
"session generation differs",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
session.session_phase == SessionPhase::New,
|
||||
"session is not New",
|
||||
)?;
|
||||
self.require_new_generation(LifecycleAction::StartNewDomain, &generation, &session)?;
|
||||
self.execute(LifecycleAction::StartNewDomain, &generation)?;
|
||||
session.session_phase = SessionPhase::Waiting;
|
||||
self.store.apply(&[
|
||||
DomainStoreMutation::PutGeneration(generation.clone()),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.phase = DomainPhase::Starting;
|
||||
self.bootstrap_owner = Some(session.session_id);
|
||||
self.sessions.insert(session.session_id, session);
|
||||
self.generation = Some(generation);
|
||||
self.progress = RuntimeProgress::default();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an unpublished, stale, mismatched, or duplicate session.
|
||||
pub fn start_ready_session(
|
||||
&mut self,
|
||||
session: Session,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.require_ready_join(LifecycleAction::StartReadySession, &session, fingerprint)?;
|
||||
Self::require(
|
||||
LifecycleAction::StartReadySession,
|
||||
session.session_phase == SessionPhase::New
|
||||
&& !self.sessions.contains_key(&session.session_id),
|
||||
"session is not a distinct New session",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::StartReadySession)?;
|
||||
self.put_new_session(
|
||||
LifecycleAction::StartReadySession,
|
||||
session,
|
||||
SessionPhase::Registering,
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a session unless the Domain is starting or stopping.
|
||||
pub fn wait_for_domain(&mut self, session: Session) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::WaitForDomain,
|
||||
matches!(self.phase, DomainPhase::Starting | DomainPhase::Stopping),
|
||||
"Domain is neither starting nor stopping",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::WaitForDomain,
|
||||
session.session_phase == SessionPhase::New,
|
||||
"session is not New",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::WaitForDomain,
|
||||
session.generation_id
|
||||
== self
|
||||
.current_generation(LifecycleAction::WaitForDomain)?
|
||||
.generation_id
|
||||
&& !self.sessions.contains_key(&session.session_id),
|
||||
"session generation differs or Session ID is already known",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::WaitForDomain)?;
|
||||
self.put_new_session(
|
||||
LifecycleAction::WaitForDomain,
|
||||
session,
|
||||
SessionPhase::Waiting,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a restart unless the Domain is absent and the session is waiting.
|
||||
pub fn restart_waiting_domain(
|
||||
&mut self,
|
||||
generation: Generation,
|
||||
session_id: SessionId,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
self.phase == DomainPhase::Absent,
|
||||
"Domain is not absent",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
generation.durable_decision == DurableDecision::None,
|
||||
"new generation decision is not None",
|
||||
)?;
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::RestartWaitingDomain)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
session.session_phase == SessionPhase::Waiting,
|
||||
"session is not Waiting",
|
||||
)?;
|
||||
let successor_session = Session {
|
||||
generation_id: generation.generation_id,
|
||||
..session.clone()
|
||||
};
|
||||
self.require_new_generation(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
&generation,
|
||||
&successor_session,
|
||||
)?;
|
||||
self.execute(LifecycleAction::RestartWaitingDomain, &generation)?;
|
||||
let previous = Session {
|
||||
session_phase: SessionPhase::Closed,
|
||||
..session.clone()
|
||||
};
|
||||
session.generation_id = generation.generation_id;
|
||||
self.store.apply(&[
|
||||
DomainStoreMutation::PutSession(previous),
|
||||
DomainStoreMutation::PutGeneration(generation.clone()),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.phase = DomainPhase::Starting;
|
||||
self.bootstrap_owner = Some(session_id);
|
||||
self.generation = Some(generation);
|
||||
self.progress = RuntimeProgress::default();
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn create_trust_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::CreateTrustAnchor,
|
||||
!self.progress.has(RuntimeMilestone::TrustAnchor),
|
||||
|p| p.mark(RuntimeMilestone::TrustAnchor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn start_runtime_supervisor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartRuntimeSupervisor,
|
||||
self.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
|p| p.mark(RuntimeMilestone::RuntimeSupervisor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn lease_prepared_lower(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::LeasePreparedLower,
|
||||
self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||||
|p| p.mark(RuntimeMilestone::PreparedLowerLease),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn mount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::MountStateAnchor,
|
||||
self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.progress.has(RuntimeMilestone::StateAnchor),
|
||||
|p| p.mark(RuntimeMilestone::StateAnchor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn start_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartSidecar,
|
||||
self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::Sidecar),
|
||||
|p| p.mark(RuntimeMilestone::Sidecar),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn start_connectivity_helper(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartConnectivityHelper,
|
||||
self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self.progress.has(RuntimeMilestone::ConnectivityHelper),
|
||||
|p| p.mark(RuntimeMilestone::ConnectivityHelper),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn mark_network_ready(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::MarkNetworkReady,
|
||||
self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& !self.progress.has(RuntimeMilestone::NetworkReady),
|
||||
|p| p.mark(RuntimeMilestone::NetworkReady),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn arm_runtime_causal_edge(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::ArmRuntimeCausalEdge,
|
||||
self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.progress.has(RuntimeMilestone::CausalEdge),
|
||||
|p| p.mark(RuntimeMilestone::CausalEdge),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses launch before network readiness and the causal edge are established.
|
||||
pub fn start_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::StartSandbox,
|
||||
self.phase == DomainPhase::Starting && allowed,
|
||||
"runtime prerequisites are incomplete",
|
||||
)?;
|
||||
let mut generation = self
|
||||
.current_generation(LifecycleAction::StartSandbox)?
|
||||
.clone();
|
||||
generation.durable_decision = DurableDecision::Preparing;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation.clone());
|
||||
self.execute(LifecycleAction::StartSandbox, &generation)?;
|
||||
self.progress.mark(RuntimeMilestone::Sandbox);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses publication until every ordered startup prerequisite is complete.
|
||||
pub fn publish_domain(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.phase == DomainPhase::Starting
|
||||
&& self.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& self.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::PublishDomain,
|
||||
allowed,
|
||||
"publication prerequisites are incomplete",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::PublishDomain)?;
|
||||
let mut generation = self
|
||||
.current_generation(LifecycleAction::PublishDomain)?
|
||||
.clone();
|
||||
generation.durable_decision = DurableDecision::Published;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation);
|
||||
self.phase = DomainPhase::Ready;
|
||||
self.bootstrap_owner = None;
|
||||
self.published = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a non-waiting session, configuration mismatch, or unavailable Domain.
|
||||
pub fn advance_waiting_session(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.require_fingerprint(LifecycleAction::AdvanceWaitingSession, fingerprint)?;
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::AdvanceWaitingSession)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::AdvanceWaitingSession,
|
||||
session.session_phase == SessionPhase::Waiting && self.admission_ready(),
|
||||
"waiting session is not admissible",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::AdvanceWaitingSession)?;
|
||||
session.session_phase = SessionPhase::Registering;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.connected.insert(session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses unless a new or waiting session differs from the ready generation.
|
||||
pub fn reject_new_configuration_mismatch(
|
||||
&mut self,
|
||||
mut session: Session,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
self.phase == DomainPhase::Ready && session.session_phase == SessionPhase::New,
|
||||
"session is not eligible for mismatch refusal",
|
||||
)?;
|
||||
let generation = self.current_generation(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
session.generation_id == generation.generation_id
|
||||
&& fingerprint != generation.configuration_fingerprint,
|
||||
"configuration does not differ from the current generation",
|
||||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
!self.sessions.contains_key(&session.session_id),
|
||||
"Session ID is already known",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
let new = session.clone();
|
||||
session.session_phase = SessionPhase::Failed;
|
||||
self.store.apply(&[
|
||||
DomainStoreMutation::PutSession(new),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.sessions.insert(session.session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses unless a new or waiting session differs from the ready generation.
|
||||
pub fn reject_configuration_mismatch(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::RejectConfigurationMismatch)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
self.phase == DomainPhase::Ready
|
||||
&& matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::New | SessionPhase::Waiting
|
||||
),
|
||||
"session is not eligible for mismatch refusal",
|
||||
)?;
|
||||
let generation = self.current_generation(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
fingerprint != generation.configuration_fingerprint,
|
||||
"configuration matches",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
session.session_phase = SessionPhase::Failed;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.connected.remove(&session_id);
|
||||
self.registered.remove(&session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses a disconnected, stale, or non-registering session.
|
||||
pub fn register_session(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::RegisterSession)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::RegisterSession,
|
||||
self.admission_ready()
|
||||
&& session.session_phase == SessionPhase::Registering
|
||||
&& self.connected.contains(&session_id),
|
||||
"session is not admissible",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::RegisterSession)?;
|
||||
session.session_phase = SessionPhase::Attached;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.registered.insert(session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an absent or terminal session, or an adapter/store failure.
|
||||
pub fn disconnect_session(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let mut session = self
|
||||
.session(session_id, LifecycleAction::DisconnectSession)?
|
||||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::DisconnectSession,
|
||||
matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::Waiting
|
||||
| SessionPhase::Registering
|
||||
| SessionPhase::Attached
|
||||
| SessionPhase::Prompting
|
||||
| SessionPhase::HostExecuting
|
||||
),
|
||||
"session is not live",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::DisconnectSession)?;
|
||||
let leader_lost =
|
||||
self.phase == DomainPhase::Starting && self.bootstrap_owner == Some(session_id);
|
||||
session.session_phase = SessionPhase::Closed;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.connected.remove(&session_id);
|
||||
self.registered.remove(&session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
if leader_lost {
|
||||
self.phase = DomainPhase::Failed;
|
||||
self.bootstrap_owner = None;
|
||||
self.published = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_teardown_ready(&self) -> bool {
|
||||
self.phase == DomainPhase::Ready
|
||||
&& self.registered.is_empty()
|
||||
&& self.connected.is_empty()
|
||||
&& self.sessions.values().all(|session| {
|
||||
!matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::Waiting
|
||||
| SessionPhase::Registering
|
||||
| SessionPhase::Attached
|
||||
| SessionPhase::Prompting
|
||||
| SessionPhase::HostExecuting
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses while any session retains Domain interest.
|
||||
pub fn begin_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::BeginTeardown,
|
||||
self.is_teardown_ready(),
|
||||
"sessions still have interest",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::BeginTeardown)?;
|
||||
self.phase = DomainPhase::Stopping;
|
||||
self.published = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn stop_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::StopSandbox,
|
||||
self.progress.has(RuntimeMilestone::Sandbox),
|
||||
|p| {
|
||||
p.clear(RuntimeMilestone::Sandbox);
|
||||
p.clear(RuntimeMilestone::CausalEdge);
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn stop_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::StopSidecar,
|
||||
!self.progress.has(RuntimeMilestone::Sandbox)
|
||||
&& self.progress.has(RuntimeMilestone::Sidecar),
|
||||
|p| {
|
||||
p.clear(RuntimeMilestone::Sidecar);
|
||||
p.clear(RuntimeMilestone::ConnectivityHelper);
|
||||
p.clear(RuntimeMilestone::NetworkReady);
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn unmount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::UnmountStateAnchor,
|
||||
!self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self.progress.has(RuntimeMilestone::StateAnchor),
|
||||
|p| p.clear(RuntimeMilestone::StateAnchor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn release_prepared_lower_lease(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::ReleasePreparedLowerLease,
|
||||
!self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||||
|p| p.clear(RuntimeMilestone::PreparedLowerLease),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses an out-of-order action or adapter failure.
|
||||
pub fn graceful_supervisor_exit(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::GracefulSupervisorExit,
|
||||
!self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
|p| p.clear(RuntimeMilestone::RuntimeSupervisor),
|
||||
)
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses while runtime resources remain or the final durable update fails.
|
||||
pub fn finish_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.phase == DomainPhase::Stopping
|
||||
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::FinishTeardown,
|
||||
allowed,
|
||||
"runtime resources remain",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::FinishTeardown)?;
|
||||
let mut generation = self
|
||||
.current_generation(LifecycleAction::FinishTeardown)?
|
||||
.clone();
|
||||
generation.durable_decision = DurableDecision::Clean;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation);
|
||||
self.phase = DomainPhase::Absent;
|
||||
self.progress = RuntimeProgress::default();
|
||||
self.bootstrap_owner = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn into_adapter(self) -> A {
|
||||
self.adapter
|
||||
}
|
||||
|
||||
fn startup_action(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
allowed: bool,
|
||||
update: impl FnOnce(&mut RuntimeProgress),
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.phase == DomainPhase::Starting && allowed,
|
||||
"startup ordering refused",
|
||||
)?;
|
||||
self.execute_current(action)?;
|
||||
update(&mut self.progress);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn teardown_action(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
allowed: bool,
|
||||
update: impl FnOnce(&mut RuntimeProgress),
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.phase == DomainPhase::Stopping && allowed,
|
||||
"teardown ordering refused",
|
||||
)?;
|
||||
self.execute_current(action)?;
|
||||
update(&mut self.progress);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn put_new_session(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
mut session: Session,
|
||||
phase: SessionPhase,
|
||||
connected: bool,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
session.session_phase == SessionPhase::New,
|
||||
"session is not New",
|
||||
)?;
|
||||
Self::require(
|
||||
action,
|
||||
!self.sessions.contains_key(&session.session_id),
|
||||
"Session ID is already known",
|
||||
)?;
|
||||
session.session_phase = phase;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
if connected {
|
||||
self.connected.insert(session.session_id);
|
||||
}
|
||||
self.sessions.insert(session.session_id, session);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_ready_join(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
session: &Session,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.admission_ready(),
|
||||
"Domain is not published and ready",
|
||||
)?;
|
||||
Self::require(
|
||||
action,
|
||||
session.generation_id == self.current_generation(action)?.generation_id,
|
||||
"session generation differs",
|
||||
)?;
|
||||
Self::require(
|
||||
action,
|
||||
fingerprint == self.current_generation(action)?.configuration_fingerprint,
|
||||
"configuration differs",
|
||||
)
|
||||
}
|
||||
|
||||
fn require_new_generation(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
generation: &Generation,
|
||||
session: &Session,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let snapshot = self.store.snapshot()?;
|
||||
let latest = snapshot
|
||||
.generations
|
||||
.iter()
|
||||
.map(|candidate| candidate.generation_id)
|
||||
.max();
|
||||
Self::require(
|
||||
action,
|
||||
generation.domain_id == snapshot.domain.domain_id
|
||||
&& session.generation_id == generation.generation_id
|
||||
&& latest.is_none_or(|current| generation.generation_id > current),
|
||||
"generation is not a fresh successor for this Domain",
|
||||
)
|
||||
}
|
||||
|
||||
fn require_fingerprint(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
fingerprint: ConfigurationFingerprint,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
fingerprint == self.current_generation(action)?.configuration_fingerprint,
|
||||
"configuration differs",
|
||||
)
|
||||
}
|
||||
|
||||
fn admission_ready(&self) -> bool {
|
||||
self.phase == DomainPhase::Ready && self.published
|
||||
}
|
||||
|
||||
fn current_generation(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
) -> Result<&Generation, LifecycleError<A::Error>> {
|
||||
self.generation
|
||||
.as_ref()
|
||||
.ok_or(LifecycleError::InvalidTransition {
|
||||
action,
|
||||
reason: "generation is absent",
|
||||
})
|
||||
}
|
||||
|
||||
fn session(
|
||||
&self,
|
||||
session_id: SessionId,
|
||||
action: LifecycleAction,
|
||||
) -> Result<&Session, LifecycleError<A::Error>> {
|
||||
self.sessions
|
||||
.get(&session_id)
|
||||
.ok_or(LifecycleError::InvalidTransition {
|
||||
action,
|
||||
reason: "session is absent",
|
||||
})
|
||||
}
|
||||
|
||||
fn execute_current(&mut self, action: LifecycleAction) -> Result<(), LifecycleError<A::Error>> {
|
||||
let generation = self.current_generation(action)?.clone();
|
||||
self.execute(action, &generation)
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
generation: &Generation,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.adapter
|
||||
.execute(action, generation)
|
||||
.map_err(LifecycleError::Adapter)
|
||||
}
|
||||
|
||||
fn require(
|
||||
action: LifecycleAction,
|
||||
condition: bool,
|
||||
reason: &'static str,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
if condition {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(LifecycleError::InvalidTransition { action, reason })
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue