918 lines
32 KiB
Rust
918 lines
32 KiB
Rust
|
|
use std::collections::{HashMap, HashSet};
|
||
|
|
use std::fmt;
|
||
|
|
|
||
|
|
use lazaret_core::identity::{ConfigurationFingerprint, SessionId};
|
||
|
|
use lazaret_core::records::{DurableDecision, Generation, Session, SessionPhase};
|
||
|
|
|
||
|
|
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
|
||
|
|
|
||
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||
|
|
pub enum LifecycleAction {
|
||
|
|
StartNewDomain,
|
||
|
|
StartReadySession,
|
||
|
|
WaitForDomain,
|
||
|
|
RestartWaitingDomain,
|
||
|
|
CreateTrustAnchor,
|
||
|
|
StartRuntimeSupervisor,
|
||
|
|
LeasePreparedLower,
|
||
|
|
MountStateAnchor,
|
||
|
|
StartSidecar,
|
||
|
|
StartConnectivityHelper,
|
||
|
|
MarkNetworkReady,
|
||
|
|
ArmRuntimeCausalEdge,
|
||
|
|
StartSandbox,
|
||
|
|
PublishDomain,
|
||
|
|
AdvanceWaitingSession,
|
||
|
|
RejectConfigurationMismatch,
|
||
|
|
RegisterSession,
|
||
|
|
DisconnectSession,
|
||
|
|
BeginTeardown,
|
||
|
|
StopSandbox,
|
||
|
|
StopSidecar,
|
||
|
|
UnmountStateAnchor,
|
||
|
|
ReleasePreparedLowerLease,
|
||
|
|
GracefulSupervisorExit,
|
||
|
|
FinishTeardown,
|
||
|
|
}
|
||
|
|
|
||
|
|
pub trait LifecycleAdapter {
|
||
|
|
type Error;
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Returns the adapter's physical or durable action failure.
|
||
|
|
fn execute(
|
||
|
|
&mut self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
generation: &Generation,
|
||
|
|
) -> Result<(), Self::Error>;
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||
|
|
pub enum DomainPhase {
|
||
|
|
Absent,
|
||
|
|
Starting,
|
||
|
|
Ready,
|
||
|
|
Stopping,
|
||
|
|
Failed,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug)]
|
||
|
|
pub enum LifecycleError<E> {
|
||
|
|
Adapter(E),
|
||
|
|
Store(DomainStoreError),
|
||
|
|
InvalidTransition {
|
||
|
|
action: LifecycleAction,
|
||
|
|
reason: &'static str,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
impl<E: fmt::Display> fmt::Display for LifecycleError<E> {
|
||
|
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||
|
|
match self {
|
||
|
|
Self::Adapter(error) => write!(formatter, "lifecycle adapter failed: {error}"),
|
||
|
|
Self::Store(error) => {
|
||
|
|
write!(formatter, "Domain Store lifecycle update failed: {error}")
|
||
|
|
}
|
||
|
|
Self::InvalidTransition { action, reason } => {
|
||
|
|
write!(formatter, "{action:?} refused: {reason}")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
impl<E: std::error::Error + 'static> std::error::Error for LifecycleError<E> {}
|
||
|
|
|
||
|
|
impl<E> From<DomainStoreError> for LifecycleError<E> {
|
||
|
|
fn from(error: DomainStoreError) -> Self {
|
||
|
|
Self::Store(error)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||
|
|
enum RuntimeMilestone {
|
||
|
|
TrustAnchor,
|
||
|
|
RuntimeSupervisor,
|
||
|
|
PreparedLowerLease,
|
||
|
|
StateAnchor,
|
||
|
|
Sidecar,
|
||
|
|
ConnectivityHelper,
|
||
|
|
NetworkReady,
|
||
|
|
CausalEdge,
|
||
|
|
Sandbox,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||
|
|
struct RuntimeProgress {
|
||
|
|
completed: HashSet<RuntimeMilestone>,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl RuntimeProgress {
|
||
|
|
fn has(&self, milestone: RuntimeMilestone) -> bool {
|
||
|
|
self.completed.contains(&milestone)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn mark(&mut self, milestone: RuntimeMilestone) {
|
||
|
|
self.completed.insert(milestone);
|
||
|
|
}
|
||
|
|
|
||
|
|
fn clear(&mut self, milestone: RuntimeMilestone) {
|
||
|
|
self.completed.remove(&milestone);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub struct HostCoordinatorLifecycle<'store, A> {
|
||
|
|
store: &'store HostCoordinatorDomainStore,
|
||
|
|
adapter: A,
|
||
|
|
phase: DomainPhase,
|
||
|
|
generation: Option<Generation>,
|
||
|
|
bootstrap_owner: Option<SessionId>,
|
||
|
|
progress: RuntimeProgress,
|
||
|
|
published: bool,
|
||
|
|
sessions: HashMap<SessionId, Session>,
|
||
|
|
connected: HashSet<SessionId>,
|
||
|
|
registered: HashSet<SessionId>,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl<'store, A: LifecycleAdapter> HostCoordinatorLifecycle<'store, A> {
|
||
|
|
#[must_use]
|
||
|
|
pub fn new(store: &'store HostCoordinatorDomainStore, adapter: A) -> Self {
|
||
|
|
Self {
|
||
|
|
store,
|
||
|
|
adapter,
|
||
|
|
phase: DomainPhase::Absent,
|
||
|
|
generation: None,
|
||
|
|
bootstrap_owner: None,
|
||
|
|
progress: RuntimeProgress::default(),
|
||
|
|
published: false,
|
||
|
|
sessions: HashMap::new(),
|
||
|
|
connected: HashSet::new(),
|
||
|
|
registered: HashSet::new(),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[must_use]
|
||
|
|
pub const fn phase(&self) -> DomainPhase {
|
||
|
|
self.phase
|
||
|
|
}
|
||
|
|
|
||
|
|
#[must_use]
|
||
|
|
pub const fn is_published(&self) -> bool {
|
||
|
|
self.published
|
||
|
|
}
|
||
|
|
|
||
|
|
#[must_use]
|
||
|
|
pub fn session_phase(&self, session_id: SessionId) -> Option<SessionPhase> {
|
||
|
|
self.sessions
|
||
|
|
.get(&session_id)
|
||
|
|
.map(|session| session.session_phase)
|
||
|
|
}
|
||
|
|
|
||
|
|
#[must_use]
|
||
|
|
pub fn registered_sessions(&self) -> usize {
|
||
|
|
self.registered.len()
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses a non-absent Domain, invalid initial records, or adapter/store failure.
|
||
|
|
pub fn start_new_domain(
|
||
|
|
&mut self,
|
||
|
|
generation: Generation,
|
||
|
|
mut session: Session,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::StartNewDomain,
|
||
|
|
self.phase == DomainPhase::Absent,
|
||
|
|
"Domain is not absent",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::StartNewDomain,
|
||
|
|
generation.durable_decision == DurableDecision::None,
|
||
|
|
"new generation decision is not None",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::StartNewDomain,
|
||
|
|
session.generation_id == generation.generation_id,
|
||
|
|
"session generation differs",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::StartNewDomain,
|
||
|
|
session.session_phase == SessionPhase::New,
|
||
|
|
"session is not New",
|
||
|
|
)?;
|
||
|
|
self.require_new_generation(LifecycleAction::StartNewDomain, &generation, &session)?;
|
||
|
|
self.execute(LifecycleAction::StartNewDomain, &generation)?;
|
||
|
|
session.session_phase = SessionPhase::Waiting;
|
||
|
|
self.store.apply(&[
|
||
|
|
DomainStoreMutation::PutGeneration(generation.clone()),
|
||
|
|
DomainStoreMutation::PutSession(session.clone()),
|
||
|
|
])?;
|
||
|
|
self.phase = DomainPhase::Starting;
|
||
|
|
self.bootstrap_owner = Some(session.session_id);
|
||
|
|
self.sessions.insert(session.session_id, session);
|
||
|
|
self.generation = Some(generation);
|
||
|
|
self.progress = RuntimeProgress::default();
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an unpublished, stale, mismatched, or duplicate session.
|
||
|
|
pub fn start_ready_session(
|
||
|
|
&mut self,
|
||
|
|
session: Session,
|
||
|
|
fingerprint: ConfigurationFingerprint,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.require_ready_join(LifecycleAction::StartReadySession, &session, fingerprint)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::StartReadySession,
|
||
|
|
session.session_phase == SessionPhase::New
|
||
|
|
&& !self.sessions.contains_key(&session.session_id),
|
||
|
|
"session is not a distinct New session",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::StartReadySession)?;
|
||
|
|
self.put_new_session(
|
||
|
|
LifecycleAction::StartReadySession,
|
||
|
|
session,
|
||
|
|
SessionPhase::Registering,
|
||
|
|
true,
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses a session unless the Domain is starting or stopping.
|
||
|
|
pub fn wait_for_domain(&mut self, session: Session) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::WaitForDomain,
|
||
|
|
matches!(self.phase, DomainPhase::Starting | DomainPhase::Stopping),
|
||
|
|
"Domain is neither starting nor stopping",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::WaitForDomain,
|
||
|
|
session.session_phase == SessionPhase::New,
|
||
|
|
"session is not New",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::WaitForDomain,
|
||
|
|
session.generation_id
|
||
|
|
== self
|
||
|
|
.current_generation(LifecycleAction::WaitForDomain)?
|
||
|
|
.generation_id
|
||
|
|
&& !self.sessions.contains_key(&session.session_id),
|
||
|
|
"session generation differs or Session ID is already known",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::WaitForDomain)?;
|
||
|
|
self.put_new_session(
|
||
|
|
LifecycleAction::WaitForDomain,
|
||
|
|
session,
|
||
|
|
SessionPhase::Waiting,
|
||
|
|
false,
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses a restart unless the Domain is absent and the session is waiting.
|
||
|
|
pub fn restart_waiting_domain(
|
||
|
|
&mut self,
|
||
|
|
generation: Generation,
|
||
|
|
session_id: SessionId,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RestartWaitingDomain,
|
||
|
|
self.phase == DomainPhase::Absent,
|
||
|
|
"Domain is not absent",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RestartWaitingDomain,
|
||
|
|
generation.durable_decision == DurableDecision::None,
|
||
|
|
"new generation decision is not None",
|
||
|
|
)?;
|
||
|
|
let mut session = self
|
||
|
|
.session(session_id, LifecycleAction::RestartWaitingDomain)?
|
||
|
|
.clone();
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RestartWaitingDomain,
|
||
|
|
session.session_phase == SessionPhase::Waiting,
|
||
|
|
"session is not Waiting",
|
||
|
|
)?;
|
||
|
|
let successor_session = Session {
|
||
|
|
generation_id: generation.generation_id,
|
||
|
|
..session.clone()
|
||
|
|
};
|
||
|
|
self.require_new_generation(
|
||
|
|
LifecycleAction::RestartWaitingDomain,
|
||
|
|
&generation,
|
||
|
|
&successor_session,
|
||
|
|
)?;
|
||
|
|
self.execute(LifecycleAction::RestartWaitingDomain, &generation)?;
|
||
|
|
let previous = Session {
|
||
|
|
session_phase: SessionPhase::Closed,
|
||
|
|
..session.clone()
|
||
|
|
};
|
||
|
|
session.generation_id = generation.generation_id;
|
||
|
|
self.store.apply(&[
|
||
|
|
DomainStoreMutation::PutSession(previous),
|
||
|
|
DomainStoreMutation::PutGeneration(generation.clone()),
|
||
|
|
DomainStoreMutation::PutSession(session.clone()),
|
||
|
|
])?;
|
||
|
|
self.phase = DomainPhase::Starting;
|
||
|
|
self.bootstrap_owner = Some(session_id);
|
||
|
|
self.generation = Some(generation);
|
||
|
|
self.progress = RuntimeProgress::default();
|
||
|
|
self.sessions.insert(session_id, session);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn create_trust_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::CreateTrustAnchor,
|
||
|
|
!self.progress.has(RuntimeMilestone::TrustAnchor),
|
||
|
|
|p| p.mark(RuntimeMilestone::TrustAnchor),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn start_runtime_supervisor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::StartRuntimeSupervisor,
|
||
|
|
self.progress.has(RuntimeMilestone::TrustAnchor)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||
|
|
|p| p.mark(RuntimeMilestone::RuntimeSupervisor),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn lease_prepared_lower(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::LeasePreparedLower,
|
||
|
|
self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||
|
|
|p| p.mark(RuntimeMilestone::PreparedLowerLease),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn mount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::MountStateAnchor,
|
||
|
|
self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::StateAnchor),
|
||
|
|
|p| p.mark(RuntimeMilestone::StateAnchor),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn start_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::StartSidecar,
|
||
|
|
self.progress.has(RuntimeMilestone::StateAnchor)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::Sidecar),
|
||
|
|
|p| p.mark(RuntimeMilestone::Sidecar),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn start_connectivity_helper(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::StartConnectivityHelper,
|
||
|
|
self.progress.has(RuntimeMilestone::Sidecar)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::ConnectivityHelper),
|
||
|
|
|p| p.mark(RuntimeMilestone::ConnectivityHelper),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn mark_network_ready(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::MarkNetworkReady,
|
||
|
|
self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::NetworkReady),
|
||
|
|
|p| p.mark(RuntimeMilestone::NetworkReady),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn arm_runtime_causal_edge(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.startup_action(
|
||
|
|
LifecycleAction::ArmRuntimeCausalEdge,
|
||
|
|
self.progress.has(RuntimeMilestone::NetworkReady)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::CausalEdge),
|
||
|
|
|p| p.mark(RuntimeMilestone::CausalEdge),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses launch before network readiness and the causal edge are established.
|
||
|
|
pub fn start_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let allowed = self.progress.has(RuntimeMilestone::CausalEdge)
|
||
|
|
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::StartSandbox,
|
||
|
|
self.phase == DomainPhase::Starting && allowed,
|
||
|
|
"runtime prerequisites are incomplete",
|
||
|
|
)?;
|
||
|
|
let mut generation = self
|
||
|
|
.current_generation(LifecycleAction::StartSandbox)?
|
||
|
|
.clone();
|
||
|
|
generation.durable_decision = DurableDecision::Preparing;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||
|
|
self.generation = Some(generation.clone());
|
||
|
|
self.execute(LifecycleAction::StartSandbox, &generation)?;
|
||
|
|
self.progress.mark(RuntimeMilestone::Sandbox);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses publication until every ordered startup prerequisite is complete.
|
||
|
|
pub fn publish_domain(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let allowed = self.phase == DomainPhase::Starting
|
||
|
|
&& self.progress.has(RuntimeMilestone::TrustAnchor)
|
||
|
|
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||
|
|
&& self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||
|
|
&& self.progress.has(RuntimeMilestone::StateAnchor)
|
||
|
|
&& self.progress.has(RuntimeMilestone::Sidecar)
|
||
|
|
&& self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||
|
|
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||
|
|
&& self.progress.has(RuntimeMilestone::CausalEdge)
|
||
|
|
&& self.progress.has(RuntimeMilestone::Sandbox);
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::PublishDomain,
|
||
|
|
allowed,
|
||
|
|
"publication prerequisites are incomplete",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::PublishDomain)?;
|
||
|
|
let mut generation = self
|
||
|
|
.current_generation(LifecycleAction::PublishDomain)?
|
||
|
|
.clone();
|
||
|
|
generation.durable_decision = DurableDecision::Published;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||
|
|
self.generation = Some(generation);
|
||
|
|
self.phase = DomainPhase::Ready;
|
||
|
|
self.bootstrap_owner = None;
|
||
|
|
self.published = true;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses a non-waiting session, configuration mismatch, or unavailable Domain.
|
||
|
|
pub fn advance_waiting_session(
|
||
|
|
&mut self,
|
||
|
|
session_id: SessionId,
|
||
|
|
fingerprint: ConfigurationFingerprint,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.require_fingerprint(LifecycleAction::AdvanceWaitingSession, fingerprint)?;
|
||
|
|
let mut session = self
|
||
|
|
.session(session_id, LifecycleAction::AdvanceWaitingSession)?
|
||
|
|
.clone();
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::AdvanceWaitingSession,
|
||
|
|
session.session_phase == SessionPhase::Waiting && self.admission_ready(),
|
||
|
|
"waiting session is not admissible",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::AdvanceWaitingSession)?;
|
||
|
|
session.session_phase = SessionPhase::Registering;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||
|
|
self.connected.insert(session_id);
|
||
|
|
self.sessions.insert(session_id, session);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses unless a new or waiting session differs from the ready generation.
|
||
|
|
pub fn reject_new_configuration_mismatch(
|
||
|
|
&mut self,
|
||
|
|
mut session: Session,
|
||
|
|
fingerprint: ConfigurationFingerprint,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RejectConfigurationMismatch,
|
||
|
|
self.phase == DomainPhase::Ready && session.session_phase == SessionPhase::New,
|
||
|
|
"session is not eligible for mismatch refusal",
|
||
|
|
)?;
|
||
|
|
let generation = self.current_generation(LifecycleAction::RejectConfigurationMismatch)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RejectConfigurationMismatch,
|
||
|
|
session.generation_id == generation.generation_id
|
||
|
|
&& fingerprint != generation.configuration_fingerprint,
|
||
|
|
"configuration does not differ from the current generation",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RejectConfigurationMismatch,
|
||
|
|
!self.sessions.contains_key(&session.session_id),
|
||
|
|
"Session ID is already known",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::RejectConfigurationMismatch)?;
|
||
|
|
let new = session.clone();
|
||
|
|
session.session_phase = SessionPhase::Failed;
|
||
|
|
self.store.apply(&[
|
||
|
|
DomainStoreMutation::PutSession(new),
|
||
|
|
DomainStoreMutation::PutSession(session.clone()),
|
||
|
|
])?;
|
||
|
|
self.sessions.insert(session.session_id, session);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses unless a new or waiting session differs from the ready generation.
|
||
|
|
pub fn reject_configuration_mismatch(
|
||
|
|
&mut self,
|
||
|
|
session_id: SessionId,
|
||
|
|
fingerprint: ConfigurationFingerprint,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let mut session = self
|
||
|
|
.session(session_id, LifecycleAction::RejectConfigurationMismatch)?
|
||
|
|
.clone();
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RejectConfigurationMismatch,
|
||
|
|
self.phase == DomainPhase::Ready
|
||
|
|
&& matches!(
|
||
|
|
session.session_phase,
|
||
|
|
SessionPhase::New | SessionPhase::Waiting
|
||
|
|
),
|
||
|
|
"session is not eligible for mismatch refusal",
|
||
|
|
)?;
|
||
|
|
let generation = self.current_generation(LifecycleAction::RejectConfigurationMismatch)?;
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RejectConfigurationMismatch,
|
||
|
|
fingerprint != generation.configuration_fingerprint,
|
||
|
|
"configuration matches",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::RejectConfigurationMismatch)?;
|
||
|
|
session.session_phase = SessionPhase::Failed;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||
|
|
self.connected.remove(&session_id);
|
||
|
|
self.registered.remove(&session_id);
|
||
|
|
self.sessions.insert(session_id, session);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses a disconnected, stale, or non-registering session.
|
||
|
|
pub fn register_session(
|
||
|
|
&mut self,
|
||
|
|
session_id: SessionId,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let mut session = self
|
||
|
|
.session(session_id, LifecycleAction::RegisterSession)?
|
||
|
|
.clone();
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::RegisterSession,
|
||
|
|
self.admission_ready()
|
||
|
|
&& session.session_phase == SessionPhase::Registering
|
||
|
|
&& self.connected.contains(&session_id),
|
||
|
|
"session is not admissible",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::RegisterSession)?;
|
||
|
|
session.session_phase = SessionPhase::Attached;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||
|
|
self.registered.insert(session_id);
|
||
|
|
self.sessions.insert(session_id, session);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an absent or terminal session, or an adapter/store failure.
|
||
|
|
pub fn disconnect_session(
|
||
|
|
&mut self,
|
||
|
|
session_id: SessionId,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let mut session = self
|
||
|
|
.session(session_id, LifecycleAction::DisconnectSession)?
|
||
|
|
.clone();
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::DisconnectSession,
|
||
|
|
matches!(
|
||
|
|
session.session_phase,
|
||
|
|
SessionPhase::Waiting
|
||
|
|
| SessionPhase::Registering
|
||
|
|
| SessionPhase::Attached
|
||
|
|
| SessionPhase::Prompting
|
||
|
|
| SessionPhase::HostExecuting
|
||
|
|
),
|
||
|
|
"session is not live",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::DisconnectSession)?;
|
||
|
|
let leader_lost =
|
||
|
|
self.phase == DomainPhase::Starting && self.bootstrap_owner == Some(session_id);
|
||
|
|
session.session_phase = SessionPhase::Closed;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||
|
|
self.connected.remove(&session_id);
|
||
|
|
self.registered.remove(&session_id);
|
||
|
|
self.sessions.insert(session_id, session);
|
||
|
|
if leader_lost {
|
||
|
|
self.phase = DomainPhase::Failed;
|
||
|
|
self.bootstrap_owner = None;
|
||
|
|
self.published = false;
|
||
|
|
}
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
#[must_use]
|
||
|
|
pub fn is_teardown_ready(&self) -> bool {
|
||
|
|
self.phase == DomainPhase::Ready
|
||
|
|
&& self.registered.is_empty()
|
||
|
|
&& self.connected.is_empty()
|
||
|
|
&& self.sessions.values().all(|session| {
|
||
|
|
!matches!(
|
||
|
|
session.session_phase,
|
||
|
|
SessionPhase::Waiting
|
||
|
|
| SessionPhase::Registering
|
||
|
|
| SessionPhase::Attached
|
||
|
|
| SessionPhase::Prompting
|
||
|
|
| SessionPhase::HostExecuting
|
||
|
|
)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses while any session retains Domain interest.
|
||
|
|
pub fn begin_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::BeginTeardown,
|
||
|
|
self.is_teardown_ready(),
|
||
|
|
"sessions still have interest",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::BeginTeardown)?;
|
||
|
|
self.phase = DomainPhase::Stopping;
|
||
|
|
self.published = false;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn stop_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.teardown_action(
|
||
|
|
LifecycleAction::StopSandbox,
|
||
|
|
self.progress.has(RuntimeMilestone::Sandbox),
|
||
|
|
|p| {
|
||
|
|
p.clear(RuntimeMilestone::Sandbox);
|
||
|
|
p.clear(RuntimeMilestone::CausalEdge);
|
||
|
|
},
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn stop_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.teardown_action(
|
||
|
|
LifecycleAction::StopSidecar,
|
||
|
|
!self.progress.has(RuntimeMilestone::Sandbox)
|
||
|
|
&& self.progress.has(RuntimeMilestone::Sidecar),
|
||
|
|
|p| {
|
||
|
|
p.clear(RuntimeMilestone::Sidecar);
|
||
|
|
p.clear(RuntimeMilestone::ConnectivityHelper);
|
||
|
|
p.clear(RuntimeMilestone::NetworkReady);
|
||
|
|
},
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn unmount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.teardown_action(
|
||
|
|
LifecycleAction::UnmountStateAnchor,
|
||
|
|
!self.progress.has(RuntimeMilestone::Sidecar)
|
||
|
|
&& self.progress.has(RuntimeMilestone::StateAnchor),
|
||
|
|
|p| p.clear(RuntimeMilestone::StateAnchor),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn release_prepared_lower_lease(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.teardown_action(
|
||
|
|
LifecycleAction::ReleasePreparedLowerLease,
|
||
|
|
!self.progress.has(RuntimeMilestone::StateAnchor)
|
||
|
|
&& self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||
|
|
|p| p.clear(RuntimeMilestone::PreparedLowerLease),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses an out-of-order action or adapter failure.
|
||
|
|
pub fn graceful_supervisor_exit(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.teardown_action(
|
||
|
|
LifecycleAction::GracefulSupervisorExit,
|
||
|
|
!self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||
|
|
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||
|
|
|p| p.clear(RuntimeMilestone::RuntimeSupervisor),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// # Errors
|
||
|
|
/// Refuses while runtime resources remain or the final durable update fails.
|
||
|
|
pub fn finish_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let allowed = self.phase == DomainPhase::Stopping
|
||
|
|
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::StateAnchor)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::Sidecar)
|
||
|
|
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||
|
|
Self::require(
|
||
|
|
LifecycleAction::FinishTeardown,
|
||
|
|
allowed,
|
||
|
|
"runtime resources remain",
|
||
|
|
)?;
|
||
|
|
self.execute_current(LifecycleAction::FinishTeardown)?;
|
||
|
|
let mut generation = self
|
||
|
|
.current_generation(LifecycleAction::FinishTeardown)?
|
||
|
|
.clone();
|
||
|
|
generation.durable_decision = DurableDecision::Clean;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||
|
|
self.generation = Some(generation);
|
||
|
|
self.phase = DomainPhase::Absent;
|
||
|
|
self.progress = RuntimeProgress::default();
|
||
|
|
self.bootstrap_owner = None;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn into_adapter(self) -> A {
|
||
|
|
self.adapter
|
||
|
|
}
|
||
|
|
|
||
|
|
fn startup_action(
|
||
|
|
&mut self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
allowed: bool,
|
||
|
|
update: impl FnOnce(&mut RuntimeProgress),
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
self.phase == DomainPhase::Starting && allowed,
|
||
|
|
"startup ordering refused",
|
||
|
|
)?;
|
||
|
|
self.execute_current(action)?;
|
||
|
|
update(&mut self.progress);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn teardown_action(
|
||
|
|
&mut self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
allowed: bool,
|
||
|
|
update: impl FnOnce(&mut RuntimeProgress),
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
self.phase == DomainPhase::Stopping && allowed,
|
||
|
|
"teardown ordering refused",
|
||
|
|
)?;
|
||
|
|
self.execute_current(action)?;
|
||
|
|
update(&mut self.progress);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn put_new_session(
|
||
|
|
&mut self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
mut session: Session,
|
||
|
|
phase: SessionPhase,
|
||
|
|
connected: bool,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
session.session_phase == SessionPhase::New,
|
||
|
|
"session is not New",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
!self.sessions.contains_key(&session.session_id),
|
||
|
|
"Session ID is already known",
|
||
|
|
)?;
|
||
|
|
session.session_phase = phase;
|
||
|
|
self.store
|
||
|
|
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||
|
|
if connected {
|
||
|
|
self.connected.insert(session.session_id);
|
||
|
|
}
|
||
|
|
self.sessions.insert(session.session_id, session);
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
fn require_ready_join(
|
||
|
|
&self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
session: &Session,
|
||
|
|
fingerprint: ConfigurationFingerprint,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
self.admission_ready(),
|
||
|
|
"Domain is not published and ready",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
session.generation_id == self.current_generation(action)?.generation_id,
|
||
|
|
"session generation differs",
|
||
|
|
)?;
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
fingerprint == self.current_generation(action)?.configuration_fingerprint,
|
||
|
|
"configuration differs",
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn require_new_generation(
|
||
|
|
&self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
generation: &Generation,
|
||
|
|
session: &Session,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let snapshot = self.store.snapshot()?;
|
||
|
|
let latest = snapshot
|
||
|
|
.generations
|
||
|
|
.iter()
|
||
|
|
.map(|candidate| candidate.generation_id)
|
||
|
|
.max();
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
generation.domain_id == snapshot.domain.domain_id
|
||
|
|
&& session.generation_id == generation.generation_id
|
||
|
|
&& latest.is_none_or(|current| generation.generation_id > current),
|
||
|
|
"generation is not a fresh successor for this Domain",
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn require_fingerprint(
|
||
|
|
&self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
fingerprint: ConfigurationFingerprint,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
Self::require(
|
||
|
|
action,
|
||
|
|
fingerprint == self.current_generation(action)?.configuration_fingerprint,
|
||
|
|
"configuration differs",
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn admission_ready(&self) -> bool {
|
||
|
|
self.phase == DomainPhase::Ready && self.published
|
||
|
|
}
|
||
|
|
|
||
|
|
fn current_generation(
|
||
|
|
&self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
) -> Result<&Generation, LifecycleError<A::Error>> {
|
||
|
|
self.generation
|
||
|
|
.as_ref()
|
||
|
|
.ok_or(LifecycleError::InvalidTransition {
|
||
|
|
action,
|
||
|
|
reason: "generation is absent",
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn session(
|
||
|
|
&self,
|
||
|
|
session_id: SessionId,
|
||
|
|
action: LifecycleAction,
|
||
|
|
) -> Result<&Session, LifecycleError<A::Error>> {
|
||
|
|
self.sessions
|
||
|
|
.get(&session_id)
|
||
|
|
.ok_or(LifecycleError::InvalidTransition {
|
||
|
|
action,
|
||
|
|
reason: "session is absent",
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
fn execute_current(&mut self, action: LifecycleAction) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
let generation = self.current_generation(action)?.clone();
|
||
|
|
self.execute(action, &generation)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn execute(
|
||
|
|
&mut self,
|
||
|
|
action: LifecycleAction,
|
||
|
|
generation: &Generation,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
self.adapter
|
||
|
|
.execute(action, generation)
|
||
|
|
.map_err(LifecycleError::Adapter)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn require(
|
||
|
|
action: LifecycleAction,
|
||
|
|
condition: bool,
|
||
|
|
reason: &'static str,
|
||
|
|
) -> Result<(), LifecycleError<A::Error>> {
|
||
|
|
if condition {
|
||
|
|
Ok(())
|
||
|
|
} else {
|
||
|
|
Err(LifecycleError::InvalidTransition { action, reason })
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|