Unify host coordinator lifecycle state
This commit is contained in:
parent
85046fdd7f
commit
2f8a9f9178
1 changed files with 720 additions and 337 deletions
|
|
@ -1,180 +1,22 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt;
|
||||
|
||||
use lazaret_core::identity::{ConfigurationFingerprint, SessionId};
|
||||
use lazaret_core::records::{DurableDecision, Generation, Session, SessionPhase};
|
||||
|
||||
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
|
||||
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum LifecycleAction {
|
||||
StartNewDomain,
|
||||
StartReadySession,
|
||||
WaitForDomain,
|
||||
RestartWaitingDomain,
|
||||
CreateTrustAnchor,
|
||||
StartRuntimeSupervisor,
|
||||
LeasePreparedLower,
|
||||
MountStateAnchor,
|
||||
StartSidecar,
|
||||
StartConnectivityHelper,
|
||||
MarkNetworkReady,
|
||||
ArmRuntimeCausalEdge,
|
||||
StartSandbox,
|
||||
PublishDomain,
|
||||
AdvanceWaitingSession,
|
||||
RejectConfigurationMismatch,
|
||||
RegisterSession,
|
||||
DisconnectSession,
|
||||
BeginTeardown,
|
||||
StopSandbox,
|
||||
StopSidecar,
|
||||
UnmountStateAnchor,
|
||||
ReleasePreparedLowerLease,
|
||||
GracefulSupervisorExit,
|
||||
FinishTeardown,
|
||||
}
|
||||
|
||||
pub trait LifecycleAdapter {
|
||||
type Error;
|
||||
|
||||
/// # Errors
|
||||
/// Returns the adapter's physical or durable action failure.
|
||||
fn execute(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
generation: &Generation,
|
||||
) -> Result<(), Self::Error>;
|
||||
|
||||
/// Completes a physical boundary that must follow its durable lifecycle commit.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the adapter's post-commit publication or cleanup failure.
|
||||
fn after_commit(
|
||||
&mut self,
|
||||
_action: LifecycleAction,
|
||||
_generation: &Generation,
|
||||
) -> Result<(), Self::Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct SessionAdmissionIdentity {
|
||||
pub generation_id: lazaret_core::identity::GenerationId,
|
||||
pub session_id: SessionId,
|
||||
pub configuration_fingerprint: ConfigurationFingerprint,
|
||||
}
|
||||
|
||||
pub trait SessionLifecycleAdapter {
|
||||
type Error;
|
||||
type Admission;
|
||||
|
||||
fn admission_identity(&self, admission: &Self::Admission) -> SessionAdmissionIdentity;
|
||||
|
||||
/// # Errors
|
||||
/// Returns credential, identity, duplicate-session, or durable registration failure.
|
||||
fn register_session(
|
||||
&self,
|
||||
store: &HostCoordinatorDomainStore,
|
||||
entry_action: LifecycleAction,
|
||||
admission: Self::Admission,
|
||||
) -> Result<Session, Self::Error>;
|
||||
|
||||
/// # Errors
|
||||
/// Returns exact-session or durable disconnect failure.
|
||||
fn disconnect_session(
|
||||
&self,
|
||||
store: &HostCoordinatorDomainStore,
|
||||
session: &Session,
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum DomainPhase {
|
||||
Absent,
|
||||
Starting,
|
||||
Ready,
|
||||
Stopping,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum LifecycleError<E> {
|
||||
Adapter(E),
|
||||
Store(DomainStoreError),
|
||||
InvalidTransition {
|
||||
action: LifecycleAction,
|
||||
reason: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl<E: fmt::Display> fmt::Display for LifecycleError<E> {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Adapter(error) => write!(formatter, "lifecycle adapter failed: {error}"),
|
||||
Self::Store(error) => {
|
||||
write!(formatter, "Domain Store lifecycle update failed: {error}")
|
||||
}
|
||||
Self::InvalidTransition { action, reason } => {
|
||||
write!(formatter, "{action:?} refused: {reason}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: std::error::Error + 'static> std::error::Error for LifecycleError<E> {}
|
||||
|
||||
impl<E> From<DomainStoreError> for LifecycleError<E> {
|
||||
fn from(error: DomainStoreError) -> Self {
|
||||
Self::Store(error)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
enum RuntimeMilestone {
|
||||
TrustAnchor,
|
||||
RuntimeSupervisor,
|
||||
PreparedLowerLease,
|
||||
StateAnchor,
|
||||
Sidecar,
|
||||
ConnectivityHelper,
|
||||
NetworkReady,
|
||||
CausalEdge,
|
||||
Sandbox,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
struct RuntimeProgress {
|
||||
completed: HashSet<RuntimeMilestone>,
|
||||
}
|
||||
|
||||
impl RuntimeProgress {
|
||||
fn has(&self, milestone: RuntimeMilestone) -> bool {
|
||||
self.completed.contains(&milestone)
|
||||
}
|
||||
|
||||
fn mark(&mut self, milestone: RuntimeMilestone) {
|
||||
self.completed.insert(milestone);
|
||||
}
|
||||
|
||||
fn clear(&mut self, milestone: RuntimeMilestone) {
|
||||
self.completed.remove(&milestone);
|
||||
}
|
||||
}
|
||||
mod contracts;
|
||||
mod state;
|
||||
pub use contracts::{
|
||||
LifecycleAction, LifecycleAdapter, LifecycleError, SessionAdmissionIdentity,
|
||||
SessionLifecycleAdapter,
|
||||
};
|
||||
pub use state::DomainPhase;
|
||||
use state::{LifecycleAggregate, RuntimeMilestone, RuntimeProgress};
|
||||
|
||||
pub struct HostCoordinatorLifecycle<'store, A, S> {
|
||||
store: &'store HostCoordinatorDomainStore,
|
||||
adapter: A,
|
||||
session_adapter: S,
|
||||
phase: DomainPhase,
|
||||
generation: Option<Generation>,
|
||||
bootstrap_owner: Option<SessionId>,
|
||||
progress: RuntimeProgress,
|
||||
published: bool,
|
||||
sessions: HashMap<SessionId, Session>,
|
||||
connected: HashSet<SessionId>,
|
||||
registered: HashSet<SessionId>,
|
||||
state: LifecycleAggregate,
|
||||
}
|
||||
|
||||
impl<'store, A, S> HostCoordinatorLifecycle<'store, A, S>
|
||||
|
|
@ -188,37 +30,31 @@ where
|
|||
store,
|
||||
adapter,
|
||||
session_adapter,
|
||||
phase: DomainPhase::Absent,
|
||||
generation: None,
|
||||
bootstrap_owner: None,
|
||||
progress: RuntimeProgress::default(),
|
||||
published: false,
|
||||
sessions: HashMap::new(),
|
||||
connected: HashSet::new(),
|
||||
registered: HashSet::new(),
|
||||
state: LifecycleAggregate::absent(),
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn phase(&self) -> DomainPhase {
|
||||
self.phase
|
||||
self.state.phase
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn is_published(&self) -> bool {
|
||||
self.published
|
||||
self.state.published
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn session_phase(&self, session_id: SessionId) -> Option<SessionPhase> {
|
||||
self.sessions
|
||||
self.state
|
||||
.sessions
|
||||
.get(&session_id)
|
||||
.map(|session| session.session_phase)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn registered_sessions(&self) -> usize {
|
||||
self.registered.len()
|
||||
self.state.registered_sessions()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
|
@ -240,7 +76,7 @@ where
|
|||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::StartNewDomain,
|
||||
self.phase == DomainPhase::Absent,
|
||||
self.state.phase == DomainPhase::Absent,
|
||||
"Domain is not absent",
|
||||
)?;
|
||||
Self::require(
|
||||
|
|
@ -265,12 +101,13 @@ where
|
|||
DomainStoreMutation::PutGeneration(generation.clone()),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.phase = DomainPhase::Starting;
|
||||
self.bootstrap_owner = Some(session.session_id);
|
||||
self.sessions.insert(session.session_id, session);
|
||||
self.generation = Some(generation);
|
||||
self.progress = RuntimeProgress::default();
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::StartNewDomain, move |state| {
|
||||
state.phase = DomainPhase::Starting;
|
||||
state.bootstrap_owner = Some(session.session_id);
|
||||
state.sessions.insert(session.session_id, session);
|
||||
state.generation = Some(generation);
|
||||
state.progress = RuntimeProgress::default();
|
||||
})
|
||||
}
|
||||
|
||||
/// Drives one cold-start leader through the complete normal publication boundary.
|
||||
|
|
@ -365,7 +202,7 @@ where
|
|||
admission: S::Admission,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
let identity = self.session_adapter.admission_identity(&admission);
|
||||
let entry_action = match self.sessions.get(&identity.session_id) {
|
||||
let entry_action = match self.state.sessions.get(&identity.session_id) {
|
||||
None => LifecycleAction::StartReadySession,
|
||||
Some(session) if session.session_phase == SessionPhase::Waiting => {
|
||||
LifecycleAction::AdvanceWaitingSession
|
||||
|
|
@ -396,10 +233,9 @@ where
|
|||
&& attached.session_phase == SessionPhase::Attached,
|
||||
"session adapter returned mismatched registration evidence",
|
||||
)?;
|
||||
self.connected.insert(attached.session_id);
|
||||
self.registered.insert(attached.session_id);
|
||||
self.sessions.insert(attached.session_id, attached);
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::RegisterSession, move |state| {
|
||||
state.sessions.insert(attached.session_id, attached);
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
|
|
@ -407,7 +243,10 @@ where
|
|||
pub fn wait_for_domain(&mut self, session: Session) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::WaitForDomain,
|
||||
matches!(self.phase, DomainPhase::Starting | DomainPhase::Stopping),
|
||||
matches!(
|
||||
self.state.phase,
|
||||
DomainPhase::Starting | DomainPhase::Stopping
|
||||
),
|
||||
"Domain is neither starting nor stopping",
|
||||
)?;
|
||||
Self::require(
|
||||
|
|
@ -421,7 +260,7 @@ where
|
|||
== self
|
||||
.current_generation(LifecycleAction::WaitForDomain)?
|
||||
.generation_id
|
||||
&& !self.sessions.contains_key(&session.session_id),
|
||||
&& !self.state.sessions.contains_key(&session.session_id),
|
||||
"session generation differs or Session ID is already known",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::WaitForDomain)?;
|
||||
|
|
@ -429,7 +268,6 @@ where
|
|||
LifecycleAction::WaitForDomain,
|
||||
session,
|
||||
SessionPhase::Waiting,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -442,7 +280,7 @@ where
|
|||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::RestartWaitingDomain,
|
||||
self.phase == DomainPhase::Absent,
|
||||
self.state.phase == DomainPhase::Absent,
|
||||
"Domain is not absent",
|
||||
)?;
|
||||
Self::require(
|
||||
|
|
@ -469,6 +307,7 @@ where
|
|||
)?;
|
||||
self.execute(LifecycleAction::RestartWaitingDomain, &generation)?;
|
||||
let waiters = self
|
||||
.state
|
||||
.sessions
|
||||
.values()
|
||||
.filter(|candidate| candidate.session_phase == SessionPhase::Waiting)
|
||||
|
|
@ -489,16 +328,17 @@ where
|
|||
}));
|
||||
}
|
||||
self.store.apply(&mutations)?;
|
||||
self.phase = DomainPhase::Starting;
|
||||
self.bootstrap_owner = Some(session_id);
|
||||
let generation_id = generation.generation_id;
|
||||
self.generation = Some(generation);
|
||||
self.progress = RuntimeProgress::default();
|
||||
for mut waiter in waiters {
|
||||
waiter.generation_id = generation_id;
|
||||
self.sessions.insert(waiter.session_id, waiter);
|
||||
}
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::RestartWaitingDomain, move |state| {
|
||||
state.phase = DomainPhase::Starting;
|
||||
state.bootstrap_owner = Some(session_id);
|
||||
state.generation = Some(generation);
|
||||
state.progress = RuntimeProgress::default();
|
||||
for mut waiter in waiters {
|
||||
waiter.generation_id = generation_id;
|
||||
state.sessions.insert(waiter.session_id, waiter);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
|
|
@ -506,7 +346,7 @@ where
|
|||
pub fn create_trust_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::CreateTrustAnchor,
|
||||
!self.progress.has(RuntimeMilestone::TrustAnchor),
|
||||
!self.state.progress.has(RuntimeMilestone::TrustAnchor),
|
||||
|p| p.mark(RuntimeMilestone::TrustAnchor),
|
||||
)
|
||||
}
|
||||
|
|
@ -516,8 +356,8 @@ where
|
|||
pub fn start_runtime_supervisor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartRuntimeSupervisor,
|
||||
self.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
self.state.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& !self.state.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
|p| p.mark(RuntimeMilestone::RuntimeSupervisor),
|
||||
)
|
||||
}
|
||||
|
|
@ -527,8 +367,11 @@ where
|
|||
pub fn lease_prepared_lower(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::LeasePreparedLower,
|
||||
self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||||
self.state.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self
|
||||
.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::PreparedLowerLease),
|
||||
|p| p.mark(RuntimeMilestone::PreparedLowerLease),
|
||||
)
|
||||
}
|
||||
|
|
@ -538,8 +381,10 @@ where
|
|||
pub fn mount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::MountStateAnchor,
|
||||
self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.progress.has(RuntimeMilestone::StateAnchor),
|
||||
self.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.state.progress.has(RuntimeMilestone::StateAnchor),
|
||||
|p| p.mark(RuntimeMilestone::StateAnchor),
|
||||
)
|
||||
}
|
||||
|
|
@ -549,8 +394,8 @@ where
|
|||
pub fn start_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartSidecar,
|
||||
self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::Sidecar),
|
||||
self.state.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.state.progress.has(RuntimeMilestone::Sidecar),
|
||||
|p| p.mark(RuntimeMilestone::Sidecar),
|
||||
)
|
||||
}
|
||||
|
|
@ -560,8 +405,11 @@ where
|
|||
pub fn start_connectivity_helper(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::StartConnectivityHelper,
|
||||
self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self.progress.has(RuntimeMilestone::ConnectivityHelper),
|
||||
self.state.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self
|
||||
.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::ConnectivityHelper),
|
||||
|p| p.mark(RuntimeMilestone::ConnectivityHelper),
|
||||
)
|
||||
}
|
||||
|
|
@ -571,8 +419,10 @@ where
|
|||
pub fn mark_network_ready(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::MarkNetworkReady,
|
||||
self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& !self.progress.has(RuntimeMilestone::NetworkReady),
|
||||
self.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& !self.state.progress.has(RuntimeMilestone::NetworkReady),
|
||||
|p| p.mark(RuntimeMilestone::NetworkReady),
|
||||
)
|
||||
}
|
||||
|
|
@ -582,8 +432,8 @@ where
|
|||
pub fn arm_runtime_causal_edge(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.startup_action(
|
||||
LifecycleAction::ArmRuntimeCausalEdge,
|
||||
self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.progress.has(RuntimeMilestone::CausalEdge),
|
||||
self.state.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.state.progress.has(RuntimeMilestone::CausalEdge),
|
||||
|p| p.mark(RuntimeMilestone::CausalEdge),
|
||||
)
|
||||
}
|
||||
|
|
@ -591,12 +441,12 @@ where
|
|||
/// # Errors
|
||||
/// Refuses launch before network readiness and the causal edge are established.
|
||||
pub fn start_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||||
let allowed = self.state.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.state.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& !self.state.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::StartSandbox,
|
||||
self.phase == DomainPhase::Starting && allowed,
|
||||
self.state.phase == DomainPhase::Starting && allowed,
|
||||
"runtime prerequisites are incomplete",
|
||||
)?;
|
||||
let mut generation = self
|
||||
|
|
@ -605,25 +455,35 @@ where
|
|||
generation.durable_decision = DurableDecision::Preparing;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation.clone());
|
||||
self.transition(LifecycleAction::StartSandbox, {
|
||||
let generation = generation.clone();
|
||||
move |state| state.generation = Some(generation)
|
||||
})?;
|
||||
self.execute(LifecycleAction::StartSandbox, &generation)?;
|
||||
self.progress.mark(RuntimeMilestone::Sandbox);
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::StartSandbox, |state| {
|
||||
state.progress.mark(RuntimeMilestone::Sandbox);
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses publication until every ordered startup prerequisite is complete.
|
||||
pub fn publish_domain(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.phase == DomainPhase::Starting
|
||||
&& self.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self.progress.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& self.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& self.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.progress.has(RuntimeMilestone::Sandbox);
|
||||
let allowed = self.state.phase == DomainPhase::Starting
|
||||
&& self.state.progress.has(RuntimeMilestone::TrustAnchor)
|
||||
&& self.state.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& self
|
||||
.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.state.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self.state.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self
|
||||
.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::ConnectivityHelper)
|
||||
&& self.state.progress.has(RuntimeMilestone::NetworkReady)
|
||||
&& self.state.progress.has(RuntimeMilestone::CausalEdge)
|
||||
&& self.state.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::PublishDomain,
|
||||
allowed,
|
||||
|
|
@ -636,20 +496,24 @@ where
|
|||
generation.durable_decision = DurableDecision::Published;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation.clone());
|
||||
if let Err(error) = self
|
||||
.adapter
|
||||
.after_commit(LifecycleAction::PublishDomain, &generation)
|
||||
{
|
||||
self.phase = DomainPhase::Failed;
|
||||
self.bootstrap_owner = None;
|
||||
self.published = false;
|
||||
self.transition(LifecycleAction::PublishDomain, move |state| {
|
||||
state.generation = Some(generation);
|
||||
state.phase = DomainPhase::Failed;
|
||||
state.bootstrap_owner = None;
|
||||
state.published = false;
|
||||
})?;
|
||||
return Err(LifecycleError::Adapter(error));
|
||||
}
|
||||
self.phase = DomainPhase::Ready;
|
||||
self.bootstrap_owner = None;
|
||||
self.published = true;
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::PublishDomain, move |state| {
|
||||
state.generation = Some(generation);
|
||||
state.phase = DomainPhase::Ready;
|
||||
state.bootstrap_owner = None;
|
||||
state.published = true;
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
|
|
@ -661,7 +525,7 @@ where
|
|||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
self.phase == DomainPhase::Ready && session.session_phase == SessionPhase::New,
|
||||
self.state.phase == DomainPhase::Ready && session.session_phase == SessionPhase::New,
|
||||
"session is not eligible for mismatch refusal",
|
||||
)?;
|
||||
let generation = self.current_generation(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
|
|
@ -673,7 +537,7 @@ where
|
|||
)?;
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
!self.sessions.contains_key(&session.session_id),
|
||||
!self.state.sessions.contains_key(&session.session_id),
|
||||
"Session ID is already known",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::RejectConfigurationMismatch)?;
|
||||
|
|
@ -683,8 +547,9 @@ where
|
|||
DomainStoreMutation::PutSession(new),
|
||||
DomainStoreMutation::PutSession(session.clone()),
|
||||
])?;
|
||||
self.sessions.insert(session.session_id, session);
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::RejectConfigurationMismatch, move |state| {
|
||||
state.sessions.insert(session.session_id, session);
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
|
|
@ -699,7 +564,7 @@ where
|
|||
.clone();
|
||||
Self::require(
|
||||
LifecycleAction::RejectConfigurationMismatch,
|
||||
self.phase == DomainPhase::Ready
|
||||
self.state.phase == DomainPhase::Ready
|
||||
&& matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::New | SessionPhase::Waiting
|
||||
|
|
@ -716,10 +581,9 @@ where
|
|||
session.session_phase = SessionPhase::Failed;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
self.connected.remove(&session_id);
|
||||
self.registered.remove(&session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::RejectConfigurationMismatch, move |state| {
|
||||
state.sessions.insert(session_id, session);
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
|
|
@ -743,38 +607,25 @@ where
|
|||
),
|
||||
"session is not live",
|
||||
)?;
|
||||
let leader_lost =
|
||||
self.phase == DomainPhase::Starting && self.bootstrap_owner == Some(session_id);
|
||||
let leader_lost = self.state.phase == DomainPhase::Starting
|
||||
&& self.state.bootstrap_owner == Some(session_id);
|
||||
self.session_adapter
|
||||
.disconnect_session(self.store, &session)
|
||||
.map_err(LifecycleError::Adapter)?;
|
||||
session.session_phase = SessionPhase::Closed;
|
||||
self.connected.remove(&session_id);
|
||||
self.registered.remove(&session_id);
|
||||
self.sessions.insert(session_id, session);
|
||||
if leader_lost {
|
||||
self.phase = DomainPhase::Failed;
|
||||
self.bootstrap_owner = None;
|
||||
self.published = false;
|
||||
}
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::DisconnectSession, move |state| {
|
||||
state.sessions.insert(session_id, session);
|
||||
if leader_lost {
|
||||
state.phase = DomainPhase::Failed;
|
||||
state.bootstrap_owner = None;
|
||||
state.published = false;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn is_teardown_ready(&self) -> bool {
|
||||
self.phase == DomainPhase::Ready
|
||||
&& self.registered.is_empty()
|
||||
&& self.connected.is_empty()
|
||||
&& self.sessions.values().all(|session| {
|
||||
!matches!(
|
||||
session.session_phase,
|
||||
SessionPhase::Waiting
|
||||
| SessionPhase::Registering
|
||||
| SessionPhase::Attached
|
||||
| SessionPhase::Prompting
|
||||
| SessionPhase::HostExecuting
|
||||
)
|
||||
})
|
||||
self.state.phase == DomainPhase::Ready && !self.state.has_session_interest()
|
||||
}
|
||||
|
||||
/// Performs the complete ordered normal teardown when the final session has disconnected.
|
||||
|
|
@ -806,7 +657,7 @@ where
|
|||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::DisconnectSession,
|
||||
self.phase == DomainPhase::Ready && self.published,
|
||||
self.state.phase == DomainPhase::Ready && self.state.published,
|
||||
"Domain is not published",
|
||||
)?;
|
||||
let phase = self
|
||||
|
|
@ -828,9 +679,10 @@ where
|
|||
"sessions still have interest",
|
||||
)?;
|
||||
self.execute_current(LifecycleAction::BeginTeardown)?;
|
||||
self.phase = DomainPhase::Stopping;
|
||||
self.published = false;
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::BeginTeardown, |state| {
|
||||
state.phase = DomainPhase::Stopping;
|
||||
state.published = false;
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
|
|
@ -838,7 +690,7 @@ where
|
|||
pub fn stop_sandbox(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::StopSandbox,
|
||||
self.progress.has(RuntimeMilestone::Sandbox),
|
||||
self.state.progress.has(RuntimeMilestone::Sandbox),
|
||||
|p| {
|
||||
p.clear(RuntimeMilestone::Sandbox);
|
||||
p.clear(RuntimeMilestone::CausalEdge);
|
||||
|
|
@ -851,8 +703,8 @@ where
|
|||
pub fn stop_sidecar(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::StopSidecar,
|
||||
!self.progress.has(RuntimeMilestone::Sandbox)
|
||||
&& self.progress.has(RuntimeMilestone::Sidecar),
|
||||
!self.state.progress.has(RuntimeMilestone::Sandbox)
|
||||
&& self.state.progress.has(RuntimeMilestone::Sidecar),
|
||||
|p| {
|
||||
p.clear(RuntimeMilestone::Sidecar);
|
||||
p.clear(RuntimeMilestone::ConnectivityHelper);
|
||||
|
|
@ -866,8 +718,8 @@ where
|
|||
pub fn unmount_state_anchor(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::UnmountStateAnchor,
|
||||
!self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self.progress.has(RuntimeMilestone::StateAnchor),
|
||||
!self.state.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& self.state.progress.has(RuntimeMilestone::StateAnchor),
|
||||
|p| p.clear(RuntimeMilestone::StateAnchor),
|
||||
)
|
||||
}
|
||||
|
|
@ -877,8 +729,11 @@ where
|
|||
pub fn release_prepared_lower_lease(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::ReleasePreparedLowerLease,
|
||||
!self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self.progress.has(RuntimeMilestone::PreparedLowerLease),
|
||||
!self.state.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& self
|
||||
.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::PreparedLowerLease),
|
||||
|p| p.clear(RuntimeMilestone::PreparedLowerLease),
|
||||
)
|
||||
}
|
||||
|
|
@ -888,8 +743,11 @@ where
|
|||
pub fn graceful_supervisor_exit(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.teardown_action(
|
||||
LifecycleAction::GracefulSupervisorExit,
|
||||
!self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
!self
|
||||
.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& self.state.progress.has(RuntimeMilestone::RuntimeSupervisor),
|
||||
|p| p.clear(RuntimeMilestone::RuntimeSupervisor),
|
||||
)
|
||||
}
|
||||
|
|
@ -897,12 +755,15 @@ where
|
|||
/// # Errors
|
||||
/// Refuses while runtime resources remain or the final durable update fails.
|
||||
pub fn finish_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
let allowed = self.phase == DomainPhase::Stopping
|
||||
&& !self.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self.progress.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self.progress.has(RuntimeMilestone::Sandbox);
|
||||
let allowed = self.state.phase == DomainPhase::Stopping
|
||||
&& !self.state.progress.has(RuntimeMilestone::RuntimeSupervisor)
|
||||
&& !self
|
||||
.state
|
||||
.progress
|
||||
.has(RuntimeMilestone::PreparedLowerLease)
|
||||
&& !self.state.progress.has(RuntimeMilestone::StateAnchor)
|
||||
&& !self.state.progress.has(RuntimeMilestone::Sidecar)
|
||||
&& !self.state.progress.has(RuntimeMilestone::Sandbox);
|
||||
Self::require(
|
||||
LifecycleAction::FinishTeardown,
|
||||
allowed,
|
||||
|
|
@ -915,11 +776,12 @@ where
|
|||
generation.durable_decision = DurableDecision::Clean;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])?;
|
||||
self.generation = Some(generation);
|
||||
self.phase = DomainPhase::Absent;
|
||||
self.progress = RuntimeProgress::default();
|
||||
self.bootstrap_owner = None;
|
||||
Ok(())
|
||||
self.transition(LifecycleAction::FinishTeardown, move |state| {
|
||||
state.generation = Some(generation);
|
||||
state.phase = DomainPhase::Absent;
|
||||
state.progress = RuntimeProgress::default();
|
||||
state.bootstrap_owner = None;
|
||||
})
|
||||
}
|
||||
|
||||
pub fn into_adapters(self) -> (A, S) {
|
||||
|
|
@ -934,12 +796,11 @@ where
|
|||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.phase == DomainPhase::Starting && allowed,
|
||||
self.state.phase == DomainPhase::Starting && allowed,
|
||||
"startup ordering refused",
|
||||
)?;
|
||||
self.execute_current(action)?;
|
||||
update(&mut self.progress);
|
||||
Ok(())
|
||||
self.transition(action, |state| update(&mut state.progress))
|
||||
}
|
||||
|
||||
fn teardown_action(
|
||||
|
|
@ -950,12 +811,11 @@ where
|
|||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
self.phase == DomainPhase::Stopping && allowed,
|
||||
self.state.phase == DomainPhase::Stopping && allowed,
|
||||
"teardown ordering refused",
|
||||
)?;
|
||||
self.execute_current(action)?;
|
||||
update(&mut self.progress);
|
||||
Ok(())
|
||||
self.transition(action, |state| update(&mut state.progress))
|
||||
}
|
||||
|
||||
fn put_new_session(
|
||||
|
|
@ -963,7 +823,6 @@ where
|
|||
action: LifecycleAction,
|
||||
mut session: Session,
|
||||
phase: SessionPhase,
|
||||
connected: bool,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
action,
|
||||
|
|
@ -972,17 +831,15 @@ where
|
|||
)?;
|
||||
Self::require(
|
||||
action,
|
||||
!self.sessions.contains_key(&session.session_id),
|
||||
!self.state.sessions.contains_key(&session.session_id),
|
||||
"Session ID is already known",
|
||||
)?;
|
||||
session.session_phase = phase;
|
||||
self.store
|
||||
.apply(&[DomainStoreMutation::PutSession(session.clone())])?;
|
||||
if connected {
|
||||
self.connected.insert(session.session_id);
|
||||
}
|
||||
self.sessions.insert(session.session_id, session);
|
||||
Ok(())
|
||||
self.transition(action, move |state| {
|
||||
state.sessions.insert(session.session_id, session);
|
||||
})
|
||||
}
|
||||
|
||||
fn require_new_generation(
|
||||
|
|
@ -1010,14 +867,15 @@ where
|
|||
}
|
||||
|
||||
fn admission_ready(&self) -> bool {
|
||||
self.phase == DomainPhase::Ready && self.published
|
||||
self.state.phase == DomainPhase::Ready && self.state.published
|
||||
}
|
||||
|
||||
fn current_generation(
|
||||
&self,
|
||||
action: LifecycleAction,
|
||||
) -> Result<&Generation, LifecycleError<A::Error>> {
|
||||
self.generation
|
||||
self.state
|
||||
.generation
|
||||
.as_ref()
|
||||
.ok_or(LifecycleError::InvalidTransition {
|
||||
action,
|
||||
|
|
@ -1030,7 +888,8 @@ where
|
|||
session_id: SessionId,
|
||||
action: LifecycleAction,
|
||||
) -> Result<&Session, LifecycleError<A::Error>> {
|
||||
self.sessions
|
||||
self.state
|
||||
.sessions
|
||||
.get(&session_id)
|
||||
.ok_or(LifecycleError::InvalidTransition {
|
||||
action,
|
||||
|
|
@ -1053,6 +912,22 @@ where
|
|||
.map_err(LifecycleError::Adapter)
|
||||
}
|
||||
|
||||
fn transition(
|
||||
&mut self,
|
||||
action: LifecycleAction,
|
||||
update: impl FnOnce(&mut LifecycleAggregate),
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
self.state
|
||||
.validate()
|
||||
.map_err(|reason| LifecycleError::InvalidAggregate { action, reason })?;
|
||||
let mut next = self.state.clone();
|
||||
update(&mut next);
|
||||
next.validate()
|
||||
.map_err(|reason| LifecycleError::InvalidAggregate { action, reason })?;
|
||||
self.state = next;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require(
|
||||
action: LifecycleAction,
|
||||
condition: bool,
|
||||
|
|
|
|||
Loading…
Reference in a new issue