use std::io; use std::os::fd::AsRawFd; use std::os::unix::ffi::OsStrExt; use std::path::Path; use std::time::{Duration, Instant}; use lazaret_core::configuration::{ConfinedPathType, ResolvedDomainConfiguration}; use lazaret_core::identity::{ ConfigurationFingerprint, DomainId, GenerationId, HostPath, MountIdentity, NamespaceIdentity, PidfsProcessIdentityV1, }; use lazaret_core::records::{ ArtifactLease, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole, ParticipantState, PersistedHostProcessOwnership, SessionPhase, }; use lazaret_linux::filesystem::{current_mount_identity, mount_identity}; use lazaret_linux::process::{ PidfsRecoveryMatch, RetainedProcess, TargetCommand, child_broker, match_recovery_process, }; use lazaret_prepared_lower::PreparedLowerStore; use lazaret_state_anchor::StateAnchorLayout; use lazaret_storage_catalog::{Catalog, CatalogError}; use rustix::fs::{AtFlags, Mode, OFlags, fstat, openat, statat, unlinkat}; use rustix::io::{FdFlags, fcntl_setfd}; use rustix::process::Signal; use crate::control::{ CONTROL_SOCKET_NAME, PUBLICATION_IDENTITY_NAME, STAGED_CONTROL_SOCKET_NAME, STAGED_PUBLICATION_IDENTITY_NAME, }; use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore}; use crate::runtime::StateAnchorAbsenceObservation; use crate::state_anchor::{ StateAnchorCoordinator, StateAnchorCoordinatorError, StateAnchorRecoveryOwner, }; use crate::state_maintenance::recover_state_reset; const PROCESS_EXIT_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum RuntimeProcessRole { HostCoordinator, RuntimeSupervisor, AttachedRunsc, ConnectivityHelper, StateAnchorDaemon, GitRoutingDaemon, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct RuntimeProcessOwner { role: RuntimeProcessRole, identity: PidfsProcessIdentityV1, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct RuntimeRecoveryOwnership { domain_id: DomainId, generation_id: GenerationId, configuration_fingerprint: ConfigurationFingerprint, participants_present: usize, processes: Vec, state_anchor_user_namespace: Option, state_anchor_mount_namespace: Option, state_anchor_mount: Option, state_anchor_path: Option, git_routing_mount: Option, git_routing_path: Option, } impl RuntimeRecoveryOwnership { fn process(&self, role: RuntimeProcessRole) -> Option { self.processes .iter() .find(|owner| owner.role == role) .map(|owner| owner.identity) } pub(crate) fn binding_matches(&self, domain_id: DomainId, generation: &Generation) -> bool { self.domain_id == domain_id && self.generation_id == generation.generation_id && self.configuration_fingerprint == generation.configuration_fingerprint } pub(crate) fn state_anchor_matches( &self, daemon: PidfsProcessIdentityV1, user_namespace: NamespaceIdentity, mount_namespace: NamespaceIdentity, mount: MountIdentity, ) -> bool { self.process(RuntimeProcessRole::StateAnchorDaemon) == Some(daemon) && self.state_anchor_user_namespace == Some(user_namespace) && self.state_anchor_mount_namespace == Some(mount_namespace) && self.state_anchor_mount == Some(mount) } } #[derive(Clone, Debug, Eq, PartialEq)] pub struct RuntimeOwnersAbsent { ownership: RuntimeRecoveryOwnership, } enum SealedProcessMatch { BootMismatch, Esrch, Matched(RetainedProcess), } struct SealedProcessOwner { role: RuntimeProcessRole, matched: SealedProcessMatch, reconciled: bool, } struct SealedRuntimeOwners { ownership: RuntimeRecoveryOwnership, processes: Vec, } impl SealedRuntimeOwners { fn process(&self, role: RuntimeProcessRole) -> Option<&SealedProcessMatch> { self.processes .iter() .find(|owner| owner.role == role) .map(|owner| &owner.matched) } fn reconcile(&mut self, role: RuntimeProcessRole) -> io::Result<()> { let Some(owner) = self.processes.iter_mut().find(|owner| owner.role == role) else { return Ok(()); }; if owner.reconciled { return Ok(()); } match &owner.matched { SealedProcessMatch::BootMismatch | SealedProcessMatch::Esrch => {} SealedProcessMatch::Matched(process) if process.has_exited()? => {} SealedProcessMatch::Matched(_) if role == RuntimeProcessRole::HostCoordinator => { return Err(io::Error::other( "recorded Host Coordinator remains present", )); } SealedProcessMatch::Matched(process) => { process.send_signal(Signal::KILL)?; let deadline = Instant::now() + PROCESS_EXIT_TIMEOUT; while !process.has_exited()? { if Instant::now() >= deadline { return Err(io::Error::other( "exact runtime owner did not exit after pidfd action", )); } std::thread::yield_now(); } } } owner.reconciled = true; Ok(()) } fn prove_absent(&mut self) -> Result { for role in [ RuntimeProcessRole::AttachedRunsc, RuntimeProcessRole::ConnectivityHelper, RuntimeProcessRole::GitRoutingDaemon, RuntimeProcessRole::StateAnchorDaemon, RuntimeProcessRole::RuntimeSupervisor, RuntimeProcessRole::HostCoordinator, ] { self.reconcile(role)?; } require_mount_absent( self.ownership.state_anchor_path.as_ref(), self.ownership.state_anchor_mount, "State Anchor", )?; require_mount_absent( self.ownership.git_routing_path.as_ref(), self.ownership.git_routing_mount, "Git-routing", )?; Ok(RuntimeOwnersAbsent { ownership: self.ownership.clone(), }) } } impl RuntimeOwnersAbsent { pub(crate) fn ownership(&self) -> &RuntimeRecoveryOwnership { &self.ownership } } /// Extracts one generation's complete exact process and private-mount recovery authority. /// /// # Errors /// Refuses duplicate roles, missing role dependencies, unequal owner copies, and every incomplete /// process, namespace, mount, or path relation required by a recorded participant phase. #[allow(clippy::too_many_lines)] pub fn runtime_recovery_ownership( participants: &[Participant], generation: &Generation, ) -> Result { let mut ownership = RuntimeRecoveryOwnership { domain_id: generation.domain_id, generation_id: generation.generation_id, configuration_fingerprint: generation.configuration_fingerprint, participants_present: 0, processes: Vec::new(), state_anchor_user_namespace: None, state_anchor_mount_namespace: None, state_anchor_mount: None, state_anchor_path: None, git_routing_mount: None, git_routing_path: None, }; let mut states = [None; 7]; for participant in participants .iter() .filter(|participant| participant.generation_id() == generation.generation_id) { let index = role_index(participant.role()); if states[index].replace(participant.state()).is_some() { return Err("recovery participant role is duplicated"); } ownership.participants_present += 1; let Some(recorded) = participant.ownership() else { if participant.state() != ParticipantState::Intent { return Err("non-Intent recovery participant has no ownership"); } continue; }; match recorded { ParticipantOwnership::Sandbox(sandbox) => { merge_process( &mut ownership, RuntimeProcessRole::HostCoordinator, &sandbox.host_coordinator, "Sandbox HC identity mismatch", )?; merge_process( &mut ownership, RuntimeProcessRole::RuntimeSupervisor, &sandbox.runtime_supervisor, "Sandbox RS identity mismatch", )?; merge_process( &mut ownership, RuntimeProcessRole::AttachedRunsc, &sandbox.attached_runsc, "attached runsc identity mismatch", )?; } ParticipantOwnership::Network(network) => merge_process( &mut ownership, RuntimeProcessRole::RuntimeSupervisor, &network.runtime_supervisor, "Network RS identity mismatch", )?, ParticipantOwnership::Helper(helper) => { merge_process( &mut ownership, RuntimeProcessRole::ConnectivityHelper, &helper.connectivity_helper, "connectivity-helper identity mismatch", )?; merge_process( &mut ownership, RuntimeProcessRole::RuntimeSupervisor, &helper.runtime_supervisor, "Helper RS identity mismatch", )?; } ParticipantOwnership::StateAnchor(anchor) => { merge_process( &mut ownership, RuntimeProcessRole::StateAnchorDaemon, &anchor.daemon, "State Anchor daemon identity mismatch", )?; let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount); if evidence != (None, None, None) { let (Some(user), Some(mount_namespace), Some(mount)) = evidence else { return Err("State Anchor recovery evidence is incomplete"); }; if ownership .process(RuntimeProcessRole::StateAnchorDaemon) .is_none() { return Err("State Anchor mount has no daemon authority"); } ownership.state_anchor_user_namespace = Some(user); ownership.state_anchor_mount_namespace = Some(mount_namespace); ownership.state_anchor_mount = Some(mount); ownership.state_anchor_path.clone_from(&anchor.merged_path); if ownership.state_anchor_path.is_none() { return Err("State Anchor mount path is absent"); } } } ParticipantOwnership::GitRouting(routing) => { merge_process( &mut ownership, RuntimeProcessRole::GitRoutingDaemon, &routing.daemon, "Git-routing daemon identity mismatch", )?; if let Some(mount) = routing.mount { if ownership .process(RuntimeProcessRole::GitRoutingDaemon) .is_none() { return Err("Git-routing mount has no daemon authority"); } ownership.git_routing_mount = Some(mount); ownership.git_routing_path.clone_from(&routing.mountpoint); if ownership.git_routing_path.is_none() { return Err("Git-routing mount path is absent"); } } } ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {} } } require_role_dependencies(states)?; require_phase_topology(participants, generation.generation_id, &ownership, states)?; Ok(ownership) } fn merge_process( ownership: &mut RuntimeRecoveryOwnership, role: RuntimeProcessRole, observed: &PersistedHostProcessOwnership, mismatch: &'static str, ) -> Result<(), &'static str> { let Ok(identity) = observed.authoritative_pidfs() else { return Ok(()); }; match ownership.process(role) { None => { if ownership .processes .iter() .any(|owner| owner.identity == *identity) { return Err("one process identity is assigned to distinct runtime roles"); } ownership.processes.push(RuntimeProcessOwner { role, identity: *identity, }); } Some(current) if current == *identity => {} Some(_) => return Err(mismatch), } Ok(()) } fn require_role_dependencies(states: [Option; 7]) -> Result<(), &'static str> { for (role, required) in [ ( ParticipantRole::StateAnchor, &[ParticipantRole::PreparedLower, ParticipantRole::Sandbox][..], ), ( ParticipantRole::Helper, &[ParticipantRole::PreparedLower, ParticipantRole::StateAnchor], ), ( ParticipantRole::Network, &[ ParticipantRole::PreparedLower, ParticipantRole::StateAnchor, ParticipantRole::Helper, ], ), ( ParticipantRole::Publication, &[ ParticipantRole::PreparedLower, ParticipantRole::StateAnchor, ParticipantRole::Helper, ParticipantRole::Network, ParticipantRole::Sandbox, ], ), ] { if states[role_index(role)].is_some() && required .iter() .any(|required| states[role_index(*required)].is_none()) { return Err("recovery participant dependency is missing"); } } Ok(()) } fn require_phase_topology( participants: &[Participant], generation_id: GenerationId, ownership: &RuntimeRecoveryOwnership, states: [Option; 7], ) -> Result<(), &'static str> { let sandbox = participants.iter().find(|participant| { participant.generation_id() == generation_id && participant.role() == ParticipantRole::Sandbox }); if let Some(participant) = sandbox { let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else { return Err("Sandbox durable intent lacks its immutable launch authority"); }; if participant.state() != ParticipantState::Clean && (sandbox.container_id.is_none() || sandbox.runtime_root.is_none() || sandbox.runtime_root_identity.is_none() || sandbox.bundle_digest.is_none() || sandbox.host_coordinator.authoritative_pidfs().is_err()) { return Err("Sandbox durable launch authority is incomplete"); } if participant.state() == ParticipantState::Committed && (sandbox.runtime_supervisor.authoritative_pidfs().is_err() || sandbox.attached_runsc.authoritative_pidfs().is_err()) { return Err("committed Sandbox process authority is incomplete"); } } let later_runtime_role = [ ParticipantRole::StateAnchor, ParticipantRole::Helper, ParticipantRole::Network, ParticipantRole::GitRouting, ParticipantRole::Publication, ] .into_iter() .any(|role| states[role_index(role)].is_some()); let sandbox_clean = states[role_index(ParticipantRole::Sandbox)] == Some(ParticipantState::Clean); if later_runtime_role && !sandbox_clean && ownership .process(RuntimeProcessRole::RuntimeSupervisor) .is_none() { return Err("later runtime phase lacks its Runtime Supervisor authority"); } if states[role_index(ParticipantRole::StateAnchor)].is_some() && !sandbox_clean && ownership .process(RuntimeProcessRole::HostCoordinator) .is_none() { return Err("State Anchor phase lacks generation Host Coordinator authority"); } Ok(()) } const fn role_index(role: ParticipantRole) -> usize { match role { ParticipantRole::PreparedLower => 0, ParticipantRole::StateAnchor => 1, ParticipantRole::GitRouting => 2, ParticipantRole::Network => 3, ParticipantRole::Helper => 4, ParticipantRole::Sandbox => 5, ParticipantRole::Publication => 6, } } include!("recovery/reconciliation.rs"); include!("recovery/tests.rs");