use std::io; use std::os::fd::AsRawFd; use std::os::unix::ffi::OsStrExt; use std::path::Path; use std::time::{Duration, Instant}; use lazaret_core::configuration::{ConfinedPathType, ResolvedDomainConfiguration}; use lazaret_core::identity::{ ConfigurationFingerprint, DomainId, GenerationId, HostPath, MountIdentity, NamespaceIdentity, PidfsProcessIdentityV1, }; use lazaret_core::records::{ ArtifactLease, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole, ParticipantState, PersistedHostProcessOwnership, SessionPhase, }; use lazaret_linux::filesystem::{current_mount_identity, mount_identity}; use lazaret_linux::process::{ PidfsRecoveryMatch, RetainedProcess, TargetCommand, child_broker, match_recovery_process, }; use lazaret_prepared_lower::PreparedLowerStore; use lazaret_state_anchor::StateAnchorLayout; use lazaret_storage_catalog::{Catalog, CatalogError}; use rustix::fs::{AtFlags, Mode, OFlags, fstat, openat, statat, unlinkat}; use rustix::io::{FdFlags, fcntl_setfd}; use rustix::process::Signal; use crate::control::{ CONTROL_SOCKET_NAME, PUBLICATION_IDENTITY_NAME, STAGED_CONTROL_SOCKET_NAME, STAGED_PUBLICATION_IDENTITY_NAME, }; use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore}; use crate::runtime::StateAnchorAbsenceObservation; use crate::state_anchor::{ StateAnchorCoordinator, StateAnchorCoordinatorError, StateAnchorRecoveryOwner, }; const PROCESS_EXIT_TIMEOUT: Duration = Duration::from_secs(5); #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum RuntimeProcessRole { HostCoordinator, RuntimeSupervisor, AttachedRunsc, ConnectivityHelper, StateAnchorDaemon, GitRoutingDaemon, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct RuntimeProcessOwner { role: RuntimeProcessRole, identity: PidfsProcessIdentityV1, } #[derive(Clone, Debug, Eq, PartialEq)] pub struct RuntimeRecoveryOwnership { domain_id: DomainId, generation_id: GenerationId, configuration_fingerprint: ConfigurationFingerprint, participants_present: usize, processes: Vec, state_anchor_user_namespace: Option, state_anchor_mount_namespace: Option, state_anchor_mount: Option, state_anchor_path: Option, git_routing_mount: Option, git_routing_path: Option, } impl RuntimeRecoveryOwnership { fn process(&self, role: RuntimeProcessRole) -> Option { self.processes .iter() .find(|owner| owner.role == role) .map(|owner| owner.identity) } pub(crate) fn binding_matches(&self, domain_id: DomainId, generation: &Generation) -> bool { self.domain_id == domain_id && self.generation_id == generation.generation_id && self.configuration_fingerprint == generation.configuration_fingerprint } pub(crate) fn state_anchor_matches( &self, daemon: PidfsProcessIdentityV1, user_namespace: NamespaceIdentity, mount_namespace: NamespaceIdentity, mount: MountIdentity, ) -> bool { self.process(RuntimeProcessRole::StateAnchorDaemon) == Some(daemon) && self.state_anchor_user_namespace == Some(user_namespace) && self.state_anchor_mount_namespace == Some(mount_namespace) && self.state_anchor_mount == Some(mount) } } #[derive(Clone, Debug, Eq, PartialEq)] pub struct RuntimeOwnersAbsent { ownership: RuntimeRecoveryOwnership, } enum SealedProcessMatch { BootMismatch, Esrch, Matched(RetainedProcess), } struct SealedProcessOwner { role: RuntimeProcessRole, matched: SealedProcessMatch, reconciled: bool, } struct SealedRuntimeOwners { ownership: RuntimeRecoveryOwnership, processes: Vec, } impl SealedRuntimeOwners { fn process(&self, role: RuntimeProcessRole) -> Option<&SealedProcessMatch> { self.processes .iter() .find(|owner| owner.role == role) .map(|owner| &owner.matched) } fn reconcile(&mut self, role: RuntimeProcessRole) -> io::Result<()> { let Some(owner) = self.processes.iter_mut().find(|owner| owner.role == role) else { return Ok(()); }; if owner.reconciled { return Ok(()); } match &owner.matched { SealedProcessMatch::BootMismatch | SealedProcessMatch::Esrch => {} SealedProcessMatch::Matched(process) if process.has_exited()? => {} SealedProcessMatch::Matched(_) if role == RuntimeProcessRole::HostCoordinator => { return Err(io::Error::other( "recorded Host Coordinator remains present", )); } SealedProcessMatch::Matched(process) => { process.send_signal(Signal::KILL)?; let deadline = Instant::now() + PROCESS_EXIT_TIMEOUT; while !process.has_exited()? { if Instant::now() >= deadline { return Err(io::Error::other( "exact runtime owner did not exit after pidfd action", )); } std::thread::yield_now(); } } } owner.reconciled = true; Ok(()) } fn prove_absent(&mut self) -> Result { for role in [ RuntimeProcessRole::AttachedRunsc, RuntimeProcessRole::ConnectivityHelper, RuntimeProcessRole::GitRoutingDaemon, RuntimeProcessRole::StateAnchorDaemon, RuntimeProcessRole::RuntimeSupervisor, RuntimeProcessRole::HostCoordinator, ] { self.reconcile(role)?; } require_mount_absent( self.ownership.state_anchor_path.as_ref(), self.ownership.state_anchor_mount, "State Anchor", )?; require_mount_absent( self.ownership.git_routing_path.as_ref(), self.ownership.git_routing_mount, "Git-routing", )?; Ok(RuntimeOwnersAbsent { ownership: self.ownership.clone(), }) } } impl RuntimeOwnersAbsent { pub(crate) fn ownership(&self) -> &RuntimeRecoveryOwnership { &self.ownership } } /// Extracts one generation's complete exact process and private-mount recovery authority. /// /// # Errors /// Refuses duplicate roles, missing role dependencies, unequal owner copies, and every incomplete /// process, namespace, mount, or path relation required by a recorded participant phase. #[allow(clippy::too_many_lines)] pub fn runtime_recovery_ownership( participants: &[Participant], generation: &Generation, ) -> Result { let mut ownership = RuntimeRecoveryOwnership { domain_id: generation.domain_id, generation_id: generation.generation_id, configuration_fingerprint: generation.configuration_fingerprint, participants_present: 0, processes: Vec::new(), state_anchor_user_namespace: None, state_anchor_mount_namespace: None, state_anchor_mount: None, state_anchor_path: None, git_routing_mount: None, git_routing_path: None, }; let mut states = [None; 7]; for participant in participants .iter() .filter(|participant| participant.generation_id() == generation.generation_id) { let index = role_index(participant.role()); if states[index].replace(participant.state()).is_some() { return Err("recovery participant role is duplicated"); } ownership.participants_present += 1; let Some(recorded) = participant.ownership() else { if participant.state() != ParticipantState::Intent { return Err("non-Intent recovery participant has no ownership"); } continue; }; match recorded { ParticipantOwnership::Sandbox(sandbox) => { merge_process( &mut ownership, RuntimeProcessRole::HostCoordinator, &sandbox.host_coordinator, "Sandbox HC identity mismatch", )?; merge_process( &mut ownership, RuntimeProcessRole::RuntimeSupervisor, &sandbox.runtime_supervisor, "Sandbox RS identity mismatch", )?; merge_process( &mut ownership, RuntimeProcessRole::AttachedRunsc, &sandbox.attached_runsc, "attached runsc identity mismatch", )?; } ParticipantOwnership::Network(network) => merge_process( &mut ownership, RuntimeProcessRole::RuntimeSupervisor, &network.runtime_supervisor, "Network RS identity mismatch", )?, ParticipantOwnership::Helper(helper) => { merge_process( &mut ownership, RuntimeProcessRole::ConnectivityHelper, &helper.connectivity_helper, "connectivity-helper identity mismatch", )?; merge_process( &mut ownership, RuntimeProcessRole::RuntimeSupervisor, &helper.runtime_supervisor, "Helper RS identity mismatch", )?; } ParticipantOwnership::StateAnchor(anchor) => { merge_process( &mut ownership, RuntimeProcessRole::StateAnchorDaemon, &anchor.daemon, "State Anchor daemon identity mismatch", )?; let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount); if evidence != (None, None, None) { let (Some(user), Some(mount_namespace), Some(mount)) = evidence else { return Err("State Anchor recovery evidence is incomplete"); }; if ownership .process(RuntimeProcessRole::StateAnchorDaemon) .is_none() { return Err("State Anchor mount has no daemon authority"); } ownership.state_anchor_user_namespace = Some(user); ownership.state_anchor_mount_namespace = Some(mount_namespace); ownership.state_anchor_mount = Some(mount); ownership.state_anchor_path.clone_from(&anchor.merged_path); if ownership.state_anchor_path.is_none() { return Err("State Anchor mount path is absent"); } } } ParticipantOwnership::GitRouting(routing) => { merge_process( &mut ownership, RuntimeProcessRole::GitRoutingDaemon, &routing.daemon, "Git-routing daemon identity mismatch", )?; if let Some(mount) = routing.mount { if ownership .process(RuntimeProcessRole::GitRoutingDaemon) .is_none() { return Err("Git-routing mount has no daemon authority"); } ownership.git_routing_mount = Some(mount); ownership.git_routing_path.clone_from(&routing.mountpoint); if ownership.git_routing_path.is_none() { return Err("Git-routing mount path is absent"); } } } ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {} } } require_role_dependencies(states)?; require_phase_topology(participants, generation.generation_id, &ownership, states)?; Ok(ownership) } fn merge_process( ownership: &mut RuntimeRecoveryOwnership, role: RuntimeProcessRole, observed: &PersistedHostProcessOwnership, mismatch: &'static str, ) -> Result<(), &'static str> { let Ok(identity) = observed.authoritative_pidfs() else { return Ok(()); }; match ownership.process(role) { None => { if ownership .processes .iter() .any(|owner| owner.identity == *identity) { return Err("one process identity is assigned to distinct runtime roles"); } ownership.processes.push(RuntimeProcessOwner { role, identity: *identity, }); } Some(current) if current == *identity => {} Some(_) => return Err(mismatch), } Ok(()) } fn require_role_dependencies(states: [Option; 7]) -> Result<(), &'static str> { for (role, required) in [ ( ParticipantRole::StateAnchor, &[ParticipantRole::PreparedLower, ParticipantRole::Sandbox][..], ), ( ParticipantRole::Helper, &[ParticipantRole::PreparedLower, ParticipantRole::StateAnchor], ), ( ParticipantRole::Network, &[ ParticipantRole::PreparedLower, ParticipantRole::StateAnchor, ParticipantRole::Helper, ], ), ( ParticipantRole::Publication, &[ ParticipantRole::PreparedLower, ParticipantRole::StateAnchor, ParticipantRole::Helper, ParticipantRole::Network, ParticipantRole::Sandbox, ], ), ] { if states[role_index(role)].is_some() && required .iter() .any(|required| states[role_index(*required)].is_none()) { return Err("recovery participant dependency is missing"); } } Ok(()) } fn require_phase_topology( participants: &[Participant], generation_id: GenerationId, ownership: &RuntimeRecoveryOwnership, states: [Option; 7], ) -> Result<(), &'static str> { let sandbox = participants.iter().find(|participant| { participant.generation_id() == generation_id && participant.role() == ParticipantRole::Sandbox }); if let Some(participant) = sandbox { let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else { return Err("Sandbox durable intent lacks its immutable launch authority"); }; if participant.state() != ParticipantState::Clean && (sandbox.container_id.is_none() || sandbox.runtime_root.is_none() || sandbox.runtime_root_identity.is_none() || sandbox.bundle_digest.is_none() || sandbox.host_coordinator.authoritative_pidfs().is_err()) { return Err("Sandbox durable launch authority is incomplete"); } if participant.state() == ParticipantState::Committed && (sandbox.runtime_supervisor.authoritative_pidfs().is_err() || sandbox.attached_runsc.authoritative_pidfs().is_err()) { return Err("committed Sandbox process authority is incomplete"); } } let later_runtime_role = [ ParticipantRole::StateAnchor, ParticipantRole::Helper, ParticipantRole::Network, ParticipantRole::GitRouting, ParticipantRole::Publication, ] .into_iter() .any(|role| states[role_index(role)].is_some()); let sandbox_clean = states[role_index(ParticipantRole::Sandbox)] == Some(ParticipantState::Clean); if later_runtime_role && !sandbox_clean && ownership .process(RuntimeProcessRole::RuntimeSupervisor) .is_none() { return Err("later runtime phase lacks its Runtime Supervisor authority"); } if states[role_index(ParticipantRole::StateAnchor)].is_some() && !sandbox_clean && ownership .process(RuntimeProcessRole::HostCoordinator) .is_none() { return Err("State Anchor phase lacks generation Host Coordinator authority"); } Ok(()) } const fn role_index(role: ParticipantRole) -> usize { match role { ParticipantRole::PreparedLower => 0, ParticipantRole::StateAnchor => 1, ParticipantRole::GitRouting => 2, ParticipantRole::Network => 3, ParticipantRole::Helper => 4, ParticipantRole::Sandbox => 5, ParticipantRole::Publication => 6, } } #[must_use] pub fn recovery_role_order() -> [ParticipantRole; 7] { [ ParticipantRole::Publication, ParticipantRole::Sandbox, ParticipantRole::Helper, ParticipantRole::Network, ParticipantRole::GitRouting, ParticipantRole::StateAnchor, ParticipantRole::PreparedLower, ] } fn reconcile_runtime_before_state_anchor(owners: &mut SealedRuntimeOwners) -> io::Result<()> { for role in [ RuntimeProcessRole::AttachedRunsc, RuntimeProcessRole::ConnectivityHelper, RuntimeProcessRole::GitRoutingDaemon, RuntimeProcessRole::RuntimeSupervisor, ] { owners.reconcile(role)?; } Ok(()) } /// Proves every recorded generation runtime holder and filesystem mount absent. /// /// # Errors /// Refuses empty or partial inventory, live HC authority, exact-pidfd action failure, unreadable /// evidence, or any recorded State Anchor/Git-routing mount that remains present. pub fn prove_runtime_owners_absent( ownership: &RuntimeRecoveryOwnership, ) -> Result { if ownership.participants_present == 0 || (ownership.processes.is_empty() && ownership.state_anchor_mount.is_none() && ownership.git_routing_mount.is_none()) { return Err(io::Error::other("runtime recovery inventory is empty")); } seal_runtime_owners(ownership, match_recovery_process)?.prove_absent() } fn seal_runtime_owners( ownership: &RuntimeRecoveryOwnership, mut open: Open, ) -> io::Result where Open: FnMut(PidfsProcessIdentityV1) -> io::Result, { let mut processes = Vec::with_capacity(ownership.processes.len()); for owner in &ownership.processes { let matched = match open(owner.identity) { Ok(PidfsRecoveryMatch::BootMismatch) => SealedProcessMatch::BootMismatch, Ok(PidfsRecoveryMatch::Matched(process)) => SealedProcessMatch::Matched(process), Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => { SealedProcessMatch::Esrch } Err(error) => return Err(error), }; processes.push(SealedProcessOwner { role: owner.role, matched, reconciled: false, }); } Ok(SealedRuntimeOwners { ownership: ownership.clone(), processes, }) } fn require_mount_absent( path: Option<&HostPath>, mount: Option, label: &str, ) -> io::Result<()> { match (path, mount) { (None, None) => Ok(()), (Some(path), Some(expected)) => { let observed = current_mount_identity(Path::new(std::ffi::OsStr::from_bytes(path.as_bytes())))?; if observed == expected { Err(io::Error::other(format!( "{label} exact mount remains present" ))) } else { Ok(()) } } _ => Err(io::Error::other(format!( "{label} recovery mount evidence is incomplete" ))), } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ProductionRecoveryBoundary { Snapshot, Publication, Sandbox, HelperAndNetwork, GitRouting, StateAnchor, PreparedLower, PrepareOperations, ArtifactGc, FinishGeneration, } pub(crate) fn recover_production_before_cold_start( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, catalog: &Catalog, prepared: &PreparedLowerStore, ) -> Result<(), String> { recover_production_before_cold_start_with( configuration, store, catalog, prepared, |_, _| Ok(()), ) } fn recover_production_before_cold_start_with( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, catalog: &Catalog, prepared: &PreparedLowerStore, failpoint: Failpoint, ) -> Result<(), String> where Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>, { recover_production_before_cold_start_with_opener( configuration, store, catalog, prepared, failpoint, match_recovery_process, |root, container| delete_runsc_container(root, container, std::ffi::OsStr::new("runsc")), ) } fn recover_production_before_cold_start_with_opener( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, catalog: &Catalog, prepared: &PreparedLowerStore, mut failpoint: Failpoint, mut open: Open, mut delete: Delete, ) -> Result<(), String> where Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>, Open: FnMut(PidfsProcessIdentityV1) -> io::Result, Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>, { failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?; let snapshot = store.snapshot().map_err(|error| error.to_string())?; if snapshot.domain.domain_id != configuration.domain_id() { return Err("recovery Domain differs from resolved configuration".into()); } let mut generations = snapshot.generations.clone(); generations.sort_by_key(|generation| generation.generation_id.get()); for generation in generations { if generation.durable_decision == DurableDecision::Clean { continue; } if generation.configuration_fingerprint != configuration.configuration_fingerprint() { return Err("unfinished generation configuration differs".into()); } recover_generation( configuration, store, catalog, &generation, &mut failpoint, &mut open, &mut delete, )?; } failpoint( ProductionRecoveryBoundary::PrepareOperations, GenerationId::new(0), )?; prepared .recover_preparations() .map_err(|error| error.to_string())?; failpoint(ProductionRecoveryBoundary::ArtifactGc, GenerationId::new(0))?; let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?; let selected = catalog_snapshot .prepared_artifacts .iter() .filter(|artifact| { !catalog_snapshot .artifact_leases .iter() .any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id) }) .map(|artifact| artifact.prepared_artifact_id) .collect::>(); prepared .gc_artifacts(&selected) .map_err(|error| error.to_string())?; Ok(()) } fn recover_generation( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, catalog: &Catalog, generation: &Generation, failpoint: &mut Failpoint, open: &mut Open, delete: &mut Delete, ) -> Result<(), String> where Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>, Open: FnMut(PidfsProcessIdentityV1) -> io::Result, Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>, { let generation_id = generation.generation_id; let snapshot = store.snapshot().map_err(|error| error.to_string())?; if snapshot .session_terminals .iter() .any(|terminal| terminal.generation_id() == generation_id) || snapshot .mailboxes .iter() .any(|mailbox| mailbox.generation_id == generation_id) { return Err("unfinished terminal or mailbox state cannot be recovered safely".into()); } let session_updates = snapshot .sessions .iter() .filter(|session| { session.generation_id == generation_id && !matches!( session.session_phase, SessionPhase::Closed | SessionPhase::Failed ) }) .map(|session| { let mut failed = session.clone(); failed.session_phase = SessionPhase::Failed; DomainStoreMutation::PutSession(failed) }) .collect::>(); if !session_updates.is_empty() { store .apply(&session_updates) .map_err(|error| error.to_string())?; } let participants = store .snapshot() .map_err(|error| error.to_string())? .participants; let ownership = runtime_recovery_ownership(&participants, generation)?; let mut owners = seal_runtime_owners(&ownership, open).map_err(|error| error.to_string())?; failpoint(ProductionRecoveryBoundary::Publication, generation_id)?; recover_publication(configuration, store, generation_id)?; failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?; reconcile_runtime_before_state_anchor(&mut owners).map_err(|error| error.to_string())?; require_sandbox_absent(configuration, store, generation_id, delete)?; clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)?; failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?; clean_role_after_absence(store, generation_id, ParticipantRole::Helper)?; clean_role_after_absence(store, generation_id, ParticipantRole::Network)?; failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?; require_mount_absent( ownership.git_routing_path.as_ref(), ownership.git_routing_mount, "Git-routing", ) .map_err(|error| error.to_string())?; clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)?; failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?; recover_state_anchor(configuration, store, catalog, generation, &mut owners)?; failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?; recover_partial_prepared_lower(configuration, store, catalog, generation_id)?; let snapshot = store.snapshot().map_err(|error| error.to_string())?; if snapshot.participants.iter().any(|participant| { participant.generation_id() == generation_id && participant.state() != ParticipantState::Clean }) { return Err("generation retains non-Clean participants after recovery".into()); } failpoint(ProductionRecoveryBoundary::FinishGeneration, generation_id)?; let mut clean = generation.clone(); clean.durable_decision = DurableDecision::Clean; store .apply(&[DomainStoreMutation::PutGeneration(clean)]) .map_err(|error| error.to_string()) } fn recover_publication( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, generation_id: GenerationId, ) -> Result<(), String> { let Some(participant) = role(store, generation_id, ParticipantRole::Publication)? else { return Ok(()); }; if participant.state() == ParticipantState::Clean { return Ok(()); } let Some(ParticipantOwnership::Publication(publication)) = participant.ownership() else { if participant.ownership().is_none() { return clean_role_after_absence(store, generation_id, ParticipantRole::Publication); } return Err("publication ownership shape is invalid".into()); }; let root = configuration .runtime_domain_root() .open_authority() .map_err(|error| error.to_string())?; require_publication_labels(publication)?; remove_exact_publication_resource( &root, [STAGED_CONTROL_SOCKET_NAME, CONTROL_SOCKET_NAME], publication .socket_identity .ok_or("publication socket identity is absent")?, true, )?; remove_exact_publication_resource( &root, [STAGED_PUBLICATION_IDENTITY_NAME, PUBLICATION_IDENTITY_NAME], publication .identity_file_identity .ok_or("publication identity-file identity is absent")?, false, )?; clean_role_after_absence(store, generation_id, ParticipantRole::Publication) } fn require_publication_labels( ownership: &lazaret_core::records::PublicationOwnership, ) -> Result<(), String> { for (observed, expected) in [ (&ownership.staged_socket_path, STAGED_CONTROL_SOCKET_NAME), ( &ownership.staged_identity_path, STAGED_PUBLICATION_IDENTITY_NAME, ), (&ownership.socket_path, CONTROL_SOCKET_NAME), (&ownership.identity_path, PUBLICATION_IDENTITY_NAME), ] { if observed.as_ref().map(HostPath::as_bytes) != Some(expected.as_bytes()) { return Err("publication durable label differs from the fixed runtime label".into()); } } Ok(()) } fn remove_exact_publication_resource( root: &std::os::fd::OwnedFd, labels: [&str; 2], expected: MountIdentity, socket: bool, ) -> Result<(), String> { let mut exact = None; for label in labels { let descriptor = match openat( root, label, OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW, Mode::empty(), ) { Ok(descriptor) => descriptor, Err(rustix::io::Errno::NOENT) => continue, Err(error) => return Err(error.to_string()), }; let stat = fstat(&descriptor).map_err(|error| error.to_string())?; let file_type = rustix::fs::FileType::from_raw_mode(stat.st_mode); if mount_identity(&descriptor).map_err(|error| error.to_string())? != expected || (socket && !file_type.is_socket()) || (!socket && !file_type.is_file()) || exact.replace(label).is_some() { return Err("publication resource changed identity, type, or location".into()); } } if let Some(label) = exact { let current = openat( root, label, OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW, Mode::empty(), ) .map_err(|error| error.to_string())?; let stat = fstat(¤t).map_err(|error| error.to_string())?; let file_type = rustix::fs::FileType::from_raw_mode(stat.st_mode); if mount_identity(¤t).map_err(|error| error.to_string())? != expected || (socket && !file_type.is_socket()) || (!socket && !file_type.is_file()) { return Err("publication resource changed before exact removal".into()); } unlinkat(root, label, AtFlags::empty()).map_err(|error| error.to_string())?; } Ok(()) } fn require_sandbox_absent( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, generation_id: GenerationId, delete: &mut impl FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>, ) -> Result<(), String> { let Some(participant) = role(store, generation_id, ParticipantRole::Sandbox)? else { return Ok(()); }; if participant.state() == ParticipantState::Clean { return Ok(()); } let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else { return Ok(()); }; let (Some(runtime_root), Some(runtime_root_identity), Some(container_id)) = ( &sandbox.runtime_root, sandbox.runtime_root_identity, sandbox.container_id, ) else { return Err("Sandbox runsc-root recovery authority is incomplete".into()); }; let expected_label = Path::new(std::ffi::OsStr::from_bytes( configuration .runtime_domain_root() .canonical_label() .as_bytes(), )) .join("runsc"); if runtime_root.as_bytes() != expected_label.as_os_str().as_bytes() { return Err("Sandbox runsc root differs from the resolved Runtime Domain".into()); } let root = configuration .runtime_domain_root() .open_beneath(Path::new("runsc"), ConfinedPathType::Directory) .map_err(|error| error.to_string())?; if mount_identity(&root).map_err(|error| error.to_string())? != runtime_root_identity { return Err("Sandbox runsc-root physical identity changed".into()); } let container = hex_lower(container_id.as_bytes()); match statat(&root, &container, AtFlags::SYMLINK_NOFOLLOW) { Err(rustix::io::Errno::NOENT) => {} Err(error) => return Err(error.to_string()), Ok(stat) => { if !rustix::fs::FileType::from_raw_mode(stat.st_mode).is_dir() { return Err("recorded runsc container changed type".into()); } delete(&root, &container)?; match statat(&root, &container, AtFlags::SYMLINK_NOFOLLOW) { Err(rustix::io::Errno::NOENT) => {} Err(error) => return Err(error.to_string()), Ok(_) => return Err("recorded runsc container remains after exact cleanup".into()), } } } Ok(()) } /// Deletes one exact container beneath a retained, already-verified runsc root descriptor. /// /// # Errors /// Returns launch, wrapper, or nonzero runsc failure without weakening the retained root. pub fn delete_runsc_container( root: &std::os::fd::OwnedFd, container: &str, program: &std::ffi::OsStr, ) -> Result<(), String> { fcntl_setfd(root, FdFlags::empty()).map_err(|error| error.to_string())?; let root_label = format!("/proc/self/fd/{}", root.as_raw_fd()); child_broker() .run_transient( &TargetCommand::new(program) .argument(format!("-root={root_label}")) .argument("delete") .argument("--force") .argument(container), ) .map_err(|error| error.to_string())?; Ok(()) } fn recover_state_anchor( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, catalog: &Catalog, generation: &Generation, owners: &mut SealedRuntimeOwners, ) -> Result<(), String> { let Some(anchor) = role( store, generation.generation_id, ParticipantRole::StateAnchor, )? else { return Ok(()); }; if anchor.state() == ParticipantState::Clean { let prepared = role( store, generation.generation_id, ParticipantRole::PreparedLower, )?; if prepared .is_some_and(|participant| participant.state() == ParticipantState::ReleaseIntent) { let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?; let coordinator = StateAnchorCoordinator::new(store, catalog); return coordinator .resume_release_after_checkpoint(generation.generation_id, &layout, || { fresh_absence(store, generation.generation_id, &layout) }) .map_err(|error| error.to_string()); } return Ok(()); } let full_mount = matches!( anchor.state(), ParticipantState::Prepared | ParticipantState::ReleaseIntent ) && owners.ownership.state_anchor_mount.is_some(); if !full_mount { let _ = owners.prove_absent().map_err(|error| error.to_string())?; return clean_role_after_absence( store, generation.generation_id, ParticipantRole::StateAnchor, ); } let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?; let coordinator = StateAnchorCoordinator::new(store, catalog); owners .ownership .process(RuntimeProcessRole::StateAnchorDaemon) .ok_or("State Anchor daemon authority is absent")?; let recovered = if let Some(SealedProcessMatch::Matched(process)) = owners.process(RuntimeProcessRole::StateAnchorDaemon) { if process.has_exited().map_err(|error| error.to_string())? { let witness = owners.prove_absent().map_err(|error| error.to_string())?; coordinator .recover_after_runtime_owner_loss( generation.generation_id, &layout, StateAnchorRecoveryOwner::Absent(&witness), ) .map_err(|error| error.to_string())? } else { coordinator .recover_after_runtime_owner_loss( generation.generation_id, &layout, StateAnchorRecoveryOwner::Matched(process), ) .map_err(|error| error.to_string())? } } else { let witness = owners.prove_absent().map_err(|error| error.to_string())?; coordinator .recover_after_runtime_owner_loss( generation.generation_id, &layout, StateAnchorRecoveryOwner::Absent(&witness), ) .map_err(|error| error.to_string())? }; let (authority, first) = recovered; let complete_witness = owners.prove_absent().map_err(|error| error.to_string())?; if !complete_witness.ownership().state_anchor_matches( authority.daemon(), authority.user_namespace(), authority.mount_namespace(), authority.mount(), ) { return Err("completed recovery witness changed State Anchor authority".into()); } coordinator .complete_release_after_unmount(&authority, &layout, first, || { fresh_absence(store, generation.generation_id, &layout) }) .map_err(|error| error.to_string()) } fn recover_partial_prepared_lower( configuration: &ResolvedDomainConfiguration, store: &HostCoordinatorDomainStore, catalog: &Catalog, generation_id: GenerationId, ) -> Result<(), String> { let Some(prepared) = role(store, generation_id, ParticipantRole::PreparedLower)? else { return Ok(()); }; if prepared.state() == ParticipantState::Clean { return Ok(()); } if prepared.state() == ParticipantState::ReleaseIntent { return Err("prepared-lower ReleaseIntent lacks complete State Anchor evidence".into()); } let Some(ParticipantOwnership::PreparedLower(ownership)) = prepared.ownership() else { return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower); }; if prepared.state() != ParticipantState::LeaseHeld { return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower); } let artifact = ownership .prepared_artifact_id .ok_or("LeaseHeld prepared artifact is absent")?; let domain_id = store .snapshot() .map_err(|error| error.to_string())? .domain .domain_id; let lease = ArtifactLease { prepared_artifact_id: artifact, domain_id, generation_id, }; require_exact_lease(catalog, &lease)?; let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?; lazaret_state_anchor::require_mount_absent(&layout).map_err(|error| error.to_string())?; let release = Participant::snapshot( generation_id, ParticipantRole::PreparedLower, ParticipantState::ReleaseIntent, Some(ParticipantOwnership::PreparedLower( lazaret_core::records::PreparedLowerOwnership { lease_state: Some(ParticipantState::ReleaseIntent), ..ownership.clone() }, )), ) .map_err(|_| "partial prepared-lower release snapshot is invalid")?; store .apply(&[DomainStoreMutation::PutParticipant(release)]) .map_err(|error| error.to_string())?; catalog .release_lease_checked(&lease, || { let snapshot = store .snapshot() .map_err(|error| CatalogError::Io(io::Error::other(error)))?; let prepared = snapshot .participants .iter() .find(|participant| { participant.generation_id() == generation_id && participant.role() == ParticipantRole::PreparedLower }) .ok_or_else(|| CatalogError::Io(io::Error::other("prepared participant absent")))?; if prepared.state() != ParticipantState::ReleaseIntent { return Err(CatalogError::Io(io::Error::other( "prepared release intent is absent", ))); } lazaret_state_anchor::require_mount_absent(&layout) .map_err(|error| CatalogError::Io(io::Error::other(error))) }) .map_err(|error| error.to_string())?; clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower) } fn require_exact_lease(catalog: &Catalog, expected: &ArtifactLease) -> Result<(), String> { match catalog .snapshot() .map_err(|error| error.to_string())? .artifact_leases .iter() .find(|lease| { lease.domain_id == expected.domain_id && lease.generation_id == expected.generation_id }) { Some(observed) if observed == expected => Ok(()), Some(_) => Err("generation catalog lease mismatches durable ownership".into()), None => Err("LeaseHeld prepared lower lacks its exact catalog lease".into()), } } fn fresh_absence( store: &HostCoordinatorDomainStore, generation_id: GenerationId, layout: &StateAnchorLayout<'_>, ) -> Result { let snapshot = store.snapshot()?; let anchor = snapshot .participants .iter() .find(|participant| { participant.generation_id() == generation_id && participant.role() == ParticipantRole::StateAnchor }) .ok_or(StateAnchorCoordinatorError::InvalidState( "State Anchor participant is absent", ))?; let mut evidence = match anchor.ownership() { Some(ParticipantOwnership::StateAnchor(ownership)) => ( ownership.user_namespace, ownership.mount_namespace, ownership.mount, ), _ => (None, None, None), }; if evidence == (None, None, None) { evidence = snapshot .participants .iter() .find(|participant| { participant.generation_id() == generation_id && participant.role() == ParticipantRole::PreparedLower }) .and_then(Participant::ownership) .and_then(|ownership| match ownership { ParticipantOwnership::PreparedLower(ownership) => ownership.released_state_anchor, _ => None, }) .map_or((None, None, None), |evidence| { ( Some(evidence.user_namespace), Some(evidence.mount_namespace), Some(evidence.mount), ) }); } let (Some(user_namespace), Some(mount_namespace), Some(mount)) = evidence else { return Err(StateAnchorCoordinatorError::InvalidState( "State Anchor absence evidence is incomplete", )); }; let observed = lazaret_state_anchor::observe_mount_absence(layout, mount)?; Ok(StateAnchorAbsenceObservation { user_namespace, mount_namespace, mount, observed, }) } fn clean_role_after_absence( store: &HostCoordinatorDomainStore, generation_id: GenerationId, role_name: ParticipantRole, ) -> Result<(), String> { let Some(current) = role(store, generation_id, role_name)? else { return Ok(()); }; if current.state() == ParticipantState::Clean { return Ok(()); } if current.state() != ParticipantState::ReleaseIntent && current.ownership().is_some() { let release = Participant::snapshot( generation_id, role_name, ParticipantState::ReleaseIntent, current.ownership().cloned(), ) .map_err(|_| "participant release snapshot is invalid")?; store .apply(&[DomainStoreMutation::PutParticipant(release)]) .map_err(|error| error.to_string())?; } let clean = Participant::snapshot( generation_id, role_name, ParticipantState::Clean, Some(ParticipantOwnership::empty(role_name)), ) .map_err(|_| "participant Clean snapshot is invalid")?; store .apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(clean)]) .map_err(|error| error.to_string()) } fn role( store: &HostCoordinatorDomainStore, generation_id: GenerationId, role_name: ParticipantRole, ) -> Result, String> { Ok(store .snapshot() .map_err(|error| error.to_string())? .participants .into_iter() .find(|participant| { participant.generation_id() == generation_id && participant.role() == role_name })) } fn hex_lower(bytes: &[u8]) -> String { const HEX: &[u8; 16] = b"0123456789abcdef"; let mut output = String::with_capacity(bytes.len() * 2); for byte in bytes { output.push(char::from(HEX[usize::from(byte >> 4)])); output.push(char::from(HEX[usize::from(byte & 0x0f)])); } output } #[cfg(test)] mod tests { use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; use std::os::unix::fs::MetadataExt; use std::os::unix::fs::symlink; use std::os::unix::net::UnixListener; use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration}; use lazaret_core::identity::{ BootId, BundleDigest, ContainerId, CoordinatorId, DeviceId, HostPidLocator, IdMapDigest, InodeId, MountId, NetworkSidecarId, PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, PolicyId, PrepareOperationId, PreparedArtifactId, ProcessStartTimeTicks, SandboxId, SelectedManifestDigest, SourceIdentity, }; use lazaret_core::records::{ ArtifactState, DurableDecision, GitRoutingOwnership, HelperOwnership, PrepareOperation, PrepareStageState, PreparedArtifact, PreparedLowerOwnership, PublicationOwnership, SandboxOwnership, StateAnchorOwnership, }; use lazaret_linux::filesystem::try_exclusive_flock; use lazaret_runtime_supervisor::sandbox::production_sandbox_layout; use super::*; fn process(value: u64) -> PidfsProcessIdentityV1 { PidfsProcessIdentityV1::new( PidfsProcessAuthorityV1 { boot_id: BootId::new([1; 16]), pidfs_st_dev: PidfsDeviceId::new(value), pidfs_st_ino: PidfsInodeId::new(value), }, HostPidLocator::new(u32::try_from(value).unwrap()), ProcessStartTimeTicks::new(value), ) } fn generation() -> Generation { Generation { domain_id: DomainId::new([1; 32]), generation_id: GenerationId::new(1), configuration_fingerprint: ConfigurationFingerprint::new([2; 32]), coordinator_id: CoordinatorId::new([3; 32]), network_sidecar_id: NetworkSidecarId::new([4; 32]), sandbox_id: SandboxId::new([5; 32]), durable_decision: DurableDecision::Preparing, } } fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant { Participant::snapshot( GenerationId::new(1), role, ParticipantState::Intent, Some(ownership), ) .unwrap() } #[test] fn inventory_rejects_default_partial_and_helper_or_fuse_omission() { let generation = generation(); let empty = runtime_recovery_ownership(&[], &generation).unwrap(); assert!(prove_runtime_owners_absent(&empty).is_err()); let helper = participant( ParticipantRole::Helper, ParticipantOwnership::Helper(HelperOwnership { network_sidecar_id: None, connectivity_helper: PersistedHostProcessOwnership::staged(process(4)), runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)), }), ); assert!(runtime_recovery_ownership(&[helper], &generation).is_err()); let incomplete_fuse = participant( ParticipantRole::StateAnchor, ParticipantOwnership::StateAnchor(StateAnchorOwnership { lower_artifact_id: None, upper_path: None, work_path: None, merged_path: Some(HostPath::new(b"/tmp/missing".to_vec())), daemon: PersistedHostProcessOwnership::unassigned(), user_namespace: Some(NamespaceIdentity { device: DeviceId::new(6), inode: InodeId::new(7), }), mount_namespace: Some(NamespaceIdentity { device: DeviceId::new(8), inode: InodeId::new(9), }), mount: Some(MountIdentity { mount_id: MountId::new(10), device: DeviceId::new(11), inode: InodeId::new(12), }), }), ); let prepared = Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower); assert!( runtime_recovery_ownership(&[prepared.clone(), incomplete_fuse], &generation).is_err() ); let staged_state = participant( ParticipantRole::StateAnchor, ParticipantOwnership::StateAnchor(StateAnchorOwnership { lower_artifact_id: None, upper_path: None, work_path: None, merged_path: None, daemon: PersistedHostProcessOwnership::staged(process(5)), user_namespace: None, mount_namespace: None, mount: None, }), ); let mut opened = 0; let partial = runtime_recovery_ownership(&[prepared, staged_state], &generation); assert!(partial.is_err()); if let Ok(partial) = partial { let _ = seal_runtime_owners(&partial, |_| { opened += 1; Err(io::Error::from_raw_os_error( rustix::io::Errno::SRCH.raw_os_error(), )) }); } assert_eq!(opened, 0); } #[test] fn complete_boot_mismatch_inventory_proves_only_after_every_holder_and_mount() { let generation = generation(); let prepared = Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower); let state = participant( ParticipantRole::StateAnchor, ParticipantOwnership::StateAnchor(StateAnchorOwnership { lower_artifact_id: None, upper_path: None, work_path: None, merged_path: None, daemon: PersistedHostProcessOwnership::staged(process(5)), user_namespace: None, mount_namespace: None, mount: None, }), ); let helper = participant( ParticipantRole::Helper, ParticipantOwnership::Helper(HelperOwnership { network_sidecar_id: None, connectivity_helper: PersistedHostProcessOwnership::staged(process(4)), runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)), }), ); let network = Participant::bare_intent(GenerationId::new(1), ParticipantRole::Network); let sandbox = participant( ParticipantRole::Sandbox, ParticipantOwnership::Sandbox(SandboxOwnership { container_id: Some(ContainerId::new([3; 32])), runtime_root: Some(HostPath::new(b"/runtime".to_vec())), runtime_root_identity: Some(MountIdentity { mount_id: MountId::new(13), device: DeviceId::new(14), inode: InodeId::new(15), }), bundle_digest: Some(BundleDigest::new([4; 32])), host_coordinator: PersistedHostProcessOwnership::committed(process(1)), runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)), attached_runsc: PersistedHostProcessOwnership::staged(process(3)), }), ); let inventory = runtime_recovery_ownership(&[prepared, state, helper, network, sandbox], &generation) .unwrap(); assert_eq!( prove_runtime_owners_absent(&inventory).unwrap().ownership(), &inventory ); } #[test] fn production_preflight_recovers_full_runtime_then_is_idempotent() { let fixture = ProductionRecoveryFixture::new(); let mut boundaries = Vec::new(); fixture .recover_with(|boundary, generation| { boundaries.push((boundary, generation)); Ok(()) }) .unwrap(); assert!( boundaries .iter() .any(|(boundary, _)| { *boundary == ProductionRecoveryBoundary::StateAnchor }) ); assert_eq!( fixture.store.snapshot().unwrap().generations[0].durable_decision, DurableDecision::Clean ); assert!( fixture .catalog .snapshot() .unwrap() .artifact_leases .is_empty() ); assert!( fixture .catalog .snapshot() .unwrap() .prepared_artifacts .is_empty() ); let runtime_domain = Path::new(std::ffi::OsStr::from_bytes( fixture .configuration .runtime_domain_root() .canonical_label() .as_bytes(), )); assert!( !runtime_domain .join("runsc") .join(hex_lower(&[6; 32])) .exists() ); recover_production_before_cold_start( &fixture.configuration, &fixture.store, &fixture.catalog, &fixture.prepared, ) .unwrap(); } #[test] fn production_preflight_retries_every_recovery_boundary() { for failed in [ ProductionRecoveryBoundary::Snapshot, ProductionRecoveryBoundary::Publication, ProductionRecoveryBoundary::Sandbox, ProductionRecoveryBoundary::HelperAndNetwork, ProductionRecoveryBoundary::GitRouting, ProductionRecoveryBoundary::StateAnchor, ProductionRecoveryBoundary::PreparedLower, ProductionRecoveryBoundary::PrepareOperations, ProductionRecoveryBoundary::ArtifactGc, ProductionRecoveryBoundary::FinishGeneration, ] { let fixture = ProductionRecoveryFixture::new(); let mut injected = false; assert!( fixture .recover_with(|boundary, _| { if boundary == failed && !injected { injected = true; Err("injected production recovery boundary".into()) } else { Ok(()) } },) .is_err() ); assert!(injected); fixture.recover_with(|_, _| Ok(())).unwrap(); assert_eq!( fixture.store.snapshot().unwrap().generations[0].durable_decision, DurableDecision::Clean ); } } #[test] fn production_preflight_opens_every_exact_owner_once() { let fixture = ProductionRecoveryFixture::new(); let mut opened = BTreeMap::new(); recover_production_before_cold_start_with_opener( &fixture.configuration, &fixture.store, &fixture.catalog, &fixture.prepared, |_, _| Ok(()), |identity| { *opened.entry(identity.host_pid().get()).or_insert(0) += 1; Ok(PidfsRecoveryMatch::BootMismatch) }, |root, container| { unlinkat(root, container, AtFlags::REMOVEDIR).map_err(|error| error.to_string()) }, ) .unwrap(); assert_eq!(opened.len(), 6); assert!(opened.values().all(|count| *count == 1)); } #[test] fn publication_replacement_and_runsc_root_replacement_refuse_untouched() { let fixture = ProductionRecoveryFixture::new(); let runtime_domain = Path::new(std::ffi::OsStr::from_bytes( fixture .configuration .runtime_domain_root() .canonical_label() .as_bytes(), )); let socket = runtime_domain.join(CONTROL_SOCKET_NAME); let old_socket = runtime_domain.join("old-control.sock"); fs::rename(&socket, &old_socket).unwrap(); let root = File::open(runtime_domain).unwrap(); drop( UnixListener::bind(format!( "/proc/self/fd/{}/{}", root.as_raw_fd(), CONTROL_SOCKET_NAME )) .unwrap(), ); assert!(fixture.recover_with(|_, _| Ok(())).is_err()); assert!(socket.exists()); let fixture = ProductionRecoveryFixture::new(); let runtime_domain = Path::new(std::ffi::OsStr::from_bytes( fixture .configuration .runtime_domain_root() .canonical_label() .as_bytes(), )); let identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME); fs::rename(&identity, runtime_domain.join("old-identity")).unwrap(); symlink("old-identity", &identity).unwrap(); assert!(fixture.recover_with(|_, _| Ok(())).is_err()); assert!( fs::symlink_metadata(&identity) .unwrap() .file_type() .is_symlink() ); let fixture = ProductionRecoveryFixture::new(); let runtime_domain = Path::new(std::ffi::OsStr::from_bytes( fixture .configuration .runtime_domain_root() .canonical_label() .as_bytes(), )); let identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME); fs::rename(&identity, runtime_domain.join("old-identity")).unwrap(); fs::write(&identity, b"replacement").unwrap(); assert!(fixture.recover_with(|_, _| Ok(())).is_err()); assert_eq!(fs::read(&identity).unwrap(), b"replacement"); let fixture = ProductionRecoveryFixture::new(); let runtime_domain = Path::new(std::ffi::OsStr::from_bytes( fixture .configuration .runtime_domain_root() .canonical_label() .as_bytes(), )); let runsc = runtime_domain.join("runsc"); let old_runsc = runtime_domain.join("old-runsc"); fs::rename(&runsc, &old_runsc).unwrap(); fs::create_dir(&runsc).unwrap(); let mut deletes = 0; let result = recover_production_before_cold_start_with_opener( &fixture.configuration, &fixture.store, &fixture.catalog, &fixture.prepared, |_, _| Ok(()), |_| Ok(PidfsRecoveryMatch::BootMismatch), |_, _| { deletes += 1; Ok(()) }, ); assert!(result.is_err()); assert_eq!(deletes, 0); assert!(runsc.exists()); let fixture = ProductionRecoveryFixture::new(); let runtime_domain = Path::new(std::ffi::OsStr::from_bytes( fixture .configuration .runtime_domain_root() .canonical_label() .as_bytes(), )); let runsc = runtime_domain.join("runsc"); fs::rename(&runsc, runtime_domain.join("old-runsc")).unwrap(); symlink("old-runsc", &runsc).unwrap(); assert!(fixture.recover_with(|_, _| Ok(())).is_err()); assert!( fs::symlink_metadata(&runsc) .unwrap() .file_type() .is_symlink() ); } struct ProductionRecoveryFixture { _tree: tempfile::TempDir, configuration: ResolvedDomainConfiguration, store: HostCoordinatorDomainStore, catalog: Catalog, prepared: PreparedLowerStore, } impl ProductionRecoveryFixture { fn recover_with(&self, failpoint: Failpoint) -> Result<(), String> where Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>, { recover_production_before_cold_start_with_opener( &self.configuration, &self.store, &self.catalog, &self.prepared, failpoint, match_recovery_process, |root, container| { unlinkat(root, container, AtFlags::REMOVEDIR).map_err(|error| error.to_string()) }, ) } #[allow(clippy::too_many_lines)] fn new() -> Self { let tree = tempfile::tempdir().unwrap(); let configuration = test_configuration(tree.path()); let catalog = Catalog::under_storage_root(configuration.storage_root()).unwrap(); catalog.initialize().unwrap(); let prepared = PreparedLowerStore::under_storage_root( configuration.storage_root(), catalog.clone(), ) .unwrap(); let artifact_name = hex_lower(&[21; 32]); let artifact_path = tree .path() .join("data/prepared-lower/artifacts") .join(&artifact_name); fs::create_dir(&artifact_path).unwrap(); let metadata = artifact_path.metadata().unwrap(); let artifact = PreparedArtifact { prepared_artifact_id: PreparedArtifactId::new([21; 32]), source_identity: SourceIdentity::new(vec![22]), selected_manifest_digest: SelectedManifestDigest::new([23; 32]), policy_id: PolicyId::new([24; 32]), id_map_digest: IdMapDigest::new([25; 32]), path: HostPath::new(format!("artifacts/{artifact_name}").into_bytes()), device: DeviceId::new(metadata.dev()), inode: InodeId::new(metadata.ino()), state: ArtifactState::Available, }; publish_artifact(&catalog, &artifact); let domain = lazaret_core::records::Domain { domain_id: configuration.domain_id(), profile_name: configuration.selected_profile().to_owned(), canonical_workspace: configuration.workspace().canonical_label().clone(), }; let lock = OpenOptions::new() .create(true) .read(true) .truncate(false) .write(true) .open(tree.path().join("domain.lock")) .unwrap(); let store = HostCoordinatorDomainStore::open( try_exclusive_flock(lock).unwrap(), &tree.path().join("domain.redb"), &domain, ) .unwrap(); let generation = Generation { domain_id: domain.domain_id, generation_id: GenerationId::new(1), configuration_fingerprint: configuration.configuration_fingerprint(), coordinator_id: CoordinatorId::new([3; 32]), network_sidecar_id: NetworkSidecarId::new([4; 32]), sandbox_id: SandboxId::new([5; 32]), durable_decision: DurableDecision::None, }; store .apply(&[DomainStoreMutation::PutGeneration(generation.clone())]) .unwrap(); let mut generation = generation; generation.durable_decision = DurableDecision::Preparing; store .apply(&[DomainStoreMutation::PutGeneration(generation.clone())]) .unwrap(); let layout = StateAnchorLayout::create(&configuration).unwrap(); let mut anchor = layout.ownership_intent(artifact.prepared_artifact_id); let mut staged_anchor = anchor.clone(); staged_anchor.daemon = PersistedHostProcessOwnership::staged(process(5)); anchor.daemon = PersistedHostProcessOwnership::committed(process(5)); anchor.user_namespace = Some(NamespaceIdentity { device: DeviceId::new(30), inode: InodeId::new(31), }); anchor.mount_namespace = Some(NamespaceIdentity { device: DeviceId::new(32), inode: InodeId::new(33), }); anchor.mount = Some(MountIdentity { mount_id: MountId::new(u64::MAX), device: DeviceId::new(34), inode: InodeId::new(35), }); let prepared_ownership = PreparedLowerOwnership { prepared_artifact_id: Some(artifact.prepared_artifact_id), source_identity: Some(artifact.source_identity.clone()), lease_state: Some(ParticipantState::LeaseHeld), path: Some(artifact.path.clone()), device: Some(artifact.device), inode: Some(artifact.inode), released_state_anchor: None, }; let helper = HelperOwnership { network_sidecar_id: None, connectivity_helper: PersistedHostProcessOwnership::staged(process(4)), runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)), }; let runtime_domain = Path::new(std::ffi::OsStr::from_bytes( configuration .runtime_domain_root() .canonical_label() .as_bytes(), )); let runsc_root = runtime_domain.join("runsc"); fs::create_dir(&runsc_root).unwrap(); let container_id = ContainerId::new([6; 32]); fs::create_dir(runsc_root.join(hex_lower(container_id.as_bytes()))).unwrap(); let sandbox = SandboxOwnership { container_id: Some(container_id), runtime_root: Some(HostPath::new(runsc_root.as_os_str().as_bytes().to_vec())), runtime_root_identity: Some(current_mount_identity(&runsc_root).unwrap()), bundle_digest: Some(BundleDigest::new([7; 32])), host_coordinator: PersistedHostProcessOwnership::committed(process(1)), runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)), attached_runsc: PersistedHostProcessOwnership::staged(process(3)), }; let publication_socket = runtime_domain.join(CONTROL_SOCKET_NAME); let publication_root = File::open(runtime_domain).unwrap(); drop( UnixListener::bind(format!( "/proc/self/fd/{}/{}", publication_root.as_raw_fd(), CONTROL_SOCKET_NAME )) .unwrap(), ); let publication_identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME); fs::write(&publication_identity, b"identity").unwrap(); let publication = PublicationOwnership { staged_socket_path: Some(HostPath::new( STAGED_CONTROL_SOCKET_NAME.as_bytes().to_vec(), )), staged_identity_path: Some(HostPath::new( STAGED_PUBLICATION_IDENTITY_NAME.as_bytes().to_vec(), )), socket_path: Some(HostPath::new(CONTROL_SOCKET_NAME.as_bytes().to_vec())), identity_path: Some(HostPath::new(PUBLICATION_IDENTITY_NAME.as_bytes().to_vec())), socket_identity: Some(path_identity(runtime_domain, &publication_socket)), identity_file_identity: Some(path_identity(runtime_domain, &publication_identity)), }; let git_path = runtime_domain.join("git-routing-fixture"); fs::create_dir(&git_path).unwrap(); let git_routing = GitRoutingOwnership { mountpoint: Some(HostPath::new(git_path.as_os_str().as_bytes().to_vec())), host_root: Some(HostPath::new(b"/git-host".to_vec())), isolated_root: Some(HostPath::new(b"/git-isolated".to_vec())), daemon: PersistedHostProcessOwnership::staged(process(6)), mount: Some(MountIdentity { mount_id: MountId::new(u64::MAX - 1), device: DeviceId::new(40), inode: InodeId::new(41), }), }; store .apply(&[ DomainStoreMutation::PutParticipant(Participant::bare_intent( generation.generation_id, ParticipantRole::PreparedLower, )), DomainStoreMutation::PutParticipant( Participant::snapshot( generation.generation_id, ParticipantRole::PreparedLower, ParticipantState::Intent, Some(ParticipantOwnership::PreparedLower( PreparedLowerOwnership { lease_state: Some(ParticipantState::Intent), ..prepared_ownership.clone() }, )), ) .unwrap(), ), ]) .unwrap(); let lease = ArtifactLease { prepared_artifact_id: artifact.prepared_artifact_id, domain_id: domain.domain_id, generation_id: generation.generation_id, }; catalog.acquire_lease_checked(&artifact, &lease).unwrap(); store .apply(&[ DomainStoreMutation::PutParticipant( Participant::snapshot( generation.generation_id, ParticipantRole::PreparedLower, ParticipantState::LeaseHeld, Some(ParticipantOwnership::PreparedLower(prepared_ownership)), ) .unwrap(), ), DomainStoreMutation::PutParticipant(Participant::bare_intent( generation.generation_id, ParticipantRole::StateAnchor, )), DomainStoreMutation::PutParticipant( Participant::snapshot( generation.generation_id, ParticipantRole::StateAnchor, ParticipantState::Intent, Some(ParticipantOwnership::StateAnchor(staged_anchor)), ) .unwrap(), ), DomainStoreMutation::PutParticipant( Participant::snapshot( generation.generation_id, ParticipantRole::StateAnchor, ParticipantState::Intent, Some(ParticipantOwnership::StateAnchor(anchor.clone())), ) .unwrap(), ), DomainStoreMutation::PutParticipant( Participant::snapshot( generation.generation_id, ParticipantRole::StateAnchor, ParticipantState::Prepared, Some(ParticipantOwnership::StateAnchor(anchor)), ) .unwrap(), ), DomainStoreMutation::PutParticipant(Participant::bare_intent( generation.generation_id, ParticipantRole::Helper, )), DomainStoreMutation::PutParticipant(participant( ParticipantRole::Helper, ParticipantOwnership::Helper(helper), )), DomainStoreMutation::PutParticipant(Participant::bare_intent( generation.generation_id, ParticipantRole::Network, )), DomainStoreMutation::PutParticipant(Participant::bare_intent( generation.generation_id, ParticipantRole::Sandbox, )), DomainStoreMutation::PutParticipant(participant( ParticipantRole::Sandbox, ParticipantOwnership::Sandbox(sandbox), )), DomainStoreMutation::PutParticipant(Participant::bare_intent( generation.generation_id, ParticipantRole::GitRouting, )), DomainStoreMutation::PutParticipant(participant( ParticipantRole::GitRouting, ParticipantOwnership::GitRouting(git_routing), )), DomainStoreMutation::PutParticipant(Participant::bare_intent( generation.generation_id, ParticipantRole::Publication, )), DomainStoreMutation::PutParticipant( Participant::snapshot( generation.generation_id, ParticipantRole::Publication, ParticipantState::Intent, Some(ParticipantOwnership::Publication(publication.clone())), ) .unwrap(), ), DomainStoreMutation::PutParticipant( Participant::snapshot( generation.generation_id, ParticipantRole::Publication, ParticipantState::Committed, Some(ParticipantOwnership::Publication(publication)), ) .unwrap(), ), ]) .unwrap(); Self { _tree: tree, configuration, store, catalog, prepared, } } } fn path_identity(root: &Path, path: &Path) -> MountIdentity { let root_identity = current_mount_identity(root).unwrap(); let metadata = fs::symlink_metadata(path).unwrap(); MountIdentity { mount_id: root_identity.mount_id, device: DeviceId::new(metadata.dev()), inode: InodeId::new(metadata.ino()), } } fn publish_artifact(catalog: &Catalog, artifact: &PreparedArtifact) { let intent = PrepareOperation { operation_id: PrepareOperationId::new([12; 32]), expected_artifact_id: artifact.prepared_artifact_id, source_identity: artifact.source_identity.clone(), selected_manifest_digest: artifact.selected_manifest_digest, policy_id: artifact.policy_id, id_map_digest: artifact.id_map_digest, stage_path: HostPath::new(b"stages/fixture".to_vec()), stage_state: PrepareStageState::Intent, device: None, inode: None, }; catalog.record_prepare(&intent).unwrap(); let owned = PrepareOperation { stage_state: PrepareStageState::Owned, device: Some(artifact.device), inode: Some(artifact.inode), ..intent.clone() }; catalog.bind_prepare_stage(&intent, &owned).unwrap(); catalog.publish(&owned, artifact).unwrap(); } fn test_configuration(root: &Path) -> ResolvedDomainConfiguration { let configuration_directory = root.join("configuration"); let workspace = root.join("workspace"); let runtime = root.join("runtime"); fs::create_dir(&configuration_directory).unwrap(); fs::create_dir(&workspace).unwrap(); fs::create_dir(&runtime).unwrap(); let configuration_path = configuration_directory.join("lazaret.toml"); fs::write( &configuration_path, format!( "schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 8443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n", root.join("data").display() ), ) .unwrap(); load_trusted_configuration(ConfigurationLoadRequest { explicit_configuration_path: Some(configuration_path), selected_profile: "default".into(), workspace, home: root.join("home"), xdg_config_home: None, xdg_data_home: None, xdg_runtime_dir: runtime, environment: BTreeMap::new(), sandbox_layout: production_sandbox_layout().unwrap(), }) .unwrap() } }