lazaret/crates/lazaret-host-coordinator/src/recovery.rs

use std::io;
use std::os::fd::AsRawFd;
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use std::time::{Duration, Instant};

use lazaret_core::configuration::{ConfinedPathType, ResolvedDomainConfiguration};
use lazaret_core::identity::{
    ConfigurationFingerprint, DomainId, GenerationId, HostPath, MountIdentity, NamespaceIdentity,
    PidfsProcessIdentityV1,
};
use lazaret_core::records::{
    ArtifactLease, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
    ParticipantState, PersistedHostProcessOwnership, SessionPhase,
};
use lazaret_linux::filesystem::{current_mount_identity, mount_identity};
use lazaret_linux::process::{
    PidfsRecoveryMatch, RetainedProcess, TargetCommand, child_broker, match_recovery_process,
};
use lazaret_prepared_lower::PreparedLowerStore;
use lazaret_state_anchor::StateAnchorLayout;
use lazaret_storage_catalog::{Catalog, CatalogError};
use rustix::fs::{AtFlags, Mode, OFlags, fstat, openat, statat, unlinkat};
use rustix::io::{FdFlags, fcntl_setfd};
use rustix::process::Signal;

use crate::control::{
    CONTROL_SOCKET_NAME, PUBLICATION_IDENTITY_NAME, STAGED_CONTROL_SOCKET_NAME,
    STAGED_PUBLICATION_IDENTITY_NAME,
};
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::runtime::StateAnchorAbsenceObservation;
use crate::state_anchor::{
    StateAnchorCoordinator, StateAnchorCoordinatorError, StateAnchorRecoveryOwner,
};

const PROCESS_EXIT_TIMEOUT: Duration = Duration::from_secs(5);

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RuntimeProcessRole {
    HostCoordinator,
    RuntimeSupervisor,
    AttachedRunsc,
    ConnectivityHelper,
    StateAnchorDaemon,
    GitRoutingDaemon,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RuntimeProcessOwner {
    role: RuntimeProcessRole,
    identity: PidfsProcessIdentityV1,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RuntimeRecoveryOwnership {
    domain_id: DomainId,
    generation_id: GenerationId,
    configuration_fingerprint: ConfigurationFingerprint,
    participants_present: usize,
    processes: Vec<RuntimeProcessOwner>,
    state_anchor_user_namespace: Option<NamespaceIdentity>,
    state_anchor_mount_namespace: Option<NamespaceIdentity>,
    state_anchor_mount: Option<MountIdentity>,
    state_anchor_path: Option<HostPath>,
    git_routing_mount: Option<MountIdentity>,
    git_routing_path: Option<HostPath>,
}

impl RuntimeRecoveryOwnership {
    fn process(&self, role: RuntimeProcessRole) -> Option<PidfsProcessIdentityV1> {
        self.processes
            .iter()
            .find(|owner| owner.role == role)
            .map(|owner| owner.identity)
    }

    pub(crate) fn binding_matches(&self, domain_id: DomainId, generation: &Generation) -> bool {
        self.domain_id == domain_id
            && self.generation_id == generation.generation_id
            && self.configuration_fingerprint == generation.configuration_fingerprint
    }

    pub(crate) fn state_anchor_matches(
        &self,
        daemon: PidfsProcessIdentityV1,
        user_namespace: NamespaceIdentity,
        mount_namespace: NamespaceIdentity,
        mount: MountIdentity,
    ) -> bool {
        self.process(RuntimeProcessRole::StateAnchorDaemon) == Some(daemon)
            && self.state_anchor_user_namespace == Some(user_namespace)
            && self.state_anchor_mount_namespace == Some(mount_namespace)
            && self.state_anchor_mount == Some(mount)
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RuntimeOwnersAbsent {
    ownership: RuntimeRecoveryOwnership,
}

enum SealedProcessMatch {
    BootMismatch,
    Esrch,
    Matched(RetainedProcess),
}

struct SealedProcessOwner {
    role: RuntimeProcessRole,
    matched: SealedProcessMatch,
    reconciled: bool,
}

struct SealedRuntimeOwners {
    ownership: RuntimeRecoveryOwnership,
    processes: Vec<SealedProcessOwner>,
}

impl SealedRuntimeOwners {
    fn process(&self, role: RuntimeProcessRole) -> Option<&SealedProcessMatch> {
        self.processes
            .iter()
            .find(|owner| owner.role == role)
            .map(|owner| &owner.matched)
    }

    fn reconcile(&mut self, role: RuntimeProcessRole) -> io::Result<()> {
        let Some(owner) = self.processes.iter_mut().find(|owner| owner.role == role) else {
            return Ok(());
        };
        if owner.reconciled {
            return Ok(());
        }
        match &owner.matched {
            SealedProcessMatch::BootMismatch | SealedProcessMatch::Esrch => {}
            SealedProcessMatch::Matched(process) if process.has_exited()? => {}
            SealedProcessMatch::Matched(_) if role == RuntimeProcessRole::HostCoordinator => {
                return Err(io::Error::other(
                    "recorded Host Coordinator remains present",
                ));
            }
            SealedProcessMatch::Matched(process) => {
                process.send_signal(Signal::KILL)?;
                let deadline = Instant::now() + PROCESS_EXIT_TIMEOUT;
                while !process.has_exited()? {
                    if Instant::now() >= deadline {
                        return Err(io::Error::other(
                            "exact runtime owner did not exit after pidfd action",
                        ));
                    }
                    std::thread::yield_now();
                }
            }
        }
        owner.reconciled = true;
        Ok(())
    }

    fn prove_absent(&mut self) -> Result<RuntimeOwnersAbsent, io::Error> {
        for role in [
            RuntimeProcessRole::AttachedRunsc,
            RuntimeProcessRole::ConnectivityHelper,
            RuntimeProcessRole::GitRoutingDaemon,
            RuntimeProcessRole::StateAnchorDaemon,
            RuntimeProcessRole::RuntimeSupervisor,
            RuntimeProcessRole::HostCoordinator,
        ] {
            self.reconcile(role)?;
        }
        require_mount_absent(
            self.ownership.state_anchor_path.as_ref(),
            self.ownership.state_anchor_mount,
            "State Anchor",
        )?;
        require_mount_absent(
            self.ownership.git_routing_path.as_ref(),
            self.ownership.git_routing_mount,
            "Git-routing",
        )?;
        Ok(RuntimeOwnersAbsent {
            ownership: self.ownership.clone(),
        })
    }
}

impl RuntimeOwnersAbsent {
    pub(crate) fn ownership(&self) -> &RuntimeRecoveryOwnership {
        &self.ownership
    }
}

/// Extracts one generation's complete exact process and private-mount recovery authority.
///
/// # Errors
/// Refuses duplicate roles, missing role dependencies, unequal owner copies, and every incomplete
/// process, namespace, mount, or path relation required by a recorded participant phase.
#[allow(clippy::too_many_lines)]
pub fn runtime_recovery_ownership(
    participants: &[Participant],
    generation: &Generation,
) -> Result<RuntimeRecoveryOwnership, &'static str> {
    let mut ownership = RuntimeRecoveryOwnership {
        domain_id: generation.domain_id,
        generation_id: generation.generation_id,
        configuration_fingerprint: generation.configuration_fingerprint,
        participants_present: 0,
        processes: Vec::new(),
        state_anchor_user_namespace: None,
        state_anchor_mount_namespace: None,
        state_anchor_mount: None,
        state_anchor_path: None,
        git_routing_mount: None,
        git_routing_path: None,
    };
    let mut states = [None; 7];
    for participant in participants
        .iter()
        .filter(|participant| participant.generation_id() == generation.generation_id)
    {
        let index = role_index(participant.role());
        if states[index].replace(participant.state()).is_some() {
            return Err("recovery participant role is duplicated");
        }
        ownership.participants_present += 1;
        let Some(recorded) = participant.ownership() else {
            if participant.state() != ParticipantState::Intent {
                return Err("non-Intent recovery participant has no ownership");
            }
            continue;
        };
        match recorded {
            ParticipantOwnership::Sandbox(sandbox) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::HostCoordinator,
                    &sandbox.host_coordinator,
                    "Sandbox HC identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::RuntimeSupervisor,
                    &sandbox.runtime_supervisor,
                    "Sandbox RS identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::AttachedRunsc,
                    &sandbox.attached_runsc,
                    "attached runsc identity mismatch",
                )?;
            }
            ParticipantOwnership::Network(network) => merge_process(
                &mut ownership,
                RuntimeProcessRole::RuntimeSupervisor,
                &network.runtime_supervisor,
                "Network RS identity mismatch",
            )?,
            ParticipantOwnership::Helper(helper) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::ConnectivityHelper,
                    &helper.connectivity_helper,
                    "connectivity-helper identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::RuntimeSupervisor,
                    &helper.runtime_supervisor,
                    "Helper RS identity mismatch",
                )?;
            }
            ParticipantOwnership::StateAnchor(anchor) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::StateAnchorDaemon,
                    &anchor.daemon,
                    "State Anchor daemon identity mismatch",
                )?;
                let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
                if evidence != (None, None, None) {
                    let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
                        return Err("State Anchor recovery evidence is incomplete");
                    };
                    if ownership
                        .process(RuntimeProcessRole::StateAnchorDaemon)
                        .is_none()
                    {
                        return Err("State Anchor mount has no daemon authority");
                    }
                    ownership.state_anchor_user_namespace = Some(user);
                    ownership.state_anchor_mount_namespace = Some(mount_namespace);
                    ownership.state_anchor_mount = Some(mount);
                    ownership.state_anchor_path.clone_from(&anchor.merged_path);
                    if ownership.state_anchor_path.is_none() {
                        return Err("State Anchor mount path is absent");
                    }
                }
            }
            ParticipantOwnership::GitRouting(routing) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::GitRoutingDaemon,
                    &routing.daemon,
                    "Git-routing daemon identity mismatch",
                )?;
                if let Some(mount) = routing.mount {
                    if ownership
                        .process(RuntimeProcessRole::GitRoutingDaemon)
                        .is_none()
                    {
                        return Err("Git-routing mount has no daemon authority");
                    }
                    ownership.git_routing_mount = Some(mount);
                    ownership.git_routing_path.clone_from(&routing.mountpoint);
                    if ownership.git_routing_path.is_none() {
                        return Err("Git-routing mount path is absent");
                    }
                }
            }
            ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
        }
    }
    require_role_dependencies(states)?;
    require_phase_topology(participants, generation.generation_id, &ownership, states)?;
    Ok(ownership)
}

fn merge_process(
    ownership: &mut RuntimeRecoveryOwnership,
    role: RuntimeProcessRole,
    observed: &PersistedHostProcessOwnership,
    mismatch: &'static str,
) -> Result<(), &'static str> {
    let Ok(identity) = observed.authoritative_pidfs() else {
        return Ok(());
    };
    match ownership.process(role) {
        None => {
            if ownership
                .processes
                .iter()
                .any(|owner| owner.identity == *identity)
            {
                return Err("one process identity is assigned to distinct runtime roles");
            }
            ownership.processes.push(RuntimeProcessOwner {
                role,
                identity: *identity,
            });
        }
        Some(current) if current == *identity => {}
        Some(_) => return Err(mismatch),
    }
    Ok(())
}

fn require_role_dependencies(states: [Option<ParticipantState>; 7]) -> Result<(), &'static str> {
    for (role, required) in [
        (
            ParticipantRole::StateAnchor,
            &[ParticipantRole::PreparedLower, ParticipantRole::Sandbox][..],
        ),
        (
            ParticipantRole::Helper,
            &[ParticipantRole::PreparedLower, ParticipantRole::StateAnchor],
        ),
        (
            ParticipantRole::Network,
            &[
                ParticipantRole::PreparedLower,
                ParticipantRole::StateAnchor,
                ParticipantRole::Helper,
            ],
        ),
        (
            ParticipantRole::Publication,
            &[
                ParticipantRole::PreparedLower,
                ParticipantRole::StateAnchor,
                ParticipantRole::Helper,
                ParticipantRole::Network,
                ParticipantRole::Sandbox,
            ],
        ),
    ] {
        if states[role_index(role)].is_some()
            && required
                .iter()
                .any(|required| states[role_index(*required)].is_none())
        {
            return Err("recovery participant dependency is missing");
        }
    }
    Ok(())
}

fn require_phase_topology(
    participants: &[Participant],
    generation_id: GenerationId,
    ownership: &RuntimeRecoveryOwnership,
    states: [Option<ParticipantState>; 7],
) -> Result<(), &'static str> {
    let sandbox = participants.iter().find(|participant| {
        participant.generation_id() == generation_id
            && participant.role() == ParticipantRole::Sandbox
    });
    if let Some(participant) = sandbox {
        let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else {
            return Err("Sandbox durable intent lacks its immutable launch authority");
        };
        if participant.state() != ParticipantState::Clean
            && (sandbox.container_id.is_none()
                || sandbox.runtime_root.is_none()
                || sandbox.runtime_root_identity.is_none()
                || sandbox.bundle_digest.is_none()
                || sandbox.host_coordinator.authoritative_pidfs().is_err())
        {
            return Err("Sandbox durable launch authority is incomplete");
        }
        if participant.state() == ParticipantState::Committed
            && (sandbox.runtime_supervisor.authoritative_pidfs().is_err()
                || sandbox.attached_runsc.authoritative_pidfs().is_err())
        {
            return Err("committed Sandbox process authority is incomplete");
        }
    }
    let later_runtime_role = [
        ParticipantRole::StateAnchor,
        ParticipantRole::Helper,
        ParticipantRole::Network,
        ParticipantRole::GitRouting,
        ParticipantRole::Publication,
    ]
    .into_iter()
    .any(|role| states[role_index(role)].is_some());
    let sandbox_clean =
        states[role_index(ParticipantRole::Sandbox)] == Some(ParticipantState::Clean);
    if later_runtime_role
        && !sandbox_clean
        && ownership
            .process(RuntimeProcessRole::RuntimeSupervisor)
            .is_none()
    {
        return Err("later runtime phase lacks its Runtime Supervisor authority");
    }
    if states[role_index(ParticipantRole::StateAnchor)].is_some()
        && !sandbox_clean
        && ownership
            .process(RuntimeProcessRole::HostCoordinator)
            .is_none()
    {
        return Err("State Anchor phase lacks generation Host Coordinator authority");
    }
    Ok(())
}

const fn role_index(role: ParticipantRole) -> usize {
    match role {
        ParticipantRole::PreparedLower => 0,
        ParticipantRole::StateAnchor => 1,
        ParticipantRole::GitRouting => 2,
        ParticipantRole::Network => 3,
        ParticipantRole::Helper => 4,
        ParticipantRole::Sandbox => 5,
        ParticipantRole::Publication => 6,
    }
}

#[must_use]
pub fn recovery_role_order() -> [ParticipantRole; 7] {
    [
        ParticipantRole::Publication,
        ParticipantRole::Sandbox,
        ParticipantRole::Helper,
        ParticipantRole::Network,
        ParticipantRole::GitRouting,
        ParticipantRole::StateAnchor,
        ParticipantRole::PreparedLower,
    ]
}

fn reconcile_runtime_before_state_anchor(owners: &mut SealedRuntimeOwners) -> io::Result<()> {
    for role in [
        RuntimeProcessRole::AttachedRunsc,
        RuntimeProcessRole::ConnectivityHelper,
        RuntimeProcessRole::GitRoutingDaemon,
        RuntimeProcessRole::RuntimeSupervisor,
    ] {
        owners.reconcile(role)?;
    }
    Ok(())
}

/// Proves every recorded generation runtime holder and filesystem mount absent.
///
/// # Errors
/// Refuses empty or partial inventory, live HC authority, exact-pidfd action failure, unreadable
/// evidence, or any recorded State Anchor/Git-routing mount that remains present.
pub fn prove_runtime_owners_absent(
    ownership: &RuntimeRecoveryOwnership,
) -> Result<RuntimeOwnersAbsent, io::Error> {
    if ownership.participants_present == 0
        || (ownership.processes.is_empty()
            && ownership.state_anchor_mount.is_none()
            && ownership.git_routing_mount.is_none())
    {
        return Err(io::Error::other("runtime recovery inventory is empty"));
    }
    seal_runtime_owners(ownership, match_recovery_process)?.prove_absent()
}

fn seal_runtime_owners<Open>(
    ownership: &RuntimeRecoveryOwnership,
    mut open: Open,
) -> io::Result<SealedRuntimeOwners>
where
    Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
{
    let mut processes = Vec::with_capacity(ownership.processes.len());
    for owner in &ownership.processes {
        let matched = match open(owner.identity) {
            Ok(PidfsRecoveryMatch::BootMismatch) => SealedProcessMatch::BootMismatch,
            Ok(PidfsRecoveryMatch::Matched(process)) => SealedProcessMatch::Matched(process),
            Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
                SealedProcessMatch::Esrch
            }
            Err(error) => return Err(error),
        };
        processes.push(SealedProcessOwner {
            role: owner.role,
            matched,
            reconciled: false,
        });
    }
    Ok(SealedRuntimeOwners {
        ownership: ownership.clone(),
        processes,
    })
}

fn require_mount_absent(
    path: Option<&HostPath>,
    mount: Option<MountIdentity>,
    label: &str,
) -> io::Result<()> {
    match (path, mount) {
        (None, None) => Ok(()),
        (Some(path), Some(expected)) => {
            let observed =
                current_mount_identity(Path::new(std::ffi::OsStr::from_bytes(path.as_bytes())))?;
            if observed == expected {
                Err(io::Error::other(format!(
                    "{label} exact mount remains present"
                )))
            } else {
                Ok(())
            }
        }
        _ => Err(io::Error::other(format!(
            "{label} recovery mount evidence is incomplete"
        ))),
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ProductionRecoveryBoundary {
    Snapshot,
    Publication,
    Sandbox,
    HelperAndNetwork,
    GitRouting,
    StateAnchor,
    PreparedLower,
    PrepareOperations,
    ArtifactGc,
    FinishGeneration,
}

pub(crate) fn recover_production_before_cold_start(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    prepared: &PreparedLowerStore,
) -> Result<(), String> {
    recover_production_before_cold_start_with(
        configuration,
        store,
        catalog,
        prepared,
        |_, _| Ok(()),
    )
}

fn recover_production_before_cold_start_with<Failpoint>(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    prepared: &PreparedLowerStore,
    failpoint: Failpoint,
) -> Result<(), String>
where
    Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
{
    recover_production_before_cold_start_with_opener(
        configuration,
        store,
        catalog,
        prepared,
        failpoint,
        match_recovery_process,
        |root, container| delete_runsc_container(root, container, std::ffi::OsStr::new("runsc")),
    )
}

fn recover_production_before_cold_start_with_opener<Failpoint, Open, Delete>(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    prepared: &PreparedLowerStore,
    mut failpoint: Failpoint,
    mut open: Open,
    mut delete: Delete,
) -> Result<(), String>
where
    Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
    Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
    Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
    failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
    if snapshot.domain.domain_id != configuration.domain_id() {
        return Err("recovery Domain differs from resolved configuration".into());
    }
    let mut generations = snapshot.generations.clone();
    generations.sort_by_key(|generation| generation.generation_id.get());
    for generation in generations {
        if generation.durable_decision == DurableDecision::Clean {
            continue;
        }
        if generation.configuration_fingerprint != configuration.configuration_fingerprint() {
            return Err("unfinished generation configuration differs".into());
        }
        recover_generation(
            configuration,
            store,
            catalog,
            &generation,
            &mut failpoint,
            &mut open,
            &mut delete,
        )?;
    }
    failpoint(
        ProductionRecoveryBoundary::PrepareOperations,
        GenerationId::new(0),
    )?;
    prepared
        .recover_preparations()
        .map_err(|error| error.to_string())?;
    failpoint(ProductionRecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
    let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
    let selected = catalog_snapshot
        .prepared_artifacts
        .iter()
        .filter(|artifact| {
            !catalog_snapshot
                .artifact_leases
                .iter()
                .any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id)
        })
        .map(|artifact| artifact.prepared_artifact_id)
        .collect::<Vec<_>>();
    prepared
        .gc_artifacts(&selected)
        .map_err(|error| error.to_string())?;
    Ok(())
}

fn recover_generation<Failpoint, Open, Delete>(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    generation: &Generation,
    failpoint: &mut Failpoint,
    open: &mut Open,
    delete: &mut Delete,
) -> Result<(), String>
where
    Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
    Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
    Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
    let generation_id = generation.generation_id;
    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
    if snapshot
        .session_terminals
        .iter()
        .any(|terminal| terminal.generation_id() == generation_id)
        || snapshot
            .mailboxes
            .iter()
            .any(|mailbox| mailbox.generation_id == generation_id)
    {
        return Err("unfinished terminal or mailbox state cannot be recovered safely".into());
    }
    let session_updates = snapshot
        .sessions
        .iter()
        .filter(|session| {
            session.generation_id == generation_id
                && !matches!(
                    session.session_phase,
                    SessionPhase::Closed | SessionPhase::Failed
                )
        })
        .map(|session| {
            let mut failed = session.clone();
            failed.session_phase = SessionPhase::Failed;
            DomainStoreMutation::PutSession(failed)
        })
        .collect::<Vec<_>>();
    if !session_updates.is_empty() {
        store
            .apply(&session_updates)
            .map_err(|error| error.to_string())?;
    }
    let participants = store
        .snapshot()
        .map_err(|error| error.to_string())?
        .participants;
    let ownership = runtime_recovery_ownership(&participants, generation)?;
    let mut owners = seal_runtime_owners(&ownership, open).map_err(|error| error.to_string())?;

    failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
    recover_publication(configuration, store, generation_id)?;

    failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
    reconcile_runtime_before_state_anchor(&mut owners).map_err(|error| error.to_string())?;
    require_sandbox_absent(configuration, store, generation_id, delete)?;
    clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)?;

    failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
    clean_role_after_absence(store, generation_id, ParticipantRole::Helper)?;
    clean_role_after_absence(store, generation_id, ParticipantRole::Network)?;

    failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
    require_mount_absent(
        ownership.git_routing_path.as_ref(),
        ownership.git_routing_mount,
        "Git-routing",
    )
    .map_err(|error| error.to_string())?;
    clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)?;

    failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
    recover_state_anchor(configuration, store, catalog, generation, &mut owners)?;

    failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
    recover_partial_prepared_lower(configuration, store, catalog, generation_id)?;

    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
    if snapshot.participants.iter().any(|participant| {
        participant.generation_id() == generation_id
            && participant.state() != ParticipantState::Clean
    }) {
        return Err("generation retains non-Clean participants after recovery".into());
    }
    failpoint(ProductionRecoveryBoundary::FinishGeneration, generation_id)?;
    let mut clean = generation.clone();
    clean.durable_decision = DurableDecision::Clean;
    store
        .apply(&[DomainStoreMutation::PutGeneration(clean)])
        .map_err(|error| error.to_string())
}

fn recover_publication(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
) -> Result<(), String> {
    let Some(participant) = role(store, generation_id, ParticipantRole::Publication)? else {
        return Ok(());
    };
    if participant.state() == ParticipantState::Clean {
        return Ok(());
    }
    let root = configuration
        .runtime_domain_root()
        .open_authority()
        .map_err(|error| error.to_string())?;
    let Some(ParticipantOwnership::Publication(publication)) = participant.ownership() else {
        if participant.ownership().is_none() {
            require_bare_publication_absent(&root)?;
            return clean_role_after_absence(store, generation_id, ParticipantRole::Publication);
        }
        return Err("publication ownership shape is invalid".into());
    };
    require_publication_labels(publication)?;
    remove_exact_publication_resource(
        &root,
        [STAGED_CONTROL_SOCKET_NAME, CONTROL_SOCKET_NAME],
        publication
            .socket_identity
            .ok_or("publication socket identity is absent")?,
        true,
    )?;
    remove_exact_publication_resource(
        &root,
        [STAGED_PUBLICATION_IDENTITY_NAME, PUBLICATION_IDENTITY_NAME],
        publication
            .identity_file_identity
            .ok_or("publication identity-file identity is absent")?,
        false,
    )?;
    clean_role_after_absence(store, generation_id, ParticipantRole::Publication)
}

fn require_bare_publication_absent(root: &std::os::fd::OwnedFd) -> Result<(), String> {
    for label in [
        STAGED_CONTROL_SOCKET_NAME,
        STAGED_PUBLICATION_IDENTITY_NAME,
        CONTROL_SOCKET_NAME,
        PUBLICATION_IDENTITY_NAME,
    ] {
        match openat(
            root,
            label,
            OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
            Mode::empty(),
        ) {
            Err(rustix::io::Errno::NOENT) => {}
            Err(error) => return Err(error.to_string()),
            Ok(_) => return Err("bare publication intent has a fixed-label artifact".into()),
        }
    }
    Ok(())
}

fn require_publication_labels(
    ownership: &lazaret_core::records::PublicationOwnership,
) -> Result<(), String> {
    for (observed, expected) in [
        (&ownership.staged_socket_path, STAGED_CONTROL_SOCKET_NAME),
        (
            &ownership.staged_identity_path,
            STAGED_PUBLICATION_IDENTITY_NAME,
        ),
        (&ownership.socket_path, CONTROL_SOCKET_NAME),
        (&ownership.identity_path, PUBLICATION_IDENTITY_NAME),
    ] {
        if observed.as_ref().map(HostPath::as_bytes) != Some(expected.as_bytes()) {
            return Err("publication durable label differs from the fixed runtime label".into());
        }
    }
    Ok(())
}

fn remove_exact_publication_resource(
    root: &std::os::fd::OwnedFd,
    labels: [&str; 2],
    expected: MountIdentity,
    socket: bool,
) -> Result<(), String> {
    let mut exact = None;
    for label in labels {
        let descriptor = match openat(
            root,
            label,
            OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
            Mode::empty(),
        ) {
            Ok(descriptor) => descriptor,
            Err(rustix::io::Errno::NOENT) => continue,
            Err(error) => return Err(error.to_string()),
        };
        let stat = fstat(&descriptor).map_err(|error| error.to_string())?;
        let file_type = rustix::fs::FileType::from_raw_mode(stat.st_mode);
        if mount_identity(&descriptor).map_err(|error| error.to_string())? != expected
            || (socket && !file_type.is_socket())
            || (!socket && !file_type.is_file())
            || exact.replace(label).is_some()
        {
            return Err("publication resource changed identity, type, or location".into());
        }
    }
    if let Some(label) = exact {
        let current = openat(
            root,
            label,
            OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
            Mode::empty(),
        )
        .map_err(|error| error.to_string())?;
        let stat = fstat(&current).map_err(|error| error.to_string())?;
        let file_type = rustix::fs::FileType::from_raw_mode(stat.st_mode);
        if mount_identity(&current).map_err(|error| error.to_string())? != expected
            || (socket && !file_type.is_socket())
            || (!socket && !file_type.is_file())
        {
            return Err("publication resource changed before exact removal".into());
        }
        unlinkat(root, label, AtFlags::empty()).map_err(|error| error.to_string())?;
    }
    Ok(())
}

fn require_sandbox_absent(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
    delete: &mut impl FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
) -> Result<(), String> {
    let Some(participant) = role(store, generation_id, ParticipantRole::Sandbox)? else {
        return Ok(());
    };
    if participant.state() == ParticipantState::Clean {
        return Ok(());
    }
    let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else {
        return Ok(());
    };
    let (Some(runtime_root), Some(runtime_root_identity), Some(container_id)) = (
        &sandbox.runtime_root,
        sandbox.runtime_root_identity,
        sandbox.container_id,
    ) else {
        return Err("Sandbox runsc-root recovery authority is incomplete".into());
    };
    let expected_label = Path::new(std::ffi::OsStr::from_bytes(
        configuration
            .runtime_domain_root()
            .canonical_label()
            .as_bytes(),
    ))
    .join("runsc");
    if runtime_root.as_bytes() != expected_label.as_os_str().as_bytes() {
        return Err("Sandbox runsc root differs from the resolved Runtime Domain".into());
    }
    let root = configuration
        .runtime_domain_root()
        .open_beneath(Path::new("runsc"), ConfinedPathType::Directory)
        .map_err(|error| error.to_string())?;
    if mount_identity(&root).map_err(|error| error.to_string())? != runtime_root_identity {
        return Err("Sandbox runsc-root physical identity changed".into());
    }
    let container = hex_lower(container_id.as_bytes());
    match statat(&root, &container, AtFlags::SYMLINK_NOFOLLOW) {
        Err(rustix::io::Errno::NOENT) => {}
        Err(error) => return Err(error.to_string()),
        Ok(stat) => {
            if !rustix::fs::FileType::from_raw_mode(stat.st_mode).is_dir() {
                return Err("recorded runsc container changed type".into());
            }
            delete(&root, &container)?;
            match statat(&root, &container, AtFlags::SYMLINK_NOFOLLOW) {
                Err(rustix::io::Errno::NOENT) => {}
                Err(error) => return Err(error.to_string()),
                Ok(_) => return Err("recorded runsc container remains after exact cleanup".into()),
            }
        }
    }
    Ok(())
}

/// Deletes one exact container beneath a retained, already-verified runsc root descriptor.
///
/// # Errors
/// Returns launch, wrapper, or nonzero runsc failure without weakening the retained root.
pub fn delete_runsc_container(
    root: &std::os::fd::OwnedFd,
    container: &str,
    program: &std::ffi::OsStr,
) -> Result<(), String> {
    fcntl_setfd(root, FdFlags::empty()).map_err(|error| error.to_string())?;
    let root_label = format!("/proc/self/fd/{}", root.as_raw_fd());
    child_broker()
        .run_transient(
            &TargetCommand::new(program)
                .argument(format!("-root={root_label}"))
                .argument("delete")
                .argument("--force")
                .argument(container),
        )
        .map_err(|error| error.to_string())?;
    Ok(())
}

fn recover_state_anchor(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    generation: &Generation,
    owners: &mut SealedRuntimeOwners,
) -> Result<(), String> {
    let Some(anchor) = role(
        store,
        generation.generation_id,
        ParticipantRole::StateAnchor,
    )?
    else {
        return Ok(());
    };
    if anchor.state() == ParticipantState::Clean {
        let prepared = role(
            store,
            generation.generation_id,
            ParticipantRole::PreparedLower,
        )?;
        if prepared
            .is_some_and(|participant| participant.state() == ParticipantState::ReleaseIntent)
        {
            let layout =
                StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
            let coordinator = StateAnchorCoordinator::new(store, catalog);
            return coordinator
                .resume_release_after_checkpoint(generation.generation_id, &layout, || {
                    fresh_absence(store, generation.generation_id, &layout)
                })
                .map_err(|error| error.to_string());
        }
        return Ok(());
    }
    let full_mount = matches!(
        anchor.state(),
        ParticipantState::Prepared | ParticipantState::ReleaseIntent
    ) && owners.ownership.state_anchor_mount.is_some();
    if !full_mount {
        let _ = owners.prove_absent().map_err(|error| error.to_string())?;
        return clean_role_after_absence(
            store,
            generation.generation_id,
            ParticipantRole::StateAnchor,
        );
    }
    let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
    let coordinator = StateAnchorCoordinator::new(store, catalog);
    owners
        .ownership
        .process(RuntimeProcessRole::StateAnchorDaemon)
        .ok_or("State Anchor daemon authority is absent")?;
    let recovered = if let Some(SealedProcessMatch::Matched(process)) =
        owners.process(RuntimeProcessRole::StateAnchorDaemon)
    {
        if process.has_exited().map_err(|error| error.to_string())? {
            let witness = owners.prove_absent().map_err(|error| error.to_string())?;
            coordinator
                .recover_after_runtime_owner_loss(
                    generation.generation_id,
                    &layout,
                    StateAnchorRecoveryOwner::Absent(&witness),
                )
                .map_err(|error| error.to_string())?
        } else {
            coordinator
                .recover_after_runtime_owner_loss(
                    generation.generation_id,
                    &layout,
                    StateAnchorRecoveryOwner::Matched(process),
                )
                .map_err(|error| error.to_string())?
        }
    } else {
        let witness = owners.prove_absent().map_err(|error| error.to_string())?;
        coordinator
            .recover_after_runtime_owner_loss(
                generation.generation_id,
                &layout,
                StateAnchorRecoveryOwner::Absent(&witness),
            )
            .map_err(|error| error.to_string())?
    };
    let (authority, first) = recovered;
    let complete_witness = owners.prove_absent().map_err(|error| error.to_string())?;
    if !complete_witness.ownership().state_anchor_matches(
        authority.daemon(),
        authority.user_namespace(),
        authority.mount_namespace(),
        authority.mount(),
    ) {
        return Err("completed recovery witness changed State Anchor authority".into());
    }
    coordinator
        .complete_release_after_unmount(&authority, &layout, first, || {
            fresh_absence(store, generation.generation_id, &layout)
        })
        .map_err(|error| error.to_string())
}

fn recover_partial_prepared_lower(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    generation_id: GenerationId,
) -> Result<(), String> {
    let Some(prepared) = role(store, generation_id, ParticipantRole::PreparedLower)? else {
        return Ok(());
    };
    if prepared.state() == ParticipantState::Clean {
        return Ok(());
    }
    if prepared.state() == ParticipantState::ReleaseIntent {
        return Err("prepared-lower ReleaseIntent lacks complete State Anchor evidence".into());
    }
    let Some(ParticipantOwnership::PreparedLower(ownership)) = prepared.ownership() else {
        return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower);
    };
    if prepared.state() != ParticipantState::LeaseHeld {
        return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower);
    }
    let artifact = ownership
        .prepared_artifact_id
        .ok_or("LeaseHeld prepared artifact is absent")?;
    let domain_id = store
        .snapshot()
        .map_err(|error| error.to_string())?
        .domain
        .domain_id;
    let lease = ArtifactLease {
        prepared_artifact_id: artifact,
        domain_id,
        generation_id,
    };
    require_exact_lease(catalog, &lease)?;
    let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
    lazaret_state_anchor::require_mount_absent(&layout).map_err(|error| error.to_string())?;
    let release = Participant::snapshot(
        generation_id,
        ParticipantRole::PreparedLower,
        ParticipantState::ReleaseIntent,
        Some(ParticipantOwnership::PreparedLower(
            lazaret_core::records::PreparedLowerOwnership {
                lease_state: Some(ParticipantState::ReleaseIntent),
                ..ownership.clone()
            },
        )),
    )
    .map_err(|_| "partial prepared-lower release snapshot is invalid")?;
    store
        .apply(&[DomainStoreMutation::PutParticipant(release)])
        .map_err(|error| error.to_string())?;
    catalog
        .release_lease_checked(&lease, || {
            let snapshot = store
                .snapshot()
                .map_err(|error| CatalogError::Io(io::Error::other(error)))?;
            let prepared = snapshot
                .participants
                .iter()
                .find(|participant| {
                    participant.generation_id() == generation_id
                        && participant.role() == ParticipantRole::PreparedLower
                })
                .ok_or_else(|| CatalogError::Io(io::Error::other("prepared participant absent")))?;
            if prepared.state() != ParticipantState::ReleaseIntent {
                return Err(CatalogError::Io(io::Error::other(
                    "prepared release intent is absent",
                )));
            }
            lazaret_state_anchor::require_mount_absent(&layout)
                .map_err(|error| CatalogError::Io(io::Error::other(error)))
        })
        .map_err(|error| error.to_string())?;
    clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower)
}

fn require_exact_lease(catalog: &Catalog, expected: &ArtifactLease) -> Result<(), String> {
    match catalog
        .snapshot()
        .map_err(|error| error.to_string())?
        .artifact_leases
        .iter()
        .find(|lease| {
            lease.domain_id == expected.domain_id && lease.generation_id == expected.generation_id
        }) {
        Some(observed) if observed == expected => Ok(()),
        Some(_) => Err("generation catalog lease mismatches durable ownership".into()),
        None => Err("LeaseHeld prepared lower lacks its exact catalog lease".into()),
    }
}

fn fresh_absence(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
    layout: &StateAnchorLayout<'_>,
) -> Result<StateAnchorAbsenceObservation, StateAnchorCoordinatorError> {
    let snapshot = store.snapshot()?;
    let anchor = snapshot
        .participants
        .iter()
        .find(|participant| {
            participant.generation_id() == generation_id
                && participant.role() == ParticipantRole::StateAnchor
        })
        .ok_or(StateAnchorCoordinatorError::InvalidState(
            "State Anchor participant is absent",
        ))?;
    let mut evidence = match anchor.ownership() {
        Some(ParticipantOwnership::StateAnchor(ownership)) => (
            ownership.user_namespace,
            ownership.mount_namespace,
            ownership.mount,
        ),
        _ => (None, None, None),
    };
    if evidence == (None, None, None) {
        evidence = snapshot
            .participants
            .iter()
            .find(|participant| {
                participant.generation_id() == generation_id
                    && participant.role() == ParticipantRole::PreparedLower
            })
            .and_then(Participant::ownership)
            .and_then(|ownership| match ownership {
                ParticipantOwnership::PreparedLower(ownership) => ownership.released_state_anchor,
                _ => None,
            })
            .map_or((None, None, None), |evidence| {
                (
                    Some(evidence.user_namespace),
                    Some(evidence.mount_namespace),
                    Some(evidence.mount),
                )
            });
    }
    let (Some(user_namespace), Some(mount_namespace), Some(mount)) = evidence else {
        return Err(StateAnchorCoordinatorError::InvalidState(
            "State Anchor absence evidence is incomplete",
        ));
    };
    let observed = lazaret_state_anchor::observe_mount_absence(layout, mount)?;
    Ok(StateAnchorAbsenceObservation {
        user_namespace,
        mount_namespace,
        mount,
        observed,
    })
}

fn clean_role_after_absence(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
    role_name: ParticipantRole,
) -> Result<(), String> {
    let Some(current) = role(store, generation_id, role_name)? else {
        return Ok(());
    };
    if current.state() == ParticipantState::Clean {
        return Ok(());
    }
    if current.state() != ParticipantState::ReleaseIntent && current.ownership().is_some() {
        let release = Participant::snapshot(
            generation_id,
            role_name,
            ParticipantState::ReleaseIntent,
            current.ownership().cloned(),
        )
        .map_err(|_| "participant release snapshot is invalid")?;
        store
            .apply(&[DomainStoreMutation::PutParticipant(release)])
            .map_err(|error| error.to_string())?;
    }
    let clean = Participant::snapshot(
        generation_id,
        role_name,
        ParticipantState::Clean,
        Some(ParticipantOwnership::empty(role_name)),
    )
    .map_err(|_| "participant Clean snapshot is invalid")?;
    store
        .apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(clean)])
        .map_err(|error| error.to_string())
}

fn role(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
    role_name: ParticipantRole,
) -> Result<Option<Participant>, String> {
    Ok(store
        .snapshot()
        .map_err(|error| error.to_string())?
        .participants
        .into_iter()
        .find(|participant| {
            participant.generation_id() == generation_id && participant.role() == role_name
        }))
}

fn hex_lower(bytes: &[u8]) -> String {
    const HEX: &[u8; 16] = b"0123456789abcdef";
    let mut output = String::with_capacity(bytes.len() * 2);
    for byte in bytes {
        output.push(char::from(HEX[usize::from(byte >> 4)]));
        output.push(char::from(HEX[usize::from(byte & 0x0f)]));
    }
    output
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::fs::{self, File, OpenOptions};
    use std::os::unix::fs::symlink;
    use std::os::unix::fs::{FileTypeExt, MetadataExt};
    use std::os::unix::net::UnixListener;
    use std::path::PathBuf;

    use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration};
    use lazaret_core::identity::{
        BootId, BundleDigest, ContainerId, CoordinatorId, DeviceId, HostPidLocator, IdMapDigest,
        InodeId, MountId, NetworkSidecarId, PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1,
        PolicyId, PrepareOperationId, PreparedArtifactId, ProcessStartTimeTicks, SandboxId,
        SelectedManifestDigest, SourceIdentity,
    };
    use lazaret_core::records::{
        ArtifactState, DurableDecision, GitRoutingOwnership, HelperOwnership, PrepareOperation,
        PrepareStageState, PreparedArtifact, PreparedLowerOwnership, PublicationOwnership,
        SandboxOwnership, StateAnchorOwnership,
    };
    use lazaret_linux::filesystem::try_exclusive_flock;
    use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;

    use super::*;

    fn process(value: u64) -> PidfsProcessIdentityV1 {
        PidfsProcessIdentityV1::new(
            PidfsProcessAuthorityV1 {
                boot_id: BootId::new([1; 16]),
                pidfs_st_dev: PidfsDeviceId::new(value),
                pidfs_st_ino: PidfsInodeId::new(value),
            },
            HostPidLocator::new(u32::try_from(value).unwrap()),
            ProcessStartTimeTicks::new(value),
        )
    }

    fn generation() -> Generation {
        Generation {
            domain_id: DomainId::new([1; 32]),
            generation_id: GenerationId::new(1),
            configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
            coordinator_id: CoordinatorId::new([3; 32]),
            network_sidecar_id: NetworkSidecarId::new([4; 32]),
            sandbox_id: SandboxId::new([5; 32]),
            durable_decision: DurableDecision::Preparing,
        }
    }

    fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant {
        Participant::snapshot(
            GenerationId::new(1),
            role,
            ParticipantState::Intent,
            Some(ownership),
        )
        .unwrap()
    }

    #[test]
    fn inventory_rejects_default_partial_and_helper_or_fuse_omission() {
        let generation = generation();
        let empty = runtime_recovery_ownership(&[], &generation).unwrap();
        assert!(prove_runtime_owners_absent(&empty).is_err());

        let helper = participant(
            ParticipantRole::Helper,
            ParticipantOwnership::Helper(HelperOwnership {
                network_sidecar_id: None,
                connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
                runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
            }),
        );
        assert!(runtime_recovery_ownership(&[helper], &generation).is_err());

        let incomplete_fuse = participant(
            ParticipantRole::StateAnchor,
            ParticipantOwnership::StateAnchor(StateAnchorOwnership {
                lower_artifact_id: None,
                upper_path: None,
                work_path: None,
                merged_path: Some(HostPath::new(b"/tmp/missing".to_vec())),
                daemon: PersistedHostProcessOwnership::unassigned(),
                user_namespace: Some(NamespaceIdentity {
                    device: DeviceId::new(6),
                    inode: InodeId::new(7),
                }),
                mount_namespace: Some(NamespaceIdentity {
                    device: DeviceId::new(8),
                    inode: InodeId::new(9),
                }),
                mount: Some(MountIdentity {
                    mount_id: MountId::new(10),
                    device: DeviceId::new(11),
                    inode: InodeId::new(12),
                }),
            }),
        );
        let prepared =
            Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower);
        assert!(
            runtime_recovery_ownership(&[prepared.clone(), incomplete_fuse], &generation).is_err()
        );

        let staged_state = participant(
            ParticipantRole::StateAnchor,
            ParticipantOwnership::StateAnchor(StateAnchorOwnership {
                lower_artifact_id: None,
                upper_path: None,
                work_path: None,
                merged_path: None,
                daemon: PersistedHostProcessOwnership::staged(process(5)),
                user_namespace: None,
                mount_namespace: None,
                mount: None,
            }),
        );
        let mut opened = 0;
        let partial = runtime_recovery_ownership(&[prepared, staged_state], &generation);
        assert!(partial.is_err());
        if let Ok(partial) = partial {
            let _ = seal_runtime_owners(&partial, |_| {
                opened += 1;
                Err(io::Error::from_raw_os_error(
                    rustix::io::Errno::SRCH.raw_os_error(),
                ))
            });
        }
        assert_eq!(opened, 0);
    }

    #[test]
    fn complete_boot_mismatch_inventory_proves_only_after_every_holder_and_mount() {
        let generation = generation();
        let prepared =
            Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower);
        let state = participant(
            ParticipantRole::StateAnchor,
            ParticipantOwnership::StateAnchor(StateAnchorOwnership {
                lower_artifact_id: None,
                upper_path: None,
                work_path: None,
                merged_path: None,
                daemon: PersistedHostProcessOwnership::staged(process(5)),
                user_namespace: None,
                mount_namespace: None,
                mount: None,
            }),
        );
        let helper = participant(
            ParticipantRole::Helper,
            ParticipantOwnership::Helper(HelperOwnership {
                network_sidecar_id: None,
                connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
                runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
            }),
        );
        let network = Participant::bare_intent(GenerationId::new(1), ParticipantRole::Network);
        let sandbox = participant(
            ParticipantRole::Sandbox,
            ParticipantOwnership::Sandbox(SandboxOwnership {
                container_id: Some(ContainerId::new([3; 32])),
                runtime_root: Some(HostPath::new(b"/runtime".to_vec())),
                runtime_root_identity: Some(MountIdentity {
                    mount_id: MountId::new(13),
                    device: DeviceId::new(14),
                    inode: InodeId::new(15),
                }),
                bundle_digest: Some(BundleDigest::new([4; 32])),
                host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
                runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)),
                attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
            }),
        );
        let inventory =
            runtime_recovery_ownership(&[prepared, state, helper, network, sandbox], &generation)
                .unwrap();
        assert_eq!(
            prove_runtime_owners_absent(&inventory).unwrap().ownership(),
            &inventory
        );
    }

    #[test]
    fn production_preflight_recovers_full_runtime_then_is_idempotent() {
        let fixture = ProductionRecoveryFixture::new();
        let mut boundaries = Vec::new();
        fixture
            .recover_with(|boundary, generation| {
                boundaries.push((boundary, generation));
                Ok(())
            })
            .unwrap();
        assert!(
            boundaries
                .iter()
                .any(|(boundary, _)| { *boundary == ProductionRecoveryBoundary::StateAnchor })
        );
        assert_eq!(
            fixture.store.snapshot().unwrap().generations[0].durable_decision,
            DurableDecision::Clean
        );
        assert!(
            fixture
                .catalog
                .snapshot()
                .unwrap()
                .artifact_leases
                .is_empty()
        );
        assert!(
            fixture
                .catalog
                .snapshot()
                .unwrap()
                .prepared_artifacts
                .is_empty()
        );
        let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
            fixture
                .configuration
                .runtime_domain_root()
                .canonical_label()
                .as_bytes(),
        ));
        assert!(
            !runtime_domain
                .join("runsc")
                .join(hex_lower(&[6; 32]))
                .exists()
        );

        recover_production_before_cold_start(
            &fixture.configuration,
            &fixture.store,
            &fixture.catalog,
            &fixture.prepared,
        )
        .unwrap();
    }

    #[derive(Clone, Copy)]
    enum PublicationFixtureBoundary {
        TrulyAbsent,
        AfterStagedIdentity,
        AfterStagedSocket,
    }

    #[test]
    fn bare_publication_intent_requires_every_fixed_label_to_be_absent() {
        let absent = ProductionRecoveryFixture::new();
        absent.prepare_bare_publication(PublicationFixtureBoundary::TrulyAbsent);
        recover_publication(&absent.configuration, &absent.store, GenerationId::new(2)).unwrap();
        assert_eq!(absent.publication(2).state(), ParticipantState::Clean);

        let identity = ProductionRecoveryFixture::new();
        identity.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedIdentity);
        assert!(
            recover_publication(
                &identity.configuration,
                &identity.store,
                GenerationId::new(2)
            )
            .is_err()
        );
        assert!(
            identity
                .runtime_domain()
                .join(STAGED_PUBLICATION_IDENTITY_NAME)
                .is_file()
        );
        assert_eq!(identity.publication(2).state(), ParticipantState::Intent);
        assert!(identity.publication(2).ownership().is_none());

        let socket = ProductionRecoveryFixture::new();
        socket.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedSocket);
        assert!(
            recover_publication(&socket.configuration, &socket.store, GenerationId::new(2))
                .is_err()
        );
        assert!(
            socket
                .runtime_domain()
                .join(STAGED_PUBLICATION_IDENTITY_NAME)
                .is_file()
        );
        assert!(
            fs::symlink_metadata(socket.runtime_domain().join(STAGED_CONTROL_SOCKET_NAME))
                .unwrap()
                .file_type()
                .is_socket()
        );
        assert_eq!(socket.publication(2).state(), ParticipantState::Intent);
        assert!(socket.publication(2).ownership().is_none());
    }

    #[test]
    fn production_preflight_retries_every_recovery_boundary() {
        for failed in [
            ProductionRecoveryBoundary::Snapshot,
            ProductionRecoveryBoundary::Publication,
            ProductionRecoveryBoundary::Sandbox,
            ProductionRecoveryBoundary::HelperAndNetwork,
            ProductionRecoveryBoundary::GitRouting,
            ProductionRecoveryBoundary::StateAnchor,
            ProductionRecoveryBoundary::PreparedLower,
            ProductionRecoveryBoundary::PrepareOperations,
            ProductionRecoveryBoundary::ArtifactGc,
            ProductionRecoveryBoundary::FinishGeneration,
        ] {
            let fixture = ProductionRecoveryFixture::new();
            let mut injected = false;
            assert!(
                fixture
                    .recover_with(|boundary, _| {
                        if boundary == failed && !injected {
                            injected = true;
                            Err("injected production recovery boundary".into())
                        } else {
                            Ok(())
                        }
                    },)
                    .is_err()
            );
            assert!(injected);
            fixture.recover_with(|_, _| Ok(())).unwrap();
            assert_eq!(
                fixture.store.snapshot().unwrap().generations[0].durable_decision,
                DurableDecision::Clean
            );
        }
    }

    #[test]
    fn production_preflight_opens_every_exact_owner_once() {
        let fixture = ProductionRecoveryFixture::new();
        let mut opened = BTreeMap::new();
        recover_production_before_cold_start_with_opener(
            &fixture.configuration,
            &fixture.store,
            &fixture.catalog,
            &fixture.prepared,
            |_, _| Ok(()),
            |identity| {
                *opened.entry(identity.host_pid().get()).or_insert(0) += 1;
                Ok(PidfsRecoveryMatch::BootMismatch)
            },
            |root, container| {
                unlinkat(root, container, AtFlags::REMOVEDIR).map_err(|error| error.to_string())
            },
        )
        .unwrap();
        assert_eq!(opened.len(), 6);
        assert!(opened.values().all(|count| *count == 1));
    }

    #[test]
    fn publication_replacement_and_runsc_root_replacement_refuse_untouched() {
        let fixture = ProductionRecoveryFixture::new();
        let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
            fixture
                .configuration
                .runtime_domain_root()
                .canonical_label()
                .as_bytes(),
        ));
        let socket = runtime_domain.join(CONTROL_SOCKET_NAME);
        let old_socket = runtime_domain.join("old-control.sock");
        fs::rename(&socket, &old_socket).unwrap();
        let root = File::open(runtime_domain).unwrap();
        drop(
            UnixListener::bind(format!(
                "/proc/self/fd/{}/{}",
                root.as_raw_fd(),
                CONTROL_SOCKET_NAME
            ))
            .unwrap(),
        );
        assert!(fixture.recover_with(|_, _| Ok(())).is_err());
        assert!(socket.exists());

        let fixture = ProductionRecoveryFixture::new();
        let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
            fixture
                .configuration
                .runtime_domain_root()
                .canonical_label()
                .as_bytes(),
        ));
        let identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME);
        fs::rename(&identity, runtime_domain.join("old-identity")).unwrap();
        symlink("old-identity", &identity).unwrap();
        assert!(fixture.recover_with(|_, _| Ok(())).is_err());
        assert!(
            fs::symlink_metadata(&identity)
                .unwrap()
                .file_type()
                .is_symlink()
        );

        let fixture = ProductionRecoveryFixture::new();
        let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
            fixture
                .configuration
                .runtime_domain_root()
                .canonical_label()
                .as_bytes(),
        ));
        let identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME);
        fs::rename(&identity, runtime_domain.join("old-identity")).unwrap();
        fs::write(&identity, b"replacement").unwrap();
        assert!(fixture.recover_with(|_, _| Ok(())).is_err());
        assert_eq!(fs::read(&identity).unwrap(), b"replacement");

        let fixture = ProductionRecoveryFixture::new();
        let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
            fixture
                .configuration
                .runtime_domain_root()
                .canonical_label()
                .as_bytes(),
        ));
        let runsc = runtime_domain.join("runsc");
        let old_runsc = runtime_domain.join("old-runsc");
        fs::rename(&runsc, &old_runsc).unwrap();
        fs::create_dir(&runsc).unwrap();
        let mut deletes = 0;
        let result = recover_production_before_cold_start_with_opener(
            &fixture.configuration,
            &fixture.store,
            &fixture.catalog,
            &fixture.prepared,
            |_, _| Ok(()),
            |_| Ok(PidfsRecoveryMatch::BootMismatch),
            |_, _| {
                deletes += 1;
                Ok(())
            },
        );
        assert!(result.is_err());
        assert_eq!(deletes, 0);
        assert!(runsc.exists());

        let fixture = ProductionRecoveryFixture::new();
        let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
            fixture
                .configuration
                .runtime_domain_root()
                .canonical_label()
                .as_bytes(),
        ));
        let runsc = runtime_domain.join("runsc");
        fs::rename(&runsc, runtime_domain.join("old-runsc")).unwrap();
        symlink("old-runsc", &runsc).unwrap();
        assert!(fixture.recover_with(|_, _| Ok(())).is_err());
        assert!(
            fs::symlink_metadata(&runsc)
                .unwrap()
                .file_type()
                .is_symlink()
        );
    }

    struct ProductionRecoveryFixture {
        _tree: tempfile::TempDir,
        configuration: ResolvedDomainConfiguration,
        store: HostCoordinatorDomainStore,
        catalog: Catalog,
        prepared: PreparedLowerStore,
    }

    impl ProductionRecoveryFixture {
        fn recover_with<Failpoint>(&self, failpoint: Failpoint) -> Result<(), String>
        where
            Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
        {
            recover_production_before_cold_start_with_opener(
                &self.configuration,
                &self.store,
                &self.catalog,
                &self.prepared,
                failpoint,
                match_recovery_process,
                |root, container| {
                    unlinkat(root, container, AtFlags::REMOVEDIR).map_err(|error| error.to_string())
                },
            )
        }

        #[allow(clippy::too_many_lines)]
        fn new() -> Self {
            let tree = tempfile::tempdir().unwrap();
            let configuration = test_configuration(tree.path());
            let catalog = Catalog::under_storage_root(configuration.storage_root()).unwrap();
            catalog.initialize().unwrap();
            let prepared = PreparedLowerStore::under_storage_root(
                configuration.storage_root(),
                catalog.clone(),
            )
            .unwrap();
            let artifact_name = hex_lower(&[21; 32]);
            let artifact_path = tree
                .path()
                .join("data/prepared-lower/artifacts")
                .join(&artifact_name);
            fs::create_dir(&artifact_path).unwrap();
            let metadata = artifact_path.metadata().unwrap();
            let artifact = PreparedArtifact {
                prepared_artifact_id: PreparedArtifactId::new([21; 32]),
                source_identity: SourceIdentity::new(vec![22]),
                selected_manifest_digest: SelectedManifestDigest::new([23; 32]),
                policy_id: PolicyId::new([24; 32]),
                id_map_digest: IdMapDigest::new([25; 32]),
                path: HostPath::new(format!("artifacts/{artifact_name}").into_bytes()),
                device: DeviceId::new(metadata.dev()),
                inode: InodeId::new(metadata.ino()),
                state: ArtifactState::Available,
            };
            publish_artifact(&catalog, &artifact);
            let domain = lazaret_core::records::Domain {
                domain_id: configuration.domain_id(),
                profile_name: configuration.selected_profile().to_owned(),
                canonical_workspace: configuration.workspace().canonical_label().clone(),
            };
            let lock = OpenOptions::new()
                .create(true)
                .read(true)
                .truncate(false)
                .write(true)
                .open(tree.path().join("domain.lock"))
                .unwrap();
            let store = HostCoordinatorDomainStore::open(
                try_exclusive_flock(lock).unwrap(),
                &tree.path().join("domain.redb"),
                &domain,
            )
            .unwrap();
            let generation = Generation {
                domain_id: domain.domain_id,
                generation_id: GenerationId::new(1),
                configuration_fingerprint: configuration.configuration_fingerprint(),
                coordinator_id: CoordinatorId::new([3; 32]),
                network_sidecar_id: NetworkSidecarId::new([4; 32]),
                sandbox_id: SandboxId::new([5; 32]),
                durable_decision: DurableDecision::None,
            };
            store
                .apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
                .unwrap();
            let mut generation = generation;
            generation.durable_decision = DurableDecision::Preparing;
            store
                .apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
                .unwrap();
            let layout = StateAnchorLayout::create(&configuration).unwrap();
            let mut anchor = layout.ownership_intent(artifact.prepared_artifact_id);
            let mut staged_anchor = anchor.clone();
            staged_anchor.daemon = PersistedHostProcessOwnership::staged(process(5));
            anchor.daemon = PersistedHostProcessOwnership::committed(process(5));
            anchor.user_namespace = Some(NamespaceIdentity {
                device: DeviceId::new(30),
                inode: InodeId::new(31),
            });
            anchor.mount_namespace = Some(NamespaceIdentity {
                device: DeviceId::new(32),
                inode: InodeId::new(33),
            });
            anchor.mount = Some(MountIdentity {
                mount_id: MountId::new(u64::MAX),
                device: DeviceId::new(34),
                inode: InodeId::new(35),
            });
            let prepared_ownership = PreparedLowerOwnership {
                prepared_artifact_id: Some(artifact.prepared_artifact_id),
                source_identity: Some(artifact.source_identity.clone()),
                lease_state: Some(ParticipantState::LeaseHeld),
                path: Some(artifact.path.clone()),
                device: Some(artifact.device),
                inode: Some(artifact.inode),
                released_state_anchor: None,
            };
            let helper = HelperOwnership {
                network_sidecar_id: None,
                connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
                runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
            };
            let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
                configuration
                    .runtime_domain_root()
                    .canonical_label()
                    .as_bytes(),
            ));
            let runsc_root = runtime_domain.join("runsc");
            fs::create_dir(&runsc_root).unwrap();
            let container_id = ContainerId::new([6; 32]);
            fs::create_dir(runsc_root.join(hex_lower(container_id.as_bytes()))).unwrap();
            let sandbox = SandboxOwnership {
                container_id: Some(container_id),
                runtime_root: Some(HostPath::new(runsc_root.as_os_str().as_bytes().to_vec())),
                runtime_root_identity: Some(current_mount_identity(&runsc_root).unwrap()),
                bundle_digest: Some(BundleDigest::new([7; 32])),
                host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
                runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)),
                attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
            };
            let publication_socket = runtime_domain.join(CONTROL_SOCKET_NAME);
            let publication_root = File::open(runtime_domain).unwrap();
            drop(
                UnixListener::bind(format!(
                    "/proc/self/fd/{}/{}",
                    publication_root.as_raw_fd(),
                    CONTROL_SOCKET_NAME
                ))
                .unwrap(),
            );
            let publication_identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME);
            fs::write(&publication_identity, b"identity").unwrap();
            let publication = PublicationOwnership {
                staged_socket_path: Some(HostPath::new(
                    STAGED_CONTROL_SOCKET_NAME.as_bytes().to_vec(),
                )),
                staged_identity_path: Some(HostPath::new(
                    STAGED_PUBLICATION_IDENTITY_NAME.as_bytes().to_vec(),
                )),
                socket_path: Some(HostPath::new(CONTROL_SOCKET_NAME.as_bytes().to_vec())),
                identity_path: Some(HostPath::new(PUBLICATION_IDENTITY_NAME.as_bytes().to_vec())),
                socket_identity: Some(path_identity(runtime_domain, &publication_socket)),
                identity_file_identity: Some(path_identity(runtime_domain, &publication_identity)),
            };
            let git_path = runtime_domain.join("git-routing-fixture");
            fs::create_dir(&git_path).unwrap();
            let git_routing = GitRoutingOwnership {
                mountpoint: Some(HostPath::new(git_path.as_os_str().as_bytes().to_vec())),
                host_root: Some(HostPath::new(b"/git-host".to_vec())),
                isolated_root: Some(HostPath::new(b"/git-isolated".to_vec())),
                daemon: PersistedHostProcessOwnership::staged(process(6)),
                mount: Some(MountIdentity {
                    mount_id: MountId::new(u64::MAX - 1),
                    device: DeviceId::new(40),
                    inode: InodeId::new(41),
                }),
            };
            store
                .apply(&[
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::PreparedLower,
                    )),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::PreparedLower,
                            ParticipantState::Intent,
                            Some(ParticipantOwnership::PreparedLower(
                                PreparedLowerOwnership {
                                    lease_state: Some(ParticipantState::Intent),
                                    ..prepared_ownership.clone()
                                },
                            )),
                        )
                        .unwrap(),
                    ),
                ])
                .unwrap();
            let lease = ArtifactLease {
                prepared_artifact_id: artifact.prepared_artifact_id,
                domain_id: domain.domain_id,
                generation_id: generation.generation_id,
            };
            catalog.acquire_lease_checked(&artifact, &lease).unwrap();
            store
                .apply(&[
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::PreparedLower,
                            ParticipantState::LeaseHeld,
                            Some(ParticipantOwnership::PreparedLower(prepared_ownership)),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::StateAnchor,
                    )),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::StateAnchor,
                            ParticipantState::Intent,
                            Some(ParticipantOwnership::StateAnchor(staged_anchor)),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::StateAnchor,
                            ParticipantState::Intent,
                            Some(ParticipantOwnership::StateAnchor(anchor.clone())),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::StateAnchor,
                            ParticipantState::Prepared,
                            Some(ParticipantOwnership::StateAnchor(anchor)),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::Helper,
                    )),
                    DomainStoreMutation::PutParticipant(participant(
                        ParticipantRole::Helper,
                        ParticipantOwnership::Helper(helper),
                    )),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::Network,
                    )),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::Sandbox,
                    )),
                    DomainStoreMutation::PutParticipant(participant(
                        ParticipantRole::Sandbox,
                        ParticipantOwnership::Sandbox(sandbox),
                    )),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::GitRouting,
                    )),
                    DomainStoreMutation::PutParticipant(participant(
                        ParticipantRole::GitRouting,
                        ParticipantOwnership::GitRouting(git_routing),
                    )),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::Publication,
                    )),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::Publication,
                            ParticipantState::Intent,
                            Some(ParticipantOwnership::Publication(publication.clone())),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::Publication,
                            ParticipantState::Committed,
                            Some(ParticipantOwnership::Publication(publication)),
                        )
                        .unwrap(),
                    ),
                ])
                .unwrap();
            Self {
                _tree: tree,
                configuration,
                store,
                catalog,
                prepared,
            }
        }

        fn runtime_domain(&self) -> PathBuf {
            Path::new(std::ffi::OsStr::from_bytes(
                self.configuration
                    .runtime_domain_root()
                    .canonical_label()
                    .as_bytes(),
            ))
            .to_path_buf()
        }

        fn prepare_bare_publication(&self, boundary: PublicationFixtureBoundary) {
            let runtime_domain = self.runtime_domain();
            fs::remove_file(runtime_domain.join(CONTROL_SOCKET_NAME)).unwrap();
            fs::remove_file(runtime_domain.join(PUBLICATION_IDENTITY_NAME)).unwrap();
            let generation = Generation {
                domain_id: self.configuration.domain_id(),
                generation_id: GenerationId::new(2),
                configuration_fingerprint: self.configuration.configuration_fingerprint(),
                coordinator_id: CoordinatorId::new([13; 32]),
                network_sidecar_id: NetworkSidecarId::new([14; 32]),
                sandbox_id: SandboxId::new([15; 32]),
                durable_decision: DurableDecision::None,
            };
            self.store
                .apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
                .unwrap();
            self.store
                .apply(&[
                    DomainStoreMutation::PutGeneration(Generation {
                        durable_decision: DurableDecision::Preparing,
                        ..generation
                    }),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        GenerationId::new(2),
                        ParticipantRole::Publication,
                    )),
                ])
                .unwrap();
            if !matches!(boundary, PublicationFixtureBoundary::TrulyAbsent) {
                fs::write(
                    runtime_domain.join(STAGED_PUBLICATION_IDENTITY_NAME),
                    b"identity",
                )
                .unwrap();
            }
            if matches!(boundary, PublicationFixtureBoundary::AfterStagedSocket) {
                let root = File::open(runtime_domain).unwrap();
                drop(
                    UnixListener::bind(format!(
                        "/proc/self/fd/{}/{}",
                        root.as_raw_fd(),
                        STAGED_CONTROL_SOCKET_NAME
                    ))
                    .unwrap(),
                );
            }
        }

        fn publication(&self, generation: u64) -> Participant {
            role(
                &self.store,
                GenerationId::new(generation),
                ParticipantRole::Publication,
            )
            .unwrap()
            .unwrap()
        }
    }

    fn path_identity(root: &Path, path: &Path) -> MountIdentity {
        let root_identity = current_mount_identity(root).unwrap();
        let metadata = fs::symlink_metadata(path).unwrap();
        MountIdentity {
            mount_id: root_identity.mount_id,
            device: DeviceId::new(metadata.dev()),
            inode: InodeId::new(metadata.ino()),
        }
    }

    fn publish_artifact(catalog: &Catalog, artifact: &PreparedArtifact) {
        let intent = PrepareOperation {
            operation_id: PrepareOperationId::new([12; 32]),
            expected_artifact_id: artifact.prepared_artifact_id,
            source_identity: artifact.source_identity.clone(),
            selected_manifest_digest: artifact.selected_manifest_digest,
            policy_id: artifact.policy_id,
            id_map_digest: artifact.id_map_digest,
            stage_path: HostPath::new(b"stages/fixture".to_vec()),
            stage_state: PrepareStageState::Intent,
            device: None,
            inode: None,
        };
        catalog.record_prepare(&intent).unwrap();
        let owned = PrepareOperation {
            stage_state: PrepareStageState::Owned,
            device: Some(artifact.device),
            inode: Some(artifact.inode),
            ..intent.clone()
        };
        catalog.bind_prepare_stage(&intent, &owned).unwrap();
        catalog.publish(&owned, artifact).unwrap();
    }

    fn test_configuration(root: &Path) -> ResolvedDomainConfiguration {
        let configuration_directory = root.join("configuration");
        let workspace = root.join("workspace");
        let runtime = root.join("runtime");
        fs::create_dir(&configuration_directory).unwrap();
        fs::create_dir(&workspace).unwrap();
        fs::create_dir(&runtime).unwrap();
        let configuration_path = configuration_directory.join("lazaret.toml");
        fs::write(
            &configuration_path,
            format!(
                "schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 8443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
                root.join("data").display()
            ),
        )
        .unwrap();
        load_trusted_configuration(ConfigurationLoadRequest {
            explicit_configuration_path: Some(configuration_path),
            selected_profile: "default".into(),
            workspace,
            home: root.join("home"),
            xdg_config_home: None,
            xdg_data_home: None,
            xdg_runtime_dir: runtime,
            environment: BTreeMap::new(),
            sandbox_layout: production_sandbox_layout().unwrap(),
        })
        .unwrap()
    }
}