lazaret/crates/lazaret-host-coordinator/src/recovery.rs

use std::io;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::FileTypeExt;
use std::path::Path;
use std::time::{Duration, Instant};

use lazaret_core::configuration::ResolvedDomainConfiguration;
use lazaret_core::identity::{
    ConfigurationFingerprint, DomainId, GenerationId, HostPath, MountIdentity, NamespaceIdentity,
    PidfsProcessIdentityV1,
};
use lazaret_core::records::{
    ArtifactLease, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
    ParticipantState, PersistedHostProcessOwnership, SessionPhase,
};
use lazaret_linux::filesystem::current_mount_identity;
use lazaret_linux::process::{
    PidfsRecoveryMatch, TargetCommand, child_broker, match_recovery_process,
};
use lazaret_prepared_lower::PreparedLowerStore;
use lazaret_state_anchor::StateAnchorLayout;
use lazaret_storage_catalog::{Catalog, CatalogError};
use rustix::process::Signal;

use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::runtime::StateAnchorAbsenceObservation;
use crate::state_anchor::{StateAnchorCoordinator, StateAnchorCoordinatorError};

const PROCESS_EXIT_TIMEOUT: Duration = Duration::from_secs(5);

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RuntimeProcessRole {
    HostCoordinator,
    RuntimeSupervisor,
    AttachedRunsc,
    ConnectivityHelper,
    StateAnchorDaemon,
    GitRoutingDaemon,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RuntimeProcessOwner {
    role: RuntimeProcessRole,
    identity: PidfsProcessIdentityV1,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RuntimeRecoveryOwnership {
    domain_id: DomainId,
    generation_id: GenerationId,
    configuration_fingerprint: ConfigurationFingerprint,
    participants_present: usize,
    processes: Vec<RuntimeProcessOwner>,
    state_anchor_user_namespace: Option<NamespaceIdentity>,
    state_anchor_mount_namespace: Option<NamespaceIdentity>,
    state_anchor_mount: Option<MountIdentity>,
    state_anchor_path: Option<HostPath>,
    git_routing_mount: Option<MountIdentity>,
    git_routing_path: Option<HostPath>,
}

impl RuntimeRecoveryOwnership {
    fn process(&self, role: RuntimeProcessRole) -> Option<PidfsProcessIdentityV1> {
        self.processes
            .iter()
            .find(|owner| owner.role == role)
            .map(|owner| owner.identity)
    }

    pub(crate) fn binding_matches(&self, domain_id: DomainId, generation: &Generation) -> bool {
        self.domain_id == domain_id
            && self.generation_id == generation.generation_id
            && self.configuration_fingerprint == generation.configuration_fingerprint
    }

    pub(crate) fn state_anchor_matches(
        &self,
        daemon: PidfsProcessIdentityV1,
        user_namespace: NamespaceIdentity,
        mount_namespace: NamespaceIdentity,
        mount: MountIdentity,
    ) -> bool {
        self.process(RuntimeProcessRole::StateAnchorDaemon) == Some(daemon)
            && self.state_anchor_user_namespace == Some(user_namespace)
            && self.state_anchor_mount_namespace == Some(mount_namespace)
            && self.state_anchor_mount == Some(mount)
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RuntimeOwnersAbsent {
    ownership: RuntimeRecoveryOwnership,
}

impl RuntimeOwnersAbsent {
    pub(crate) fn ownership(&self) -> &RuntimeRecoveryOwnership {
        &self.ownership
    }
}

/// Extracts one generation's complete exact process and private-mount recovery authority.
///
/// # Errors
/// Refuses duplicate roles, missing role dependencies, unequal owner copies, and every incomplete
/// process, namespace, mount, or path relation required by a recorded participant phase.
#[allow(clippy::too_many_lines)]
pub fn runtime_recovery_ownership(
    participants: &[Participant],
    generation: &Generation,
) -> Result<RuntimeRecoveryOwnership, &'static str> {
    let mut ownership = RuntimeRecoveryOwnership {
        domain_id: generation.domain_id,
        generation_id: generation.generation_id,
        configuration_fingerprint: generation.configuration_fingerprint,
        participants_present: 0,
        processes: Vec::new(),
        state_anchor_user_namespace: None,
        state_anchor_mount_namespace: None,
        state_anchor_mount: None,
        state_anchor_path: None,
        git_routing_mount: None,
        git_routing_path: None,
    };
    let mut states = [None; 7];
    for participant in participants
        .iter()
        .filter(|participant| participant.generation_id() == generation.generation_id)
    {
        let index = role_index(participant.role());
        if states[index].replace(participant.state()).is_some() {
            return Err("recovery participant role is duplicated");
        }
        ownership.participants_present += 1;
        let Some(recorded) = participant.ownership() else {
            if participant.state() != ParticipantState::Intent {
                return Err("non-Intent recovery participant has no ownership");
            }
            continue;
        };
        match recorded {
            ParticipantOwnership::Sandbox(sandbox) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::HostCoordinator,
                    &sandbox.host_coordinator,
                    "Sandbox HC identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::RuntimeSupervisor,
                    &sandbox.runtime_supervisor,
                    "Sandbox RS identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::AttachedRunsc,
                    &sandbox.attached_runsc,
                    "attached runsc identity mismatch",
                )?;
            }
            ParticipantOwnership::Network(network) => merge_process(
                &mut ownership,
                RuntimeProcessRole::RuntimeSupervisor,
                &network.runtime_supervisor,
                "Network RS identity mismatch",
            )?,
            ParticipantOwnership::Helper(helper) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::ConnectivityHelper,
                    &helper.connectivity_helper,
                    "connectivity-helper identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::RuntimeSupervisor,
                    &helper.runtime_supervisor,
                    "Helper RS identity mismatch",
                )?;
            }
            ParticipantOwnership::StateAnchor(anchor) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::StateAnchorDaemon,
                    &anchor.daemon,
                    "State Anchor daemon identity mismatch",
                )?;
                let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
                if evidence != (None, None, None) {
                    let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
                        return Err("State Anchor recovery evidence is incomplete");
                    };
                    if ownership
                        .process(RuntimeProcessRole::StateAnchorDaemon)
                        .is_none()
                    {
                        return Err("State Anchor mount has no daemon authority");
                    }
                    ownership.state_anchor_user_namespace = Some(user);
                    ownership.state_anchor_mount_namespace = Some(mount_namespace);
                    ownership.state_anchor_mount = Some(mount);
                    ownership.state_anchor_path.clone_from(&anchor.merged_path);
                    if ownership.state_anchor_path.is_none() {
                        return Err("State Anchor mount path is absent");
                    }
                }
            }
            ParticipantOwnership::GitRouting(routing) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::GitRoutingDaemon,
                    &routing.daemon,
                    "Git-routing daemon identity mismatch",
                )?;
                if let Some(mount) = routing.mount {
                    if ownership
                        .process(RuntimeProcessRole::GitRoutingDaemon)
                        .is_none()
                    {
                        return Err("Git-routing mount has no daemon authority");
                    }
                    ownership.git_routing_mount = Some(mount);
                    ownership.git_routing_path.clone_from(&routing.mountpoint);
                    if ownership.git_routing_path.is_none() {
                        return Err("Git-routing mount path is absent");
                    }
                }
            }
            ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
        }
    }
    require_role_dependencies(states)?;
    Ok(ownership)
}

fn merge_process(
    ownership: &mut RuntimeRecoveryOwnership,
    role: RuntimeProcessRole,
    observed: &PersistedHostProcessOwnership,
    mismatch: &'static str,
) -> Result<(), &'static str> {
    let Ok(identity) = observed.authoritative_pidfs() else {
        return Ok(());
    };
    match ownership.process(role) {
        None => ownership.processes.push(RuntimeProcessOwner {
            role,
            identity: *identity,
        }),
        Some(current) if current == *identity => {}
        Some(_) => return Err(mismatch),
    }
    Ok(())
}

fn require_role_dependencies(states: [Option<ParticipantState>; 7]) -> Result<(), &'static str> {
    for (role, required) in [
        (
            ParticipantRole::StateAnchor,
            &[ParticipantRole::PreparedLower][..],
        ),
        (
            ParticipantRole::Helper,
            &[ParticipantRole::PreparedLower, ParticipantRole::StateAnchor],
        ),
        (
            ParticipantRole::Network,
            &[
                ParticipantRole::PreparedLower,
                ParticipantRole::StateAnchor,
                ParticipantRole::Helper,
            ],
        ),
        (
            ParticipantRole::Sandbox,
            &[
                ParticipantRole::PreparedLower,
                ParticipantRole::StateAnchor,
                ParticipantRole::Helper,
                ParticipantRole::Network,
            ],
        ),
        (
            ParticipantRole::Publication,
            &[
                ParticipantRole::PreparedLower,
                ParticipantRole::StateAnchor,
                ParticipantRole::Helper,
                ParticipantRole::Network,
                ParticipantRole::Sandbox,
            ],
        ),
    ] {
        if states[role_index(role)].is_some()
            && required
                .iter()
                .any(|required| states[role_index(*required)].is_none())
        {
            return Err("recovery participant dependency is missing");
        }
    }
    Ok(())
}

const fn role_index(role: ParticipantRole) -> usize {
    match role {
        ParticipantRole::PreparedLower => 0,
        ParticipantRole::StateAnchor => 1,
        ParticipantRole::GitRouting => 2,
        ParticipantRole::Network => 3,
        ParticipantRole::Helper => 4,
        ParticipantRole::Sandbox => 5,
        ParticipantRole::Publication => 6,
    }
}

#[must_use]
pub fn recovery_role_order() -> [ParticipantRole; 7] {
    [
        ParticipantRole::Publication,
        ParticipantRole::Sandbox,
        ParticipantRole::Helper,
        ParticipantRole::Network,
        ParticipantRole::GitRouting,
        ParticipantRole::StateAnchor,
        ParticipantRole::PreparedLower,
    ]
}

pub(crate) fn reconcile_runtime_before_state_anchor(
    ownership: &RuntimeRecoveryOwnership,
) -> io::Result<()> {
    for role in [
        RuntimeProcessRole::AttachedRunsc,
        RuntimeProcessRole::ConnectivityHelper,
        RuntimeProcessRole::GitRoutingDaemon,
        RuntimeProcessRole::RuntimeSupervisor,
    ] {
        if let Some(identity) = ownership.process(role) {
            reconcile_process(role, identity)?;
        }
    }
    Ok(())
}

/// Proves every recorded generation runtime holder and filesystem mount absent.
///
/// # Errors
/// Refuses empty or partial inventory, live HC authority, exact-pidfd action failure, unreadable
/// evidence, or any recorded State Anchor/Git-routing mount that remains present.
pub fn prove_runtime_owners_absent(
    ownership: &RuntimeRecoveryOwnership,
) -> Result<RuntimeOwnersAbsent, io::Error> {
    if ownership.participants_present == 0
        || (ownership.processes.is_empty()
            && ownership.state_anchor_mount.is_none()
            && ownership.git_routing_mount.is_none())
    {
        return Err(io::Error::other("runtime recovery inventory is empty"));
    }
    for role in [
        RuntimeProcessRole::AttachedRunsc,
        RuntimeProcessRole::ConnectivityHelper,
        RuntimeProcessRole::GitRoutingDaemon,
        RuntimeProcessRole::StateAnchorDaemon,
        RuntimeProcessRole::RuntimeSupervisor,
        RuntimeProcessRole::HostCoordinator,
    ] {
        if let Some(identity) = ownership.process(role) {
            reconcile_process(role, identity)?;
        }
    }
    require_mount_absent(
        ownership.state_anchor_path.as_ref(),
        ownership.state_anchor_mount,
        "State Anchor",
    )?;
    require_mount_absent(
        ownership.git_routing_path.as_ref(),
        ownership.git_routing_mount,
        "Git-routing",
    )?;
    Ok(RuntimeOwnersAbsent {
        ownership: ownership.clone(),
    })
}

fn reconcile_process(role: RuntimeProcessRole, identity: PidfsProcessIdentityV1) -> io::Result<()> {
    match match_recovery_process(identity) {
        Ok(PidfsRecoveryMatch::BootMismatch) => Ok(()),
        Ok(PidfsRecoveryMatch::Matched(process)) if process.has_exited()? => Ok(()),
        Ok(PidfsRecoveryMatch::Matched(process)) => {
            if role == RuntimeProcessRole::HostCoordinator {
                return Err(io::Error::other(
                    "recorded Host Coordinator remains present",
                ));
            }
            process.send_signal(Signal::KILL)?;
            let deadline = Instant::now() + PROCESS_EXIT_TIMEOUT;
            while !process.has_exited()? {
                if Instant::now() >= deadline {
                    return Err(io::Error::other(
                        "exact runtime owner did not exit after pidfd action",
                    ));
                }
                std::thread::yield_now();
            }
            Ok(())
        }
        Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
            Ok(())
        }
        Err(error) => Err(error),
    }
}

fn require_mount_absent(
    path: Option<&HostPath>,
    mount: Option<MountIdentity>,
    label: &str,
) -> io::Result<()> {
    match (path, mount) {
        (None, None) => Ok(()),
        (Some(path), Some(expected)) => {
            let observed =
                current_mount_identity(Path::new(std::ffi::OsStr::from_bytes(path.as_bytes())))?;
            if observed == expected {
                Err(io::Error::other(format!(
                    "{label} exact mount remains present"
                )))
            } else {
                Ok(())
            }
        }
        _ => Err(io::Error::other(format!(
            "{label} recovery mount evidence is incomplete"
        ))),
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ProductionRecoveryBoundary {
    Snapshot,
    Publication,
    Sandbox,
    HelperAndNetwork,
    GitRouting,
    StateAnchor,
    PreparedLower,
    PrepareOperations,
    ArtifactGc,
    FinishGeneration,
}

pub(crate) fn recover_production_before_cold_start(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    prepared: &PreparedLowerStore,
) -> Result<(), String> {
    recover_production_before_cold_start_with(
        configuration,
        store,
        catalog,
        prepared,
        |_, _| Ok(()),
    )
}

fn recover_production_before_cold_start_with<Failpoint>(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    prepared: &PreparedLowerStore,
    mut failpoint: Failpoint,
) -> Result<(), String>
where
    Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
{
    failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
    if snapshot.domain.domain_id != configuration.domain_id() {
        return Err("recovery Domain differs from resolved configuration".into());
    }
    let mut generations = snapshot.generations.clone();
    generations.sort_by_key(|generation| generation.generation_id.get());
    for generation in generations {
        if generation.durable_decision == DurableDecision::Clean {
            continue;
        }
        if generation.configuration_fingerprint != configuration.configuration_fingerprint() {
            return Err("unfinished generation configuration differs".into());
        }
        recover_generation(configuration, store, catalog, &generation, &mut failpoint)?;
    }
    failpoint(
        ProductionRecoveryBoundary::PrepareOperations,
        GenerationId::new(0),
    )?;
    prepared
        .recover_preparations()
        .map_err(|error| error.to_string())?;
    failpoint(ProductionRecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
    let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
    let selected = catalog_snapshot
        .prepared_artifacts
        .iter()
        .filter(|artifact| {
            !catalog_snapshot
                .artifact_leases
                .iter()
                .any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id)
        })
        .map(|artifact| artifact.prepared_artifact_id)
        .collect::<Vec<_>>();
    prepared
        .gc_artifacts(&selected)
        .map_err(|error| error.to_string())?;
    Ok(())
}

fn recover_generation<Failpoint>(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    generation: &Generation,
    failpoint: &mut Failpoint,
) -> Result<(), String>
where
    Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
{
    let generation_id = generation.generation_id;
    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
    if snapshot
        .session_terminals
        .iter()
        .any(|terminal| terminal.generation_id() == generation_id)
        || snapshot
            .mailboxes
            .iter()
            .any(|mailbox| mailbox.generation_id == generation_id)
    {
        return Err("unfinished terminal or mailbox state cannot be recovered safely".into());
    }
    let session_updates = snapshot
        .sessions
        .iter()
        .filter(|session| {
            session.generation_id == generation_id
                && !matches!(
                    session.session_phase,
                    SessionPhase::Closed | SessionPhase::Failed
                )
        })
        .map(|session| {
            let mut failed = session.clone();
            failed.session_phase = SessionPhase::Failed;
            DomainStoreMutation::PutSession(failed)
        })
        .collect::<Vec<_>>();
    if !session_updates.is_empty() {
        store
            .apply(&session_updates)
            .map_err(|error| error.to_string())?;
    }
    let participants = store
        .snapshot()
        .map_err(|error| error.to_string())?
        .participants;
    let ownership = runtime_recovery_ownership(&participants, generation)?;

    failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
    recover_publication(store, generation_id)?;

    failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
    reconcile_runtime_before_state_anchor(&ownership).map_err(|error| error.to_string())?;
    require_sandbox_absent(store, generation_id)?;
    clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)?;

    failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
    clean_role_after_absence(store, generation_id, ParticipantRole::Helper)?;
    clean_role_after_absence(store, generation_id, ParticipantRole::Network)?;

    failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
    require_mount_absent(
        ownership.git_routing_path.as_ref(),
        ownership.git_routing_mount,
        "Git-routing",
    )
    .map_err(|error| error.to_string())?;
    clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)?;

    failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
    recover_state_anchor(configuration, store, catalog, generation, &ownership)?;

    failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
    recover_partial_prepared_lower(configuration, store, catalog, generation_id)?;

    let snapshot = store.snapshot().map_err(|error| error.to_string())?;
    if snapshot.participants.iter().any(|participant| {
        participant.generation_id() == generation_id
            && participant.state() != ParticipantState::Clean
    }) {
        return Err("generation retains non-Clean participants after recovery".into());
    }
    failpoint(ProductionRecoveryBoundary::FinishGeneration, generation_id)?;
    let mut clean = generation.clone();
    clean.durable_decision = DurableDecision::Clean;
    store
        .apply(&[DomainStoreMutation::PutGeneration(clean)])
        .map_err(|error| error.to_string())
}

fn recover_publication(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
) -> Result<(), String> {
    let Some(participant) = role(store, generation_id, ParticipantRole::Publication)? else {
        return Ok(());
    };
    if participant.state() == ParticipantState::Clean {
        return Ok(());
    }
    let Some(ParticipantOwnership::Publication(publication)) = participant.ownership() else {
        if participant.ownership().is_none() {
            return clean_role_after_absence(store, generation_id, ParticipantRole::Publication);
        }
        return Err("publication ownership shape is invalid".into());
    };
    for path in [&publication.socket_path, &publication.identity_path]
        .into_iter()
        .flatten()
    {
        let path = Path::new(std::ffi::OsStr::from_bytes(path.as_bytes()));
        match std::fs::symlink_metadata(path) {
            Err(error) if error.kind() == io::ErrorKind::NotFound => {}
            Ok(metadata) if metadata.file_type().is_socket() || metadata.is_file() => {
                std::fs::remove_file(path).map_err(|error| error.to_string())?;
            }
            Ok(_) => return Err("publication path changed type during recovery".into()),
            Err(error) => return Err(error.to_string()),
        }
    }
    clean_role_after_absence(store, generation_id, ParticipantRole::Publication)
}

fn require_sandbox_absent(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
) -> Result<(), String> {
    let Some(participant) = role(store, generation_id, ParticipantRole::Sandbox)? else {
        return Ok(());
    };
    let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else {
        return Ok(());
    };
    let (Some(runtime_root), Some(container_id)) = (&sandbox.runtime_root, sandbox.container_id)
    else {
        return Ok(());
    };
    let container = hex_lower(container_id.as_bytes());
    let runtime_root = Path::new(std::ffi::OsStr::from_bytes(runtime_root.as_bytes()));
    let path = runtime_root.join(&container);
    if path.exists() {
        let root = runtime_root
            .to_str()
            .ok_or("recorded runsc root is not UTF-8")?;
        child_broker()
            .run_transient(
                &TargetCommand::new("runsc")
                    .argument(format!("-root={root}"))
                    .argument("delete")
                    .argument("--force")
                    .argument(&container),
            )
            .map_err(|error| error.to_string())?;
        if path.exists() {
            return Err("recorded runsc container remains after exact cleanup".into());
        }
    }
    Ok(())
}

fn recover_state_anchor(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    generation: &Generation,
    ownership: &RuntimeRecoveryOwnership,
) -> Result<(), String> {
    let Some(anchor) = role(
        store,
        generation.generation_id,
        ParticipantRole::StateAnchor,
    )?
    else {
        return Ok(());
    };
    if anchor.state() == ParticipantState::Clean {
        let prepared = role(
            store,
            generation.generation_id,
            ParticipantRole::PreparedLower,
        )?;
        if prepared
            .is_some_and(|participant| participant.state() == ParticipantState::ReleaseIntent)
        {
            let layout =
                StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
            let coordinator = StateAnchorCoordinator::new(store, catalog);
            return coordinator
                .resume_release_after_checkpoint(generation.generation_id, &layout, || {
                    fresh_absence(store, generation.generation_id, &layout)
                })
                .map_err(|error| error.to_string());
        }
        return Ok(());
    }
    let full_mount = matches!(
        anchor.state(),
        ParticipantState::Prepared | ParticipantState::ReleaseIntent
    ) && ownership.state_anchor_mount.is_some();
    if !full_mount {
        let _ = prove_runtime_owners_absent(ownership).map_err(|error| error.to_string())?;
        return clean_role_after_absence(
            store,
            generation.generation_id,
            ParticipantRole::StateAnchor,
        );
    }
    let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
    let coordinator = StateAnchorCoordinator::new(store, catalog);
    let daemon = ownership
        .process(RuntimeProcessRole::StateAnchorDaemon)
        .ok_or("State Anchor daemon authority is absent")?;
    let matched_live = matches!(
        match_recovery_process(daemon),
        Ok(PidfsRecoveryMatch::Matched(ref process)) if !process.has_exited().map_err(|error| error.to_string())?
    );
    let witness = if matched_live {
        None
    } else {
        Some(prove_runtime_owners_absent(ownership).map_err(|error| error.to_string())?)
    };
    let (authority, first) = coordinator
        .recover_after_runtime_owner_loss(generation.generation_id, &layout, witness.as_ref())
        .map_err(|error| error.to_string())?;
    let complete_witness =
        prove_runtime_owners_absent(ownership).map_err(|error| error.to_string())?;
    if !complete_witness.ownership().state_anchor_matches(
        authority.daemon(),
        authority.user_namespace(),
        authority.mount_namespace(),
        authority.mount(),
    ) {
        return Err("completed recovery witness changed State Anchor authority".into());
    }
    coordinator
        .complete_release_after_unmount(&authority, &layout, first, || {
            fresh_absence(store, generation.generation_id, &layout)
        })
        .map_err(|error| error.to_string())
}

fn recover_partial_prepared_lower(
    configuration: &ResolvedDomainConfiguration,
    store: &HostCoordinatorDomainStore,
    catalog: &Catalog,
    generation_id: GenerationId,
) -> Result<(), String> {
    let Some(prepared) = role(store, generation_id, ParticipantRole::PreparedLower)? else {
        return Ok(());
    };
    if prepared.state() == ParticipantState::Clean {
        return Ok(());
    }
    if prepared.state() == ParticipantState::ReleaseIntent {
        return Err("prepared-lower ReleaseIntent lacks complete State Anchor evidence".into());
    }
    let Some(ParticipantOwnership::PreparedLower(ownership)) = prepared.ownership() else {
        return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower);
    };
    if prepared.state() != ParticipantState::LeaseHeld {
        return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower);
    }
    let artifact = ownership
        .prepared_artifact_id
        .ok_or("LeaseHeld prepared artifact is absent")?;
    let domain_id = store
        .snapshot()
        .map_err(|error| error.to_string())?
        .domain
        .domain_id;
    let lease = ArtifactLease {
        prepared_artifact_id: artifact,
        domain_id,
        generation_id,
    };
    require_exact_lease(catalog, &lease)?;
    let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
    lazaret_state_anchor::require_mount_absent(&layout).map_err(|error| error.to_string())?;
    let release = Participant::snapshot(
        generation_id,
        ParticipantRole::PreparedLower,
        ParticipantState::ReleaseIntent,
        Some(ParticipantOwnership::PreparedLower(
            lazaret_core::records::PreparedLowerOwnership {
                lease_state: Some(ParticipantState::ReleaseIntent),
                ..ownership.clone()
            },
        )),
    )
    .map_err(|_| "partial prepared-lower release snapshot is invalid")?;
    store
        .apply(&[DomainStoreMutation::PutParticipant(release)])
        .map_err(|error| error.to_string())?;
    catalog
        .release_lease_checked(&lease, || {
            let snapshot = store
                .snapshot()
                .map_err(|error| CatalogError::Io(io::Error::other(error)))?;
            let prepared = snapshot
                .participants
                .iter()
                .find(|participant| {
                    participant.generation_id() == generation_id
                        && participant.role() == ParticipantRole::PreparedLower
                })
                .ok_or_else(|| CatalogError::Io(io::Error::other("prepared participant absent")))?;
            if prepared.state() != ParticipantState::ReleaseIntent {
                return Err(CatalogError::Io(io::Error::other(
                    "prepared release intent is absent",
                )));
            }
            lazaret_state_anchor::require_mount_absent(&layout)
                .map_err(|error| CatalogError::Io(io::Error::other(error)))
        })
        .map_err(|error| error.to_string())?;
    clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower)
}

fn require_exact_lease(catalog: &Catalog, expected: &ArtifactLease) -> Result<(), String> {
    match catalog
        .snapshot()
        .map_err(|error| error.to_string())?
        .artifact_leases
        .iter()
        .find(|lease| {
            lease.domain_id == expected.domain_id && lease.generation_id == expected.generation_id
        }) {
        Some(observed) if observed == expected => Ok(()),
        Some(_) => Err("generation catalog lease mismatches durable ownership".into()),
        None => Err("LeaseHeld prepared lower lacks its exact catalog lease".into()),
    }
}

fn fresh_absence(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
    layout: &StateAnchorLayout<'_>,
) -> Result<StateAnchorAbsenceObservation, StateAnchorCoordinatorError> {
    let snapshot = store.snapshot()?;
    let anchor = snapshot
        .participants
        .iter()
        .find(|participant| {
            participant.generation_id() == generation_id
                && participant.role() == ParticipantRole::StateAnchor
        })
        .ok_or(StateAnchorCoordinatorError::InvalidState(
            "State Anchor participant is absent",
        ))?;
    let mut evidence = match anchor.ownership() {
        Some(ParticipantOwnership::StateAnchor(ownership)) => (
            ownership.user_namespace,
            ownership.mount_namespace,
            ownership.mount,
        ),
        _ => (None, None, None),
    };
    if evidence == (None, None, None) {
        evidence = snapshot
            .participants
            .iter()
            .find(|participant| {
                participant.generation_id() == generation_id
                    && participant.role() == ParticipantRole::PreparedLower
            })
            .and_then(Participant::ownership)
            .and_then(|ownership| match ownership {
                ParticipantOwnership::PreparedLower(ownership) => ownership.released_state_anchor,
                _ => None,
            })
            .map_or((None, None, None), |evidence| {
                (
                    Some(evidence.user_namespace),
                    Some(evidence.mount_namespace),
                    Some(evidence.mount),
                )
            });
    }
    let (Some(user_namespace), Some(mount_namespace), Some(mount)) = evidence else {
        return Err(StateAnchorCoordinatorError::InvalidState(
            "State Anchor absence evidence is incomplete",
        ));
    };
    let observed = lazaret_state_anchor::observe_mount_absence(layout, mount)?;
    Ok(StateAnchorAbsenceObservation {
        user_namespace,
        mount_namespace,
        mount,
        observed,
    })
}

fn clean_role_after_absence(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
    role_name: ParticipantRole,
) -> Result<(), String> {
    let Some(current) = role(store, generation_id, role_name)? else {
        return Ok(());
    };
    if current.state() == ParticipantState::Clean {
        return Ok(());
    }
    if current.state() != ParticipantState::ReleaseIntent && current.ownership().is_some() {
        let release = Participant::snapshot(
            generation_id,
            role_name,
            ParticipantState::ReleaseIntent,
            current.ownership().cloned(),
        )
        .map_err(|_| "participant release snapshot is invalid")?;
        store
            .apply(&[DomainStoreMutation::PutParticipant(release)])
            .map_err(|error| error.to_string())?;
    }
    let clean = Participant::snapshot(
        generation_id,
        role_name,
        ParticipantState::Clean,
        Some(ParticipantOwnership::empty(role_name)),
    )
    .map_err(|_| "participant Clean snapshot is invalid")?;
    store
        .apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(clean)])
        .map_err(|error| error.to_string())
}

fn role(
    store: &HostCoordinatorDomainStore,
    generation_id: GenerationId,
    role_name: ParticipantRole,
) -> Result<Option<Participant>, String> {
    Ok(store
        .snapshot()
        .map_err(|error| error.to_string())?
        .participants
        .into_iter()
        .find(|participant| {
            participant.generation_id() == generation_id && participant.role() == role_name
        }))
}

fn hex_lower(bytes: &[u8]) -> String {
    const HEX: &[u8; 16] = b"0123456789abcdef";
    let mut output = String::with_capacity(bytes.len() * 2);
    for byte in bytes {
        output.push(char::from(HEX[usize::from(byte >> 4)]));
        output.push(char::from(HEX[usize::from(byte & 0x0f)]));
    }
    output
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::fs::{self, OpenOptions};
    use std::os::unix::fs::MetadataExt;

    use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration};
    use lazaret_core::identity::{
        BootId, BundleDigest, ContainerId, CoordinatorId, DeviceId, HostPidLocator, IdMapDigest,
        InodeId, MountId, NetworkSidecarId, PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1,
        PolicyId, PrepareOperationId, PreparedArtifactId, ProcessStartTimeTicks, SandboxId,
        SelectedManifestDigest, SourceIdentity,
    };
    use lazaret_core::records::{
        ArtifactState, DurableDecision, HelperOwnership, PrepareOperation, PrepareStageState,
        PreparedArtifact, PreparedLowerOwnership, SandboxOwnership, StateAnchorOwnership,
    };
    use lazaret_linux::filesystem::try_exclusive_flock;
    use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;

    use super::*;

    fn process(value: u64) -> PidfsProcessIdentityV1 {
        PidfsProcessIdentityV1::new(
            PidfsProcessAuthorityV1 {
                boot_id: BootId::new([1; 16]),
                pidfs_st_dev: PidfsDeviceId::new(value),
                pidfs_st_ino: PidfsInodeId::new(value),
            },
            HostPidLocator::new(u32::try_from(value).unwrap()),
            ProcessStartTimeTicks::new(value),
        )
    }

    fn generation() -> Generation {
        Generation {
            domain_id: DomainId::new([1; 32]),
            generation_id: GenerationId::new(1),
            configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
            coordinator_id: CoordinatorId::new([3; 32]),
            network_sidecar_id: NetworkSidecarId::new([4; 32]),
            sandbox_id: SandboxId::new([5; 32]),
            durable_decision: DurableDecision::Preparing,
        }
    }

    fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant {
        Participant::snapshot(
            GenerationId::new(1),
            role,
            ParticipantState::Intent,
            Some(ownership),
        )
        .unwrap()
    }

    #[test]
    fn inventory_rejects_default_partial_and_helper_or_fuse_omission() {
        let generation = generation();
        let empty = runtime_recovery_ownership(&[], &generation).unwrap();
        assert!(prove_runtime_owners_absent(&empty).is_err());

        let helper = participant(
            ParticipantRole::Helper,
            ParticipantOwnership::Helper(HelperOwnership {
                network_sidecar_id: None,
                connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
                runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
            }),
        );
        assert!(runtime_recovery_ownership(&[helper], &generation).is_err());

        let incomplete_fuse = participant(
            ParticipantRole::StateAnchor,
            ParticipantOwnership::StateAnchor(StateAnchorOwnership {
                lower_artifact_id: None,
                upper_path: None,
                work_path: None,
                merged_path: Some(HostPath::new(b"/tmp/missing".to_vec())),
                daemon: PersistedHostProcessOwnership::unassigned(),
                user_namespace: Some(NamespaceIdentity {
                    device: DeviceId::new(6),
                    inode: InodeId::new(7),
                }),
                mount_namespace: Some(NamespaceIdentity {
                    device: DeviceId::new(8),
                    inode: InodeId::new(9),
                }),
                mount: Some(MountIdentity {
                    mount_id: MountId::new(10),
                    device: DeviceId::new(11),
                    inode: InodeId::new(12),
                }),
            }),
        );
        let prepared =
            Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower);
        assert!(runtime_recovery_ownership(&[prepared, incomplete_fuse], &generation).is_err());
    }

    #[test]
    fn complete_boot_mismatch_inventory_proves_only_after_every_holder_and_mount() {
        let generation = generation();
        let prepared =
            Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower);
        let state = participant(
            ParticipantRole::StateAnchor,
            ParticipantOwnership::StateAnchor(StateAnchorOwnership {
                lower_artifact_id: None,
                upper_path: None,
                work_path: None,
                merged_path: None,
                daemon: PersistedHostProcessOwnership::staged(process(5)),
                user_namespace: None,
                mount_namespace: None,
                mount: None,
            }),
        );
        let helper = participant(
            ParticipantRole::Helper,
            ParticipantOwnership::Helper(HelperOwnership {
                network_sidecar_id: None,
                connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
                runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
            }),
        );
        let network = Participant::bare_intent(GenerationId::new(1), ParticipantRole::Network);
        let sandbox = participant(
            ParticipantRole::Sandbox,
            ParticipantOwnership::Sandbox(SandboxOwnership {
                container_id: Some(ContainerId::new([3; 32])),
                runtime_root: Some(HostPath::new(b"/runtime".to_vec())),
                bundle_digest: Some(BundleDigest::new([4; 32])),
                host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
                runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)),
                attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
            }),
        );
        let inventory =
            runtime_recovery_ownership(&[prepared, state, helper, network, sandbox], &generation)
                .unwrap();
        assert_eq!(
            prove_runtime_owners_absent(&inventory).unwrap().ownership(),
            &inventory
        );
    }

    #[test]
    fn production_preflight_recovers_full_runtime_then_is_idempotent() {
        let fixture = ProductionRecoveryFixture::new();
        let mut boundaries = Vec::new();
        recover_production_before_cold_start_with(
            &fixture.configuration,
            &fixture.store,
            &fixture.catalog,
            &fixture.prepared,
            |boundary, generation| {
                boundaries.push((boundary, generation));
                Ok(())
            },
        )
        .unwrap();
        assert!(
            boundaries
                .iter()
                .any(|(boundary, _)| { *boundary == ProductionRecoveryBoundary::StateAnchor })
        );
        assert_eq!(
            fixture.store.snapshot().unwrap().generations[0].durable_decision,
            DurableDecision::Clean
        );
        assert!(
            fixture
                .catalog
                .snapshot()
                .unwrap()
                .artifact_leases
                .is_empty()
        );
        assert!(
            fixture
                .catalog
                .snapshot()
                .unwrap()
                .prepared_artifacts
                .is_empty()
        );

        recover_production_before_cold_start(
            &fixture.configuration,
            &fixture.store,
            &fixture.catalog,
            &fixture.prepared,
        )
        .unwrap();
    }

    #[test]
    fn production_preflight_retries_every_package_boundary() {
        for failed in [
            ProductionRecoveryBoundary::Snapshot,
            ProductionRecoveryBoundary::Publication,
            ProductionRecoveryBoundary::Sandbox,
            ProductionRecoveryBoundary::HelperAndNetwork,
            ProductionRecoveryBoundary::GitRouting,
            ProductionRecoveryBoundary::StateAnchor,
            ProductionRecoveryBoundary::PreparedLower,
            ProductionRecoveryBoundary::PrepareOperations,
            ProductionRecoveryBoundary::ArtifactGc,
            ProductionRecoveryBoundary::FinishGeneration,
        ] {
            let fixture = ProductionRecoveryFixture::new();
            let mut injected = false;
            assert!(
                recover_production_before_cold_start_with(
                    &fixture.configuration,
                    &fixture.store,
                    &fixture.catalog,
                    &fixture.prepared,
                    |boundary, _| {
                        if boundary == failed && !injected {
                            injected = true;
                            Err("injected production recovery boundary".into())
                        } else {
                            Ok(())
                        }
                    },
                )
                .is_err()
            );
            assert!(injected);
            recover_production_before_cold_start(
                &fixture.configuration,
                &fixture.store,
                &fixture.catalog,
                &fixture.prepared,
            )
            .unwrap();
            assert_eq!(
                fixture.store.snapshot().unwrap().generations[0].durable_decision,
                DurableDecision::Clean
            );
        }
    }

    struct ProductionRecoveryFixture {
        _tree: tempfile::TempDir,
        configuration: ResolvedDomainConfiguration,
        store: HostCoordinatorDomainStore,
        catalog: Catalog,
        prepared: PreparedLowerStore,
    }

    impl ProductionRecoveryFixture {
        #[allow(clippy::too_many_lines)]
        fn new() -> Self {
            let tree = tempfile::tempdir().unwrap();
            let configuration = test_configuration(tree.path());
            let catalog = Catalog::under_storage_root(configuration.storage_root()).unwrap();
            catalog.initialize().unwrap();
            let prepared = PreparedLowerStore::under_storage_root(
                configuration.storage_root(),
                catalog.clone(),
            )
            .unwrap();
            let artifact_name = hex_lower(&[21; 32]);
            let artifact_path = tree
                .path()
                .join("data/prepared-lower/artifacts")
                .join(&artifact_name);
            fs::create_dir(&artifact_path).unwrap();
            let metadata = artifact_path.metadata().unwrap();
            let artifact = PreparedArtifact {
                prepared_artifact_id: PreparedArtifactId::new([21; 32]),
                source_identity: SourceIdentity::new(vec![22]),
                selected_manifest_digest: SelectedManifestDigest::new([23; 32]),
                policy_id: PolicyId::new([24; 32]),
                id_map_digest: IdMapDigest::new([25; 32]),
                path: HostPath::new(format!("artifacts/{artifact_name}").into_bytes()),
                device: DeviceId::new(metadata.dev()),
                inode: InodeId::new(metadata.ino()),
                state: ArtifactState::Available,
            };
            publish_artifact(&catalog, &artifact);
            let domain = lazaret_core::records::Domain {
                domain_id: configuration.domain_id(),
                profile_name: configuration.selected_profile().to_owned(),
                canonical_workspace: configuration.workspace().canonical_label().clone(),
            };
            let lock = OpenOptions::new()
                .create(true)
                .read(true)
                .truncate(false)
                .write(true)
                .open(tree.path().join("domain.lock"))
                .unwrap();
            let store = HostCoordinatorDomainStore::open(
                try_exclusive_flock(lock).unwrap(),
                &tree.path().join("domain.redb"),
                &domain,
            )
            .unwrap();
            let generation = Generation {
                domain_id: domain.domain_id,
                generation_id: GenerationId::new(1),
                configuration_fingerprint: configuration.configuration_fingerprint(),
                coordinator_id: CoordinatorId::new([3; 32]),
                network_sidecar_id: NetworkSidecarId::new([4; 32]),
                sandbox_id: SandboxId::new([5; 32]),
                durable_decision: DurableDecision::None,
            };
            store
                .apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
                .unwrap();
            let mut generation = generation;
            generation.durable_decision = DurableDecision::Preparing;
            store
                .apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
                .unwrap();
            let layout = StateAnchorLayout::create(&configuration).unwrap();
            let mut anchor = layout.ownership_intent(artifact.prepared_artifact_id);
            let mut staged_anchor = anchor.clone();
            staged_anchor.daemon = PersistedHostProcessOwnership::staged(process(5));
            anchor.daemon = PersistedHostProcessOwnership::committed(process(5));
            anchor.user_namespace = Some(NamespaceIdentity {
                device: DeviceId::new(30),
                inode: InodeId::new(31),
            });
            anchor.mount_namespace = Some(NamespaceIdentity {
                device: DeviceId::new(32),
                inode: InodeId::new(33),
            });
            anchor.mount = Some(MountIdentity {
                mount_id: MountId::new(u64::MAX),
                device: DeviceId::new(34),
                inode: InodeId::new(35),
            });
            let prepared_ownership = PreparedLowerOwnership {
                prepared_artifact_id: Some(artifact.prepared_artifact_id),
                source_identity: Some(artifact.source_identity.clone()),
                lease_state: Some(ParticipantState::LeaseHeld),
                path: Some(artifact.path.clone()),
                device: Some(artifact.device),
                inode: Some(artifact.inode),
                released_state_anchor: None,
            };
            let helper = HelperOwnership {
                network_sidecar_id: None,
                connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
                runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
            };
            let sandbox = SandboxOwnership {
                container_id: Some(ContainerId::new([6; 32])),
                runtime_root: Some(HostPath::new(
                    tree.path()
                        .join("missing-runsc")
                        .as_os_str()
                        .as_bytes()
                        .to_vec(),
                )),
                bundle_digest: Some(BundleDigest::new([7; 32])),
                host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
                runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)),
                attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
            };
            store
                .apply(&[
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::PreparedLower,
                    )),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::PreparedLower,
                            ParticipantState::Intent,
                            Some(ParticipantOwnership::PreparedLower(
                                PreparedLowerOwnership {
                                    lease_state: Some(ParticipantState::Intent),
                                    ..prepared_ownership.clone()
                                },
                            )),
                        )
                        .unwrap(),
                    ),
                ])
                .unwrap();
            let lease = ArtifactLease {
                prepared_artifact_id: artifact.prepared_artifact_id,
                domain_id: domain.domain_id,
                generation_id: generation.generation_id,
            };
            catalog.acquire_lease_checked(&artifact, &lease).unwrap();
            store
                .apply(&[
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::PreparedLower,
                            ParticipantState::LeaseHeld,
                            Some(ParticipantOwnership::PreparedLower(prepared_ownership)),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::StateAnchor,
                    )),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::StateAnchor,
                            ParticipantState::Intent,
                            Some(ParticipantOwnership::StateAnchor(staged_anchor)),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::StateAnchor,
                            ParticipantState::Intent,
                            Some(ParticipantOwnership::StateAnchor(anchor.clone())),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(
                        Participant::snapshot(
                            generation.generation_id,
                            ParticipantRole::StateAnchor,
                            ParticipantState::Prepared,
                            Some(ParticipantOwnership::StateAnchor(anchor)),
                        )
                        .unwrap(),
                    ),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::Helper,
                    )),
                    DomainStoreMutation::PutParticipant(participant(
                        ParticipantRole::Helper,
                        ParticipantOwnership::Helper(helper),
                    )),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::Network,
                    )),
                    DomainStoreMutation::PutParticipant(Participant::bare_intent(
                        generation.generation_id,
                        ParticipantRole::Sandbox,
                    )),
                    DomainStoreMutation::PutParticipant(participant(
                        ParticipantRole::Sandbox,
                        ParticipantOwnership::Sandbox(sandbox),
                    )),
                ])
                .unwrap();
            Self {
                _tree: tree,
                configuration,
                store,
                catalog,
                prepared,
            }
        }
    }

    fn publish_artifact(catalog: &Catalog, artifact: &PreparedArtifact) {
        let intent = PrepareOperation {
            operation_id: PrepareOperationId::new([12; 32]),
            expected_artifact_id: artifact.prepared_artifact_id,
            source_identity: artifact.source_identity.clone(),
            selected_manifest_digest: artifact.selected_manifest_digest,
            policy_id: artifact.policy_id,
            id_map_digest: artifact.id_map_digest,
            stage_path: HostPath::new(b"stages/fixture".to_vec()),
            stage_state: PrepareStageState::Intent,
            device: None,
            inode: None,
        };
        catalog.record_prepare(&intent).unwrap();
        let owned = PrepareOperation {
            stage_state: PrepareStageState::Owned,
            device: Some(artifact.device),
            inode: Some(artifact.inode),
            ..intent.clone()
        };
        catalog.bind_prepare_stage(&intent, &owned).unwrap();
        catalog.publish(&owned, artifact).unwrap();
    }

    fn test_configuration(root: &Path) -> ResolvedDomainConfiguration {
        let configuration_directory = root.join("configuration");
        let workspace = root.join("workspace");
        let runtime = root.join("runtime");
        fs::create_dir(&configuration_directory).unwrap();
        fs::create_dir(&workspace).unwrap();
        fs::create_dir(&runtime).unwrap();
        let configuration_path = configuration_directory.join("lazaret.toml");
        fs::write(
            &configuration_path,
            format!(
                "schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 8443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
                root.join("data").display()
            ),
        )
        .unwrap();
        load_trusted_configuration(ConfigurationLoadRequest {
            explicit_configuration_path: Some(configuration_path),
            selected_profile: "default".into(),
            workspace,
            home: root.join("home"),
            xdg_config_home: None,
            xdg_data_home: None,
            xdg_runtime_dir: runtime,
            environment: BTreeMap::new(),
            sandbox_layout: production_sandbox_layout().unwrap(),
        })
        .unwrap()
    }
}