lazaret/crates/lazaret-host-coordinator/src/recovery.rs

use std::io;
use std::os::fd::AsRawFd;
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use std::time::{Duration, Instant};

use lazaret_core::configuration::{ConfinedPathType, ResolvedDomainConfiguration};
use lazaret_core::identity::{
    ConfigurationFingerprint, DomainId, GenerationId, HostPath, MountIdentity, NamespaceIdentity,
    PidfsProcessIdentityV1,
};
use lazaret_core::records::{
    ArtifactLease, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
    ParticipantState, PersistedHostProcessOwnership, SessionPhase,
};
use lazaret_linux::filesystem::{current_mount_identity, mount_identity};
use lazaret_linux::process::{
    PidfsRecoveryMatch, RetainedProcess, TargetCommand, child_broker, match_recovery_process,
};
use lazaret_prepared_lower::PreparedLowerStore;
use lazaret_state_anchor::StateAnchorLayout;
use lazaret_storage_catalog::{Catalog, CatalogError};
use rustix::fs::{AtFlags, Mode, OFlags, fstat, openat, statat, unlinkat};
use rustix::io::{FdFlags, fcntl_setfd};
use rustix::process::Signal;

use crate::control::{
    CONTROL_SOCKET_NAME, PUBLICATION_IDENTITY_NAME, STAGED_CONTROL_SOCKET_NAME,
    STAGED_PUBLICATION_IDENTITY_NAME,
};
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::runtime::StateAnchorAbsenceObservation;
use crate::state_anchor::{
    StateAnchorCoordinator, StateAnchorCoordinatorError, StateAnchorRecoveryOwner,
};
use crate::state_maintenance::recover_state_reset;

const PROCESS_EXIT_TIMEOUT: Duration = Duration::from_secs(5);

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RuntimeProcessRole {
    HostCoordinator,
    RuntimeSupervisor,
    AttachedRunsc,
    ConnectivityHelper,
    StateAnchorDaemon,
    GitRoutingDaemon,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct RuntimeProcessOwner {
    role: RuntimeProcessRole,
    identity: PidfsProcessIdentityV1,
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RuntimeRecoveryOwnership {
    domain_id: DomainId,
    generation_id: GenerationId,
    configuration_fingerprint: ConfigurationFingerprint,
    participants_present: usize,
    processes: Vec<RuntimeProcessOwner>,
    state_anchor_user_namespace: Option<NamespaceIdentity>,
    state_anchor_mount_namespace: Option<NamespaceIdentity>,
    state_anchor_mount: Option<MountIdentity>,
    state_anchor_path: Option<HostPath>,
    git_routing_mount: Option<MountIdentity>,
    git_routing_path: Option<HostPath>,
}

impl RuntimeRecoveryOwnership {
    fn process(&self, role: RuntimeProcessRole) -> Option<PidfsProcessIdentityV1> {
        self.processes
            .iter()
            .find(|owner| owner.role == role)
            .map(|owner| owner.identity)
    }

    pub(crate) fn binding_matches(&self, domain_id: DomainId, generation: &Generation) -> bool {
        self.domain_id == domain_id
            && self.generation_id == generation.generation_id
            && self.configuration_fingerprint == generation.configuration_fingerprint
    }

    pub(crate) fn state_anchor_matches(
        &self,
        daemon: PidfsProcessIdentityV1,
        user_namespace: NamespaceIdentity,
        mount_namespace: NamespaceIdentity,
        mount: MountIdentity,
    ) -> bool {
        self.process(RuntimeProcessRole::StateAnchorDaemon) == Some(daemon)
            && self.state_anchor_user_namespace == Some(user_namespace)
            && self.state_anchor_mount_namespace == Some(mount_namespace)
            && self.state_anchor_mount == Some(mount)
    }
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RuntimeOwnersAbsent {
    ownership: RuntimeRecoveryOwnership,
}

enum SealedProcessMatch {
    BootMismatch,
    Esrch,
    Matched(RetainedProcess),
}

struct SealedProcessOwner {
    role: RuntimeProcessRole,
    matched: SealedProcessMatch,
    reconciled: bool,
}

struct SealedRuntimeOwners {
    ownership: RuntimeRecoveryOwnership,
    processes: Vec<SealedProcessOwner>,
}

impl SealedRuntimeOwners {
    fn process(&self, role: RuntimeProcessRole) -> Option<&SealedProcessMatch> {
        self.processes
            .iter()
            .find(|owner| owner.role == role)
            .map(|owner| &owner.matched)
    }

    fn reconcile(&mut self, role: RuntimeProcessRole) -> io::Result<()> {
        let Some(owner) = self.processes.iter_mut().find(|owner| owner.role == role) else {
            return Ok(());
        };
        if owner.reconciled {
            return Ok(());
        }
        match &owner.matched {
            SealedProcessMatch::BootMismatch | SealedProcessMatch::Esrch => {}
            SealedProcessMatch::Matched(process) if process.has_exited()? => {}
            SealedProcessMatch::Matched(_) if role == RuntimeProcessRole::HostCoordinator => {
                return Err(io::Error::other(
                    "recorded Host Coordinator remains present",
                ));
            }
            SealedProcessMatch::Matched(process) => {
                process.send_signal(Signal::KILL)?;
                let deadline = Instant::now() + PROCESS_EXIT_TIMEOUT;
                while !process.has_exited()? {
                    if Instant::now() >= deadline {
                        return Err(io::Error::other(
                            "exact runtime owner did not exit after pidfd action",
                        ));
                    }
                    std::thread::yield_now();
                }
            }
        }
        owner.reconciled = true;
        Ok(())
    }

    fn prove_absent(&mut self) -> Result<RuntimeOwnersAbsent, io::Error> {
        for role in [
            RuntimeProcessRole::AttachedRunsc,
            RuntimeProcessRole::ConnectivityHelper,
            RuntimeProcessRole::GitRoutingDaemon,
            RuntimeProcessRole::StateAnchorDaemon,
            RuntimeProcessRole::RuntimeSupervisor,
            RuntimeProcessRole::HostCoordinator,
        ] {
            self.reconcile(role)?;
        }
        require_mount_absent(
            self.ownership.state_anchor_path.as_ref(),
            self.ownership.state_anchor_mount,
            "State Anchor",
        )?;
        require_mount_absent(
            self.ownership.git_routing_path.as_ref(),
            self.ownership.git_routing_mount,
            "Git-routing",
        )?;
        Ok(RuntimeOwnersAbsent {
            ownership: self.ownership.clone(),
        })
    }
}

impl RuntimeOwnersAbsent {
    pub(crate) fn ownership(&self) -> &RuntimeRecoveryOwnership {
        &self.ownership
    }
}

/// Extracts one generation's complete exact process and private-mount recovery authority.
///
/// # Errors
/// Refuses duplicate roles, missing role dependencies, unequal owner copies, and every incomplete
/// process, namespace, mount, or path relation required by a recorded participant phase.
#[allow(clippy::too_many_lines)]
pub fn runtime_recovery_ownership(
    participants: &[Participant],
    generation: &Generation,
) -> Result<RuntimeRecoveryOwnership, &'static str> {
    let mut ownership = RuntimeRecoveryOwnership {
        domain_id: generation.domain_id,
        generation_id: generation.generation_id,
        configuration_fingerprint: generation.configuration_fingerprint,
        participants_present: 0,
        processes: Vec::new(),
        state_anchor_user_namespace: None,
        state_anchor_mount_namespace: None,
        state_anchor_mount: None,
        state_anchor_path: None,
        git_routing_mount: None,
        git_routing_path: None,
    };
    let mut states = [None; 7];
    for participant in participants
        .iter()
        .filter(|participant| participant.generation_id() == generation.generation_id)
    {
        let index = role_index(participant.role());
        if states[index].replace(participant.state()).is_some() {
            return Err("recovery participant role is duplicated");
        }
        ownership.participants_present += 1;
        let Some(recorded) = participant.ownership() else {
            if participant.state() != ParticipantState::Intent {
                return Err("non-Intent recovery participant has no ownership");
            }
            continue;
        };
        match recorded {
            ParticipantOwnership::Sandbox(sandbox) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::HostCoordinator,
                    &sandbox.host_coordinator,
                    "Sandbox HC identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::RuntimeSupervisor,
                    &sandbox.runtime_supervisor,
                    "Sandbox RS identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::AttachedRunsc,
                    &sandbox.attached_runsc,
                    "attached runsc identity mismatch",
                )?;
            }
            ParticipantOwnership::Network(network) => merge_process(
                &mut ownership,
                RuntimeProcessRole::RuntimeSupervisor,
                &network.runtime_supervisor,
                "Network RS identity mismatch",
            )?,
            ParticipantOwnership::Helper(helper) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::ConnectivityHelper,
                    &helper.connectivity_helper,
                    "connectivity-helper identity mismatch",
                )?;
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::RuntimeSupervisor,
                    &helper.runtime_supervisor,
                    "Helper RS identity mismatch",
                )?;
            }
            ParticipantOwnership::StateAnchor(anchor) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::StateAnchorDaemon,
                    &anchor.daemon,
                    "State Anchor daemon identity mismatch",
                )?;
                let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
                if evidence != (None, None, None) {
                    let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
                        return Err("State Anchor recovery evidence is incomplete");
                    };
                    if ownership
                        .process(RuntimeProcessRole::StateAnchorDaemon)
                        .is_none()
                    {
                        return Err("State Anchor mount has no daemon authority");
                    }
                    ownership.state_anchor_user_namespace = Some(user);
                    ownership.state_anchor_mount_namespace = Some(mount_namespace);
                    ownership.state_anchor_mount = Some(mount);
                    ownership.state_anchor_path.clone_from(&anchor.merged_path);
                    if ownership.state_anchor_path.is_none() {
                        return Err("State Anchor mount path is absent");
                    }
                }
            }
            ParticipantOwnership::GitRouting(routing) => {
                merge_process(
                    &mut ownership,
                    RuntimeProcessRole::GitRoutingDaemon,
                    &routing.daemon,
                    "Git-routing daemon identity mismatch",
                )?;
                if let Some(mount) = routing.mount {
                    if ownership
                        .process(RuntimeProcessRole::GitRoutingDaemon)
                        .is_none()
                    {
                        return Err("Git-routing mount has no daemon authority");
                    }
                    ownership.git_routing_mount = Some(mount);
                    ownership.git_routing_path.clone_from(&routing.mountpoint);
                    if ownership.git_routing_path.is_none() {
                        return Err("Git-routing mount path is absent");
                    }
                }
            }
            ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
        }
    }
    require_role_dependencies(states)?;
    require_phase_topology(participants, generation.generation_id, &ownership, states)?;
    Ok(ownership)
}

fn merge_process(
    ownership: &mut RuntimeRecoveryOwnership,
    role: RuntimeProcessRole,
    observed: &PersistedHostProcessOwnership,
    mismatch: &'static str,
) -> Result<(), &'static str> {
    let Ok(identity) = observed.authoritative_pidfs() else {
        return Ok(());
    };
    match ownership.process(role) {
        None => {
            if ownership
                .processes
                .iter()
                .any(|owner| owner.identity == *identity)
            {
                return Err("one process identity is assigned to distinct runtime roles");
            }
            ownership.processes.push(RuntimeProcessOwner {
                role,
                identity: *identity,
            });
        }
        Some(current) if current == *identity => {}
        Some(_) => return Err(mismatch),
    }
    Ok(())
}

fn require_role_dependencies(states: [Option<ParticipantState>; 7]) -> Result<(), &'static str> {
    for (role, required) in [
        (
            ParticipantRole::StateAnchor,
            &[ParticipantRole::PreparedLower, ParticipantRole::Sandbox][..],
        ),
        (
            ParticipantRole::Helper,
            &[ParticipantRole::PreparedLower, ParticipantRole::StateAnchor],
        ),
        (
            ParticipantRole::Network,
            &[
                ParticipantRole::PreparedLower,
                ParticipantRole::StateAnchor,
                ParticipantRole::Helper,
            ],
        ),
        (
            ParticipantRole::Publication,
            &[
                ParticipantRole::PreparedLower,
                ParticipantRole::StateAnchor,
                ParticipantRole::Helper,
                ParticipantRole::Network,
                ParticipantRole::Sandbox,
            ],
        ),
    ] {
        if states[role_index(role)].is_some()
            && required
                .iter()
                .any(|required| states[role_index(*required)].is_none())
        {
            return Err("recovery participant dependency is missing");
        }
    }
    Ok(())
}

fn require_phase_topology(
    participants: &[Participant],
    generation_id: GenerationId,
    ownership: &RuntimeRecoveryOwnership,
    states: [Option<ParticipantState>; 7],
) -> Result<(), &'static str> {
    let sandbox = participants.iter().find(|participant| {
        participant.generation_id() == generation_id
            && participant.role() == ParticipantRole::Sandbox
    });
    if let Some(participant) = sandbox {
        let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else {
            return Err("Sandbox durable intent lacks its immutable launch authority");
        };
        if participant.state() != ParticipantState::Clean
            && (sandbox.container_id.is_none()
                || sandbox.runtime_root.is_none()
                || sandbox.runtime_root_identity.is_none()
                || sandbox.bundle_digest.is_none()
                || sandbox.host_coordinator.authoritative_pidfs().is_err())
        {
            return Err("Sandbox durable launch authority is incomplete");
        }
        if participant.state() == ParticipantState::Committed
            && (sandbox.runtime_supervisor.authoritative_pidfs().is_err()
                || sandbox.attached_runsc.authoritative_pidfs().is_err())
        {
            return Err("committed Sandbox process authority is incomplete");
        }
    }
    let later_runtime_role = [
        ParticipantRole::StateAnchor,
        ParticipantRole::Helper,
        ParticipantRole::Network,
        ParticipantRole::GitRouting,
        ParticipantRole::Publication,
    ]
    .into_iter()
    .any(|role| states[role_index(role)].is_some());
    let sandbox_clean =
        states[role_index(ParticipantRole::Sandbox)] == Some(ParticipantState::Clean);
    if later_runtime_role
        && !sandbox_clean
        && ownership
            .process(RuntimeProcessRole::RuntimeSupervisor)
            .is_none()
    {
        return Err("later runtime phase lacks its Runtime Supervisor authority");
    }
    if states[role_index(ParticipantRole::StateAnchor)].is_some()
        && !sandbox_clean
        && ownership
            .process(RuntimeProcessRole::HostCoordinator)
            .is_none()
    {
        return Err("State Anchor phase lacks generation Host Coordinator authority");
    }
    Ok(())
}

const fn role_index(role: ParticipantRole) -> usize {
    match role {
        ParticipantRole::PreparedLower => 0,
        ParticipantRole::StateAnchor => 1,
        ParticipantRole::GitRouting => 2,
        ParticipantRole::Network => 3,
        ParticipantRole::Helper => 4,
        ParticipantRole::Sandbox => 5,
        ParticipantRole::Publication => 6,
    }
}

include!("recovery/reconciliation.rs");
include!("recovery/tests.rs");