lazaret/crates/lazaret-host-coordinator/src/runtime.rs

use std::fmt;
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::path::Path;
use std::time::{Duration, Instant};

use crate::state_anchor::StateAnchorCoordinator;
use lazaret_core::identity::{
    BundleDigest, ContainerId, GenerationId, HostPath, MountIdentity, NamespaceIdentity,
    PidfsProcessIdentityV1, PreparedArtifactId,
};
use lazaret_core::network::{CanonicalDerivedNetworkPolicy, ReachabilityExpectation};
use lazaret_core::records::{
    HostProcessOwnershipSlot, NetworkReadinessEvidence, Participant, ParticipantOwnership,
    ParticipantRole, ParticipantState, PersistedHostProcessOwnership, ProcessOwnershipCommitAck,
    SandboxOwnership, StagedProcessOwnership,
};
use lazaret_linux::ipc::private_seqpacket_pair;
use lazaret_linux::process::{RetainedProcess, child_broker, current_process_identity};
use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::SandboxHandoffAuthority;
use lazaret_runtime_supervisor::supervision::{
    AttachedRunscSignal, CoordinatorChannel, RuntimeSupervisorLaunch, SupervisorEvent,
};
use rustix::pipe::{PipeFlags, pipe_with};
use rustix::process::Signal;

use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
use crate::git_routing::GitRoutingCoordinator;
use crate::network::{NetworkCoordinator, NetworkCoordinatorError};

const SUPERVISOR_EXIT_TIMEOUT: Duration = Duration::from_secs(5);

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct StateAnchorAbsenceObservation {
    pub user_namespace: NamespaceIdentity,
    pub mount_namespace: NamespaceIdentity,
    pub mount: MountIdentity,
    pub observed: MountIdentity,
}

#[derive(Debug)]
pub enum SandboxCoordinatorError {
    Store(DomainStoreError),
    Network(NetworkCoordinatorError),
    Io(io::Error),
    InvalidState(&'static str),
}

impl fmt::Display for SandboxCoordinatorError {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Store(error) => {
                write!(formatter, "Sandbox Domain Store operation failed: {error}")
            }
            Self::Network(error) => write!(formatter, "Sandbox network operation failed: {error}"),
            Self::Io(error) => write!(formatter, "Sandbox process operation failed: {error}"),
            Self::InvalidState(detail) => write!(formatter, "Sandbox state refused: {detail}"),
        }
    }
}

impl std::error::Error for SandboxCoordinatorError {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Self::Store(error) => Some(error),
            Self::Network(error) => Some(error),
            Self::Io(error) => Some(error),
            Self::InvalidState(_) => None,
        }
    }
}

impl From<DomainStoreError> for SandboxCoordinatorError {
    fn from(error: DomainStoreError) -> Self {
        Self::Store(error)
    }
}

impl From<io::Error> for SandboxCoordinatorError {
    fn from(error: io::Error) -> Self {
        Self::Io(error)
    }
}

impl From<NetworkCoordinatorError> for SandboxCoordinatorError {
    fn from(error: NetworkCoordinatorError) -> Self {
        Self::Network(error)
    }
}

pub struct SandboxCoordinator<'store> {
    domain_store: &'store HostCoordinatorDomainStore,
}

impl<'store> SandboxCoordinator<'store> {
    #[must_use]
    pub const fn new(domain_store: &'store HostCoordinatorDomainStore) -> Self {
        Self { domain_store }
    }

    /// Persists bare Sandbox intent and the immutable physical launch snapshot.
    ///
    /// # Errors
    /// Refuses a missing generation, changed physical identity, or existing nonmatching state.
    pub fn begin_intent(
        &self,
        generation_id: GenerationId,
        container_id: ContainerId,
        runtime_root: &Path,
        runtime_root_identity: MountIdentity,
        bundle_digest: BundleDigest,
        host_coordinator: PidfsProcessIdentityV1,
    ) -> Result<(), SandboxCoordinatorError> {
        let snapshot = self.domain_store.snapshot()?;
        if !snapshot
            .generations
            .iter()
            .any(|generation| generation.generation_id == generation_id)
        {
            return Err(SandboxCoordinatorError::InvalidState(
                "Sandbox generation is absent",
            ));
        }
        let ownership = SandboxOwnership {
            container_id: Some(container_id),
            runtime_root: Some(HostPath::new(runtime_root.as_os_str().as_bytes().to_vec())),
            runtime_root_identity: Some(runtime_root_identity),
            bundle_digest: Some(bundle_digest),
            host_coordinator: PersistedHostProcessOwnership::committed(host_coordinator),
            runtime_supervisor: PersistedHostProcessOwnership::unassigned(),
            attached_runsc: PersistedHostProcessOwnership::unassigned(),
        };
        let intent = sandbox_participant(generation_id, ParticipantState::Intent, ownership)?;
        match sandbox_role(&snapshot.participants, generation_id) {
            None => self.domain_store.apply(&[
                DomainStoreMutation::PutParticipant(Participant::bare_intent(
                    generation_id,
                    ParticipantRole::Sandbox,
                )),
                DomainStoreMutation::PutParticipant(intent),
            ])?,
            Some(current) if current == &intent => {}
            Some(_) => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Sandbox intent already differs",
                ));
            }
        }
        Ok(())
    }

    /// # Errors
    /// Refuses a mismatched RS tuple or any skipped ownership phase.
    pub fn stage_runtime_supervisor(
        &self,
        staged: StagedProcessOwnership,
    ) -> Result<(), SandboxCoordinatorError> {
        self.require_staged(staged, HostProcessOwnershipSlot::SandboxRuntimeSupervisor)?;
        let (current, mut ownership) = self.intent_ownership(staged.generation_id())?;
        let expected = PersistedHostProcessOwnership::staged(staged.identity());
        if ownership.runtime_supervisor == expected {
            return Ok(());
        }
        if !ownership.runtime_supervisor.is_unassigned() {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor ownership changed",
            ));
        }
        ownership.runtime_supervisor = expected;
        self.put_intent(current.generation_id(), ownership)
    }

    /// # Errors
    /// Refuses an RS staged identity that is not durably exact.
    pub fn acknowledge_runtime_supervisor(
        &self,
        staged: StagedProcessOwnership,
    ) -> Result<ProcessOwnershipCommitAck, SandboxCoordinatorError> {
        self.require_staged(staged, HostProcessOwnershipSlot::SandboxRuntimeSupervisor)?;
        let (_, ownership) = self.intent_ownership(staged.generation_id())?;
        if ownership.runtime_supervisor != PersistedHostProcessOwnership::staged(staged.identity())
        {
            return Err(SandboxCoordinatorError::InvalidState(
                "staged Runtime Supervisor ownership is not durable",
            ));
        }
        Ok(ProcessOwnershipCommitAck::v1(staged))
    }

    /// Commits the executing RS and durably stages its exact attached-runsc child.
    ///
    /// # Errors
    /// Refuses any tuple, phase, or immutable-evidence mismatch.
    pub fn stage_attached_runsc(
        &self,
        staged: StagedProcessOwnership,
    ) -> Result<ProcessOwnershipCommitAck, SandboxCoordinatorError> {
        self.require_staged(staged, HostProcessOwnershipSlot::SandboxAttachedRunsc)?;
        let (current, mut ownership) = self.intent_ownership(staged.generation_id())?;
        let rs = *ownership
            .runtime_supervisor
            .authoritative_pidfs()
            .map_err(|_| {
                SandboxCoordinatorError::InvalidState("Runtime Supervisor is not staged")
            })?;
        if !ownership.runtime_supervisor.is_staged() {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor readiness cannot skip staging",
            ));
        }
        if !ownership.attached_runsc.is_unassigned() {
            return Err(SandboxCoordinatorError::InvalidState(
                "attached runsc ownership changed",
            ));
        }
        ownership.runtime_supervisor = PersistedHostProcessOwnership::committed(rs);
        ownership.attached_runsc = PersistedHostProcessOwnership::staged(staged.identity());
        self.put_intent(current.generation_id(), ownership)?;
        Ok(ProcessOwnershipCommitAck::v1(staged))
    }

    /// Promotes the exact attached runsc only after the RS reports target readiness.
    ///
    /// # Errors
    /// Refuses any incomplete physical snapshot or changed staged identity.
    pub fn commit_ready(
        &self,
        staged: StagedProcessOwnership,
    ) -> Result<(), SandboxCoordinatorError> {
        self.require_staged(staged, HostProcessOwnershipSlot::SandboxAttachedRunsc)?;
        let (_, mut ownership) = self.intent_ownership(staged.generation_id())?;
        if ownership.attached_runsc != PersistedHostProcessOwnership::staged(staged.identity())
            || !ownership.runtime_supervisor.is_current_committed()
            || !ownership.host_coordinator.is_current_committed()
        {
            return Err(SandboxCoordinatorError::InvalidState(
                "Sandbox readiness does not match staged ownership",
            ));
        }
        ownership.attached_runsc = PersistedHostProcessOwnership::committed(staged.identity());
        self.domain_store
            .apply(&[DomainStoreMutation::PutParticipant(sandbox_participant(
                staged.generation_id(),
                ParticipantState::Committed,
                ownership,
            )?)])?;
        Ok(())
    }

    /// # Errors
    /// Refuses any Sandbox participant that is not exactly Committed.
    pub fn begin_release(
        &self,
        generation_id: GenerationId,
    ) -> Result<(), SandboxCoordinatorError> {
        let current = self.participant(generation_id)?;
        if current.state() != ParticipantState::Committed {
            return Err(SandboxCoordinatorError::InvalidState(
                "Sandbox is not committed",
            ));
        }
        let ownership =
            current
                .ownership()
                .cloned()
                .ok_or(SandboxCoordinatorError::InvalidState(
                    "Sandbox ownership is absent",
                ))?;
        self.domain_store
            .apply(&[DomainStoreMutation::PutParticipant(
                Participant::snapshot(
                    generation_id,
                    ParticipantRole::Sandbox,
                    ParticipantState::ReleaseIntent,
                    Some(ownership),
                )
                .map_err(|_| SandboxCoordinatorError::InvalidState("Sandbox release is invalid"))?,
            )])?;
        Ok(())
    }

    /// # Errors
    /// Refuses cleanup without exact `ReleaseIntent` and observed runtime absence.
    pub fn complete_clean_after_absence(
        &self,
        generation_id: GenerationId,
    ) -> Result<(), SandboxCoordinatorError> {
        let current = self.participant(generation_id)?;
        if current.state() != ParticipantState::ReleaseIntent {
            return Err(SandboxCoordinatorError::InvalidState(
                "Sandbox release intent is absent",
            ));
        }
        let clean = Participant::snapshot(
            generation_id,
            ParticipantRole::Sandbox,
            ParticipantState::Clean,
            Some(ParticipantOwnership::empty(ParticipantRole::Sandbox)),
        )
        .map_err(|_| SandboxCoordinatorError::InvalidState("Sandbox Clean is invalid"))?;
        self.domain_store
            .apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(clean)])?;
        Ok(())
    }

    fn require_staged(
        &self,
        staged: StagedProcessOwnership,
        slot: HostProcessOwnershipSlot,
    ) -> Result<(), SandboxCoordinatorError> {
        let snapshot = self.domain_store.snapshot()?;
        let generation = snapshot
            .generations
            .iter()
            .find(|generation| generation.generation_id == staged.generation_id())
            .ok_or(SandboxCoordinatorError::InvalidState(
                "staged Sandbox generation is absent",
            ))?;
        if staged.domain_id() != snapshot.domain.domain_id
            || staged.coordinator_id() != generation.coordinator_id
            || staged.slot() != slot
        {
            return Err(SandboxCoordinatorError::InvalidState(
                "staged Sandbox ownership tuple mismatches generation",
            ));
        }
        Ok(())
    }

    fn intent_ownership(
        &self,
        generation_id: GenerationId,
    ) -> Result<(Participant, SandboxOwnership), SandboxCoordinatorError> {
        let current = self.participant(generation_id)?;
        if current.state() != ParticipantState::Intent {
            return Err(SandboxCoordinatorError::InvalidState(
                "Sandbox acquisition is not in Intent",
            ));
        }
        let Some(ParticipantOwnership::Sandbox(ownership)) = current.ownership() else {
            return Err(SandboxCoordinatorError::InvalidState(
                "Sandbox ownership shape is invalid",
            ));
        };
        Ok((current.clone(), ownership.clone()))
    }

    fn put_intent(
        &self,
        generation_id: GenerationId,
        ownership: SandboxOwnership,
    ) -> Result<(), SandboxCoordinatorError> {
        self.domain_store
            .apply(&[DomainStoreMutation::PutParticipant(sandbox_participant(
                generation_id,
                ParticipantState::Intent,
                ownership,
            )?)])?;
        Ok(())
    }

    fn participant(
        &self,
        generation_id: GenerationId,
    ) -> Result<Participant, SandboxCoordinatorError> {
        sandbox_role(&self.domain_store.snapshot()?.participants, generation_id)
            .cloned()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Sandbox participant is absent",
            ))
    }
}

pub struct AwaitingAttachedRunsc {
    supervisor: Option<RetainedProcess>,
    control: Option<CoordinatorChannel>,
    lifetime: Option<std::os::fd::OwnedFd>,
    staged: Option<StagedProcessOwnership>,
    helper_staged: Option<StagedProcessOwnership>,
    git_routing_required: bool,
    git_routing_mount: Option<MountIdentity>,
    network_evidence: Option<NetworkReadinessEvidence>,
    runsc_command: lazaret_runtime_supervisor::sandbox::RunscCommand,
}

impl AwaitingAttachedRunsc {
    #[must_use]
    pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
        self.supervisor.as_ref().map(RetainedProcess::identity)
    }

    /// Requests RS-owned State Anchor acquisition and commits exact staged and mount evidence.
    ///
    /// # Errors
    /// Refuses missing intent, event mismatch, changed ownership, or channel failure.
    pub fn mount_state_anchor(
        &mut self,
        coordinator: &StateAnchorCoordinator<'_>,
        prepared_artifact_id: PreparedArtifactId,
    ) -> Result<MountIdentity, SandboxCoordinatorError> {
        let control = self
            .control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?;
        control.mount_state_anchor()?;
        let staged = match control.receive()? {
            SupervisorEvent::StateAnchorStaged(staged) => staged,
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not stage the State Anchor daemon",
                ));
            }
        };
        coordinator
            .stage_daemon(staged)
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        let acknowledgment = coordinator
            .acknowledge_staged_daemon(staged)
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        control.acknowledge_state_anchor(acknowledgment)?;
        let (ready_staged, user_namespace, mount_namespace, mount) = match control.receive()? {
            SupervisorEvent::StateAnchorReady {
                staged,
                user_namespace,
                mount_namespace,
                mount,
            } => (staged, user_namespace, mount_namespace, mount),
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not report State Anchor readiness",
                ));
            }
        };
        if ready_staged != staged {
            return Err(SandboxCoordinatorError::InvalidState(
                "State Anchor readiness changed daemon ownership",
            ));
        }
        let readiness =
            lazaret_state_anchor::VerifiedStateAnchorReadiness::from_parts(ready_staged, mount);
        coordinator
            .commit_daemon_ready(readiness, user_namespace, mount_namespace)
            .and_then(|()| {
                coordinator.commit_prepared(staged.generation_id(), prepared_artifact_id, mount)
            })
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        Ok(mount)
    }

    /// Requests generation-local routing and commits exact staged and mount evidence.
    ///
    /// # Errors
    /// Refuses missing intent, event mismatch, changed ownership, or channel failure.
    pub fn start_git_routing(
        &mut self,
        coordinator: &GitRoutingCoordinator<'_>,
    ) -> Result<MountIdentity, SandboxCoordinatorError> {
        if !self.git_routing_required || self.git_routing_mount.is_some() {
            return Err(SandboxCoordinatorError::InvalidState(
                "Git routing is not awaiting start",
            ));
        }
        let control = self
            .control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?;
        control.start_git_routing()?;
        let staged = match control.receive()? {
            SupervisorEvent::GitRoutingStaged(staged) => staged,
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not stage the Git router",
                ));
            }
        };
        coordinator
            .stage_daemon(staged)
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        let acknowledgment = coordinator
            .acknowledge_daemon(staged)
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        control.acknowledge_git_routing(acknowledgment)?;
        let (ready_staged, mount) = match control.receive()? {
            SupervisorEvent::GitRoutingReady { staged, mount } => (staged, mount),
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not report Git-routing readiness",
                ));
            }
        };
        if ready_staged != staged {
            return Err(SandboxCoordinatorError::InvalidState(
                "Git-routing readiness changed daemon ownership",
            ));
        }
        coordinator
            .commit_ready(staged, mount)
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        self.git_routing_mount = Some(mount);
        Ok(mount)
    }

    /// Releases the RS-owned sidecar install path after exact State Anchor readiness.
    ///
    /// # Errors
    /// Refuses an unexpected RS response or channel failure.
    pub fn start_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
        let control = self
            .control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?;
        control.start_sidecar()?;
        if control.receive()? == SupervisorEvent::SidecarStarted {
            Ok(())
        } else {
            Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not start the sidecar path",
            ))
        }
    }

    /// Launches the RS through the HC broker and commits its staged ownership before execution.
    ///
    /// # Errors
    /// Refuses any pipe, channel, broker, staging, or acknowledgment failure.
    pub fn launch(
        coordinator: &SandboxCoordinator<'_>,
        launch: &RuntimeSupervisorLaunch,
    ) -> Result<Self, SandboxCoordinatorError> {
        if launch.network_sidecar_id().is_some() {
            return Err(SandboxCoordinatorError::InvalidState(
                "networked Runtime Supervisor requires helper intent",
            ));
        }
        Self::launch_inner(coordinator, None, launch)
    }

    /// Launches the RS only after independent durable helper intent is recorded.
    ///
    /// # Errors
    /// Refuses a non-network launch or any sandbox/helper staging failure.
    pub fn launch_networked(
        coordinator: &SandboxCoordinator<'_>,
        network: &NetworkCoordinator<'_>,
        launch: &RuntimeSupervisorLaunch,
    ) -> Result<Self, SandboxCoordinatorError> {
        if launch.network_sidecar_id().is_none() {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor launch has no network policy",
            ));
        }
        Self::launch_inner(coordinator, Some(network), launch)
    }

    fn launch_inner(
        coordinator: &SandboxCoordinator<'_>,
        network: Option<&NetworkCoordinator<'_>>,
        launch: &RuntimeSupervisorLaunch,
    ) -> Result<Self, SandboxCoordinatorError> {
        let parent = current_process_identity()?;
        let (coordinator_control, supervisor_control) = private_seqpacket_pair()?;
        let (supervisor_lifetime, coordinator_lifetime) =
            pipe_with(PipeFlags::CLOEXEC).map_err(io::Error::from)?;
        let process = child_broker().acquire_with_stdio(
            parent,
            launch.ownership(),
            launch.target(),
            launch.child_standard_io(supervisor_control, supervisor_lifetime)?,
            |staged| {
                coordinator
                    .stage_runtime_supervisor(staged)
                    .map_err(io::Error::other)?;
                if let Some(network) = network {
                    network
                        .begin_helper_intent(
                            staged.generation_id(),
                            launch.network_sidecar_id().ok_or_else(|| {
                                io::Error::other("network sidecar identity is absent")
                            })?,
                            staged.identity(),
                        )
                        .map_err(io::Error::other)?;
                }
                Ok(())
            },
            |staged| {
                coordinator
                    .acknowledge_runtime_supervisor(staged)
                    .map_err(io::Error::other)
            },
        )?;
        Ok(Self {
            supervisor: Some(process),
            control: Some(CoordinatorChannel::new(coordinator_control)),
            lifetime: Some(coordinator_lifetime),
            staged: None,
            helper_staged: None,
            git_routing_required: launch.has_git_routing(),
            git_routing_mount: None,
            network_evidence: None,
            runsc_command: launch.runsc_command().clone(),
        })
    }

    /// Receives, persists, and acknowledges the exact helper child acquisition.
    ///
    /// # Errors
    /// Refuses unexpected events, changed ownership, or a missing network launch.
    pub fn receive_helper_staged(
        &mut self,
        network: &NetworkCoordinator<'_>,
    ) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
        if self.helper_staged.is_some() || self.network_evidence.is_some() {
            return Err(SandboxCoordinatorError::InvalidState(
                "helper stage was already received",
            ));
        }
        let control = self
            .control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?;
        let staged = match control.receive()? {
            SupervisorEvent::HelperStaged(staged) => staged,
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not stage the helper first",
                ));
            }
        };
        let runtime_supervisor = self
            .supervisor
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor handle is absent",
            ))?
            .identity();
        network.stage_helper(staged, runtime_supervisor)?;
        let acknowledgment = network.acknowledge_helper(staged)?;
        control.acknowledge_helper(acknowledgment)?;
        self.helper_staged = Some(staged);
        Ok(staged)
    }

    /// Commits helper readiness, independently observes the namespace, and validates attestation.
    ///
    /// # Errors
    /// Refuses event ordering, namespace mutation, or any attestation mismatch.
    pub fn receive_network_readiness<BuildProbeRequest>(
        &mut self,
        network: &NetworkCoordinator<'_>,
        policy: &CanonicalDerivedNetworkPolicy,
        build_probe_request: BuildProbeRequest,
        expected_reachability: &ReachabilityExpectation,
    ) -> Result<NetworkReadinessEvidence, SandboxCoordinatorError>
    where
        BuildProbeRequest: FnOnce(NamespaceIdentity) -> NetworkProbeRequest,
    {
        let staged = self
            .helper_staged
            .ok_or(SandboxCoordinatorError::InvalidState(
                "helper ownership was not staged",
            ))?;
        let control = self
            .control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?;
        if control.receive()? != SupervisorEvent::HelperReady(staged) {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor helper readiness mismatched ownership",
            ));
        }
        network.commit_helper(staged)?;
        let attestation = match control.receive()? {
            SupervisorEvent::NetworkAttestation(attestation) => attestation,
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not provide network attestation",
                ));
            }
        };
        let namespace = network.begin_network_intent(
            attestation.generation_id,
            policy.digest(),
            attestation.runtime_supervisor,
        )?;
        network.validate_attestation(policy, &attestation)?;
        let probe_request = build_probe_request(namespace);
        if probe_request.domain_id != attestation.domain_id
            || probe_request.generation_id != attestation.generation_id
            || probe_request.enforcement_namespace != attestation.enforcement_namespace
        {
            return Err(SandboxCoordinatorError::InvalidState(
                "network probe request mismatches attestation",
            ));
        }
        control.acknowledge_network_attestation(probe_request)?;
        let probe = match control.receive()? {
            SupervisorEvent::NetworkProbe(probe) => probe,
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not return the network probe",
                ));
            }
        };
        let evidence =
            network.commit_readiness(policy, attestation, probe, expected_reachability)?;
        control.acknowledge_network_readiness()?;
        self.network_evidence = Some(evidence.clone());
        Ok(evidence)
    }

    /// Receives and durably stages the exact RS child without releasing its barrier.
    ///
    /// # Errors
    /// Refuses unexpected, duplicate, or mismatched messages and durable-state failures.
    pub fn receive_staged(
        &mut self,
        coordinator: &SandboxCoordinator<'_>,
    ) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
        if self.staged.is_some() {
            return Err(SandboxCoordinatorError::InvalidState(
                "attached runsc stage was already received",
            ));
        }
        if self.helper_staged.is_some() && self.network_evidence.is_none() {
            return Err(SandboxCoordinatorError::InvalidState(
                "attached runsc cannot start before network attestation",
            ));
        }
        if self.git_routing_required && self.git_routing_mount.is_none() {
            return Err(SandboxCoordinatorError::InvalidState(
                "attached runsc cannot start before Git-routing readiness",
            ));
        }
        let event = self
            .control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?
            .receive()?;
        let staged = match event {
            SupervisorEvent::Staged(staged) => staged,
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor sent readiness before staged ownership",
                ));
            }
        };
        let acknowledgment = coordinator.stage_attached_runsc(staged)?;
        self.staged = Some(acknowledgment.committed());
        Ok(staged)
    }

    /// Releases the exact pending runsc barrier with its durable typed acknowledgment.
    ///
    /// # Errors
    /// Refuses a different or absent staged tuple or channel failure.
    pub fn acknowledge(
        mut self,
        staged: StagedProcessOwnership,
        handoff: &SandboxHandoffAuthority<'_>,
    ) -> Result<AwaitingSandboxReadiness, SandboxCoordinatorError> {
        if self.staged != Some(staged) {
            return Err(SandboxCoordinatorError::InvalidState(
                "attached runsc acknowledgment does not match pending stage",
            ));
        }
        if staged.domain_id() != handoff.domain_id()
            || !handoff.matches_command(&self.runsc_command)
        {
            return Err(SandboxCoordinatorError::InvalidState(
                "sandbox handoff authority does not match attached runsc",
            ));
        }
        handoff
            .revalidate()
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        self.control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?
            .acknowledge_staged(ProcessOwnershipCommitAck::v1(staged))?;
        Ok(AwaitingSandboxReadiness {
            supervisor: self.supervisor.take(),
            control: self.control.take(),
            lifetime: self.lifetime.take(),
            staged,
            network_evidence: self.network_evidence.take(),
        })
    }

    /// # Errors
    /// Returns an error if the exact RS cannot be terminated and reaped.
    pub fn abort(mut self) -> Result<(), SandboxCoordinatorError> {
        self.lifetime.take();
        cleanup_supervisor(&mut self.supervisor)?;
        Ok(())
    }
}

impl Drop for AwaitingAttachedRunsc {
    fn drop(&mut self) {
        self.lifetime.take();
        let _ = cleanup_supervisor(&mut self.supervisor);
    }
}

pub struct AwaitingSandboxReadiness {
    supervisor: Option<RetainedProcess>,
    control: Option<CoordinatorChannel>,
    lifetime: Option<std::os::fd::OwnedFd>,
    staged: StagedProcessOwnership,
    network_evidence: Option<NetworkReadinessEvidence>,
}

impl AwaitingSandboxReadiness {
    /// Commits only the exact ready attached runsc after RS capability removal.
    ///
    /// # Errors
    /// Refuses a mismatched readiness tuple, durable commit failure, or channel failure.
    pub fn commit(
        self,
        coordinator: &SandboxCoordinator<'_>,
    ) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
        if self.network_evidence.is_some() {
            return Err(SandboxCoordinatorError::InvalidState(
                "networked sandbox requires a successful sandbox-view probe",
            ));
        }
        self.commit_after_probe(coordinator, None)
    }

    /// Commits the Sandbox only after the enforcement-namespace probe is durably accepted.
    ///
    /// # Errors
    /// Refuses missing attestation, failed/mutated probe evidence, or any runtime mismatch.
    pub fn commit_networked(
        mut self,
        coordinator: &SandboxCoordinator<'_>,
    ) -> Result<(CommittedSandboxRuntime, NetworkReadinessEvidence), SandboxCoordinatorError> {
        let evidence =
            self.network_evidence
                .take()
                .ok_or(SandboxCoordinatorError::InvalidState(
                    "network readiness evidence is absent",
                ))?;
        let runtime = self.commit_after_probe(coordinator, Some(evidence.clone()))?;
        Ok((runtime, evidence))
    }

    fn commit_after_probe(
        mut self,
        coordinator: &SandboxCoordinator<'_>,
        network_evidence: Option<NetworkReadinessEvidence>,
    ) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
        let control = self
            .control
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor control channel is absent",
            ))?;
        let event = control.receive()?;
        let staged = match event {
            SupervisorEvent::Ready { staged, .. } => staged,
            SupervisorEvent::Refused(detail) => {
                return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
            }
            _ => {
                return Err(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor did not report sandbox readiness",
                ));
            }
        };
        if staged != self.staged {
            return Err(SandboxCoordinatorError::InvalidState(
                "sandbox readiness changed attached-runsc ownership",
            ));
        }
        coordinator.commit_ready(staged)?;
        control.acknowledge_sandbox_committed()?;
        Ok(CommittedSandboxRuntime {
            generation_id: staged.generation_id(),
            supervisor: self.supervisor.take(),
            control: self
                .control
                .take()
                .ok_or(SandboxCoordinatorError::InvalidState(
                    "Runtime Supervisor control channel is absent",
                ))?,
            lifetime: self.lifetime.take(),
            network_evidence,
        })
    }
}

impl Drop for AwaitingSandboxReadiness {
    fn drop(&mut self) {
        self.lifetime.take();
        let _ = cleanup_supervisor(&mut self.supervisor);
    }
}

pub struct CommittedSandboxRuntime {
    generation_id: GenerationId,
    supervisor: Option<RetainedProcess>,
    control: CoordinatorChannel,
    lifetime: Option<std::os::fd::OwnedFd>,
    network_evidence: Option<NetworkReadinessEvidence>,
}

impl CommittedSandboxRuntime {
    fn receive_runtime_event(&mut self) -> Result<SupervisorEvent, SandboxCoordinatorError> {
        loop {
            let event = self.control.receive()?;
            if event == SupervisorEvent::HelperExited {
                self.network_evidence = None;
            } else {
                return Ok(event);
            }
        }
    }

    #[must_use]
    pub const fn network_evidence(&self) -> Option<&NetworkReadinessEvidence> {
        self.network_evidence.as_ref()
    }

    /// Withdraws live readiness after the RS reports exact helper-only loss.
    ///
    /// # Errors
    /// Refuses any event other than connectivity-helper loss.
    pub fn wait_for_helper_exit(&mut self) -> Result<(), SandboxCoordinatorError> {
        if self.control.receive()? != SupervisorEvent::HelperExited {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not report helper-only loss",
            ));
        }
        self.network_evidence = None;
        Ok(())
    }
    #[must_use]
    pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
        self.supervisor.as_ref().map(RetainedProcess::identity)
    }

    /// # Errors
    /// Returns an error if the exact retained RS pidfd action fails.
    pub fn signal_runtime_supervisor(&self, signal: Signal) -> Result<(), SandboxCoordinatorError> {
        self.supervisor
            .as_ref()
            .ok_or(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor handle is absent",
            ))?
            .send_signal(signal)?;
        Ok(())
    }

    /// # Errors
    /// Returns an error if the RS cannot apply the exact retained-runsc pidfd action.
    pub fn signal_attached_runsc(
        &mut self,
        signal: AttachedRunscSignal,
    ) -> Result<(), SandboxCoordinatorError> {
        self.control.signal_attached_runsc(signal)?;
        if self.receive_runtime_event()? == SupervisorEvent::SignalApplied(signal) {
            Ok(())
        } else {
            Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not confirm the exact runsc action",
            ))
        }
    }

    /// Stops and durably cleans only the exact attached sandbox runtime.
    ///
    /// # Errors
    /// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
    pub fn stop_sandbox(
        &mut self,
        coordinator: &SandboxCoordinator<'_>,
    ) -> Result<(), SandboxCoordinatorError> {
        coordinator.begin_release(self.generation_id)?;
        self.control.stop_sandbox()?;
        if self.receive_runtime_event()? != SupervisorEvent::SandboxStopped {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not stop the sandbox",
            ));
        }
        coordinator.complete_clean_after_absence(self.generation_id)
    }

    /// Stops exact generation-local Git routing after sandbox absence.
    ///
    /// # Errors
    /// Refuses missing release intent, changed mount evidence, or an unexpected RS response.
    pub fn stop_git_routing(
        &mut self,
        coordinator: &GitRoutingCoordinator<'_>,
    ) -> Result<(), SandboxCoordinatorError> {
        let expected = coordinator
            .begin_release(self.generation_id)
            .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
        self.control.stop_git_routing()?;
        match self.receive_runtime_event()? {
            SupervisorEvent::GitRoutingStopped { mount } if mount == expected => coordinator
                .complete_clean(self.generation_id, mount)
                .map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error))),
            SupervisorEvent::Refused(detail) => {
                Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
            }
            _ => Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not stop exact Git routing",
            )),
        }
    }

    /// Stops only the RS-owned network resources after sandbox absence.
    ///
    /// # Errors
    /// Refuses an unexpected RS response or channel failure.
    pub fn stop_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
        self.control.stop_sidecar()?;
        if self.receive_runtime_event()? == SupervisorEvent::SidecarStopped {
            self.network_evidence = None;
            Ok(())
        } else {
            Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not stop the sidecar",
            ))
        }
    }

    /// Requests exact RS-owned State Anchor unmount after dependent runtime resources stop.
    ///
    /// # Errors
    /// Refuses an unexpected RS response or channel failure.
    pub fn unmount_state_anchor(
        &mut self,
        expected_mount: MountIdentity,
    ) -> Result<(), SandboxCoordinatorError> {
        self.control.unmount_state_anchor()?;
        match self.receive_runtime_event()? {
            SupervisorEvent::StateAnchorUnmounted { mount } if mount == expected_mount => Ok(()),
            _ => Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not confirm the exact State Anchor unmount",
            )),
        }
    }

    /// Requests a fresh namespace-local observation of exact State Anchor absence.
    ///
    /// # Errors
    /// Refuses an RS refusal, echoed mount, mismatched authority, or channel failure.
    pub fn observe_state_anchor_absence(
        &mut self,
        user_namespace: NamespaceIdentity,
        mount_namespace: NamespaceIdentity,
        mount: MountIdentity,
    ) -> Result<StateAnchorAbsenceObservation, SandboxCoordinatorError> {
        self.control
            .observe_state_anchor_absence(user_namespace, mount_namespace, mount)?;
        match self.receive_runtime_event()? {
            SupervisorEvent::StateAnchorAbsenceObserved {
                user_namespace: observed_user_namespace,
                mount_namespace: observed_mount_namespace,
                mount: observed_mount,
                observed,
            } if observed_user_namespace == user_namespace
                && observed_mount_namespace == mount_namespace
                && observed_mount == mount
                && observed != mount =>
            {
                Ok(StateAnchorAbsenceObservation {
                    user_namespace: observed_user_namespace,
                    mount_namespace: observed_mount_namespace,
                    mount: observed_mount,
                    observed,
                })
            }
            SupervisorEvent::Refused(detail) => {
                Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
            }
            _ => Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not provide exact State Anchor absence evidence",
            )),
        }
    }

    /// Gracefully exits and reaps the resource-empty exact Runtime Supervisor.
    ///
    /// # Errors
    /// Refuses an unexpected response or an RS that does not exit.
    pub fn graceful_exit(mut self) -> Result<(), SandboxCoordinatorError> {
        self.control.exit()?;
        if self.receive_runtime_event()? != SupervisorEvent::Exited {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not acknowledge graceful exit",
            ));
        }
        self.lifetime.take();
        wait_and_reap_supervisor(&mut self.supervisor)
    }

    /// Waits for an exact attached-runsc loss report and reaps the retained RS.
    ///
    /// # Errors
    /// Refuses any other event or an RS that does not then exit.
    pub fn wait_for_attached_runsc_exit(mut self) -> Result<(), SandboxCoordinatorError> {
        if self.receive_runtime_event()? != SupervisorEvent::AttachedRunscExited {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not report attached-runsc loss",
            ));
        }
        self.lifetime.take();
        wait_and_reap_supervisor(&mut self.supervisor)
    }

    /// Waits for an externally stopped exact RS handle to exit and reaps it.
    ///
    /// # Errors
    /// Refuses an RS that remains present beyond the bounded observation.
    pub fn wait_for_runtime_supervisor_exit(mut self) -> Result<(), SandboxCoordinatorError> {
        self.lifetime.take();
        wait_and_reap_supervisor(&mut self.supervisor)
    }

    /// Performs ordered normal stop and reaps the same retained RS handle.
    ///
    /// # Errors
    /// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
    pub fn stop(
        mut self,
        coordinator: &SandboxCoordinator<'_>,
    ) -> Result<(), SandboxCoordinatorError> {
        coordinator.begin_release(self.generation_id)?;
        self.control.stop()?;
        if self.receive_runtime_event()? != SupervisorEvent::Stopped {
            return Err(SandboxCoordinatorError::InvalidState(
                "Runtime Supervisor did not complete sandbox stop",
            ));
        }
        self.lifetime.take();
        wait_and_reap_supervisor(&mut self.supervisor)?;
        coordinator.complete_clean_after_absence(self.generation_id)
    }
}

impl Drop for CommittedSandboxRuntime {
    fn drop(&mut self) {
        self.lifetime.take();
        let _ = cleanup_supervisor(&mut self.supervisor);
    }
}

fn sandbox_participant(
    generation_id: GenerationId,
    state: ParticipantState,
    ownership: SandboxOwnership,
) -> Result<Participant, SandboxCoordinatorError> {
    Participant::snapshot(
        generation_id,
        ParticipantRole::Sandbox,
        state,
        Some(ParticipantOwnership::Sandbox(ownership)),
    )
    .map_err(|_| SandboxCoordinatorError::InvalidState("Sandbox participant shape is invalid"))
}

fn sandbox_role(participants: &[Participant], generation_id: GenerationId) -> Option<&Participant> {
    participants.iter().find(|participant| {
        participant.generation_id() == generation_id
            && participant.role() == ParticipantRole::Sandbox
    })
}

fn cleanup_supervisor(
    supervisor: &mut Option<RetainedProcess>,
) -> Result<(), SandboxCoordinatorError> {
    let Some(process) = supervisor.take() else {
        return Ok(());
    };
    if !process.has_exited()? {
        process.send_signal(Signal::KILL)?;
    }
    child_broker().reap(process)?;
    Ok(())
}

fn wait_and_reap_supervisor(
    supervisor: &mut Option<RetainedProcess>,
) -> Result<(), SandboxCoordinatorError> {
    let Some(process) = supervisor.take() else {
        return Err(SandboxCoordinatorError::InvalidState(
            "Runtime Supervisor handle is absent",
        ));
    };
    let deadline = Instant::now() + SUPERVISOR_EXIT_TIMEOUT;
    while !process.has_exited()? && Instant::now() < deadline {
        std::thread::sleep(Duration::from_millis(5));
    }
    if !process.has_exited()? {
        process.send_signal(Signal::KILL)?;
        child_broker().reap(process)?;
        return Err(SandboxCoordinatorError::InvalidState(
            "Runtime Supervisor did not exit after sandbox stop",
        ));
    }
    child_broker().reap(process)?;
    Ok(())
}

#[cfg(test)]
mod tests;