lazaret/crates/lazaret-runtime-supervisor/src/supervision.rs

use std::ffi::{OsStr, OsString};
use std::fs::File;
use std::io::{self, Read};
use std::os::fd::{AsFd, AsRawFd, OwnedFd};
use std::os::unix::ffi::{OsStrExt, OsStringExt};
use std::os::unix::fs::FileExt;
use std::path::PathBuf;
use std::time::{Duration, Instant};

use lazaret_core::codec::{decode, encode};
use lazaret_core::identity::{
    ConfigurationFingerprint, CoordinatorId, DerivedPolicyDigest, DomainId, GenerationId,
    MountIdentity, NamespaceIdentity, NetworkSidecarId,
};
use lazaret_core::network::CanonicalDerivedNetworkPolicy;
use lazaret_core::records::{
    HostProcessOwnershipSlot, NetworkReadinessAttestation, ProcessOwnershipCommitAck,
    StagedProcessOwnership,
};
use lazaret_linux::filesystem::{mount_identity, require_write_sealed, sealed_memfd};
use lazaret_linux::process::{
    ChildStandardIo, ParentEdge, ProcessOwnershipContext, RetainedProcess, TargetCommand,
    child_broker,
};
use lazaret_linux::user_namespace::UserNamespaceMap;
use lazaret_state_anchor::{
    RuntimeStateAnchorAbsenceAuthority, RuntimeStateAnchorMount, StateAnchorLaunchAuthority,
};
use rustix::event::{PollFd, PollFlags, Timespec, poll};
use rustix::fs::{FileType, fstat};
use rustix::io::{FdFlags, fcntl_setfd, read, write};
use rustix::process::Signal;
use serde::{Deserialize, Serialize};

use crate::network::{NetworkProbeRequest, PreparedNetwork};
use crate::sandbox::RunscCommand;

use handoff::{build_runtime_handoff, parse_runtime_handoff};
use runtime::{receive_message, send_message};
use startup::{RuntimeGitRouting, inherited_directory_at, run_supervisor};

pub const PRIVATE_RUNTIME_ROLE: &str = "--lazaret-runtime-supervisor-role";
const MAX_CONTROL_MESSAGE: usize = 16 * 1024;
const MAX_RUNTIME_HANDOFF_BYTES: usize = 64 * 1024;
const READINESS_TIMEOUT: Duration = Duration::from_secs(30);
const TERMINATION_GRACE: Duration = Duration::from_secs(2);

pub const RUNTIME_SUPERVISOR_SERIALIZED_RECORD_TYPE_NAMES: &[&str] = &[
    "RuntimeSupervisorHandoff",
    "NetworkProbeRequest",
    "AttachedRunscSignal",
    "SupervisorEvent",
    "CoordinatorMessage",
];

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum AttachedRunscSignal {
    Stop,
    Continue,
    Kill,
}

impl lazaret_core::records::SchemaType for AttachedRunscSignal {
    const NAME: &'static str = "AttachedRunscSignal";
}

impl AttachedRunscSignal {
    const fn signal(self) -> Signal {
        match self {
            Self::Stop => Signal::STOP,
            Self::Continue => Signal::CONT,
            Self::Kill => Signal::KILL,
        }
    }
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum SupervisorEvent {
    StateAnchorStaged(StagedProcessOwnership),
    StateAnchorReady {
        staged: StagedProcessOwnership,
        user_namespace: lazaret_core::identity::NamespaceIdentity,
        mount_namespace: lazaret_core::identity::NamespaceIdentity,
        mount: lazaret_core::identity::MountIdentity,
    },
    GitRoutingStaged(StagedProcessOwnership),
    GitRoutingReady {
        staged: StagedProcessOwnership,
        mount: MountIdentity,
    },
    SidecarStarted,
    HelperStaged(StagedProcessOwnership),
    HelperReady(StagedProcessOwnership),
    NetworkAttestation(NetworkReadinessAttestation),
    NetworkProbe(lazaret_core::records::NetworkReachabilityProbe),
    HelperExited,
    Staged(StagedProcessOwnership),
    Ready {
        staged: StagedProcessOwnership,
        sentry_pid_inventory: u32,
    },
    SignalApplied(AttachedRunscSignal),
    SandboxStopped,
    GitRoutingStopped {
        mount: MountIdentity,
    },
    SidecarStopped,
    StateAnchorUnmounted {
        mount: lazaret_core::identity::MountIdentity,
    },
    StateAnchorAbsenceObserved {
        user_namespace: NamespaceIdentity,
        mount_namespace: NamespaceIdentity,
        mount: MountIdentity,
        observed: MountIdentity,
    },
    Exited,
    AttachedRunscExited,
    Stopped,
    Refused(String),
}

impl lazaret_core::records::SchemaType for SupervisorEvent {
    const NAME: &'static str = "SupervisorEvent";
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
enum CoordinatorMessage {
    MountStateAnchor,
    StateAnchorStagedAck(ProcessOwnershipCommitAck),
    StartGitRouting,
    GitRoutingStagedAck(ProcessOwnershipCommitAck),
    StartSidecar,
    HelperStagedAck(ProcessOwnershipCommitAck),
    NetworkAttestationAccepted(NetworkProbeRequest),
    NetworkReadinessCommitted,
    StagedAck(ProcessOwnershipCommitAck),
    SandboxCommitted,
    SignalAttachedRunsc(AttachedRunscSignal),
    StopSandbox,
    StopGitRouting,
    StopSidecar,
    UnmountStateAnchor,
    ObserveStateAnchorAbsence {
        user_namespace: NamespaceIdentity,
        mount_namespace: NamespaceIdentity,
        mount: MountIdentity,
    },
    Exit,
    Stop,
}

impl lazaret_core::records::SchemaType for CoordinatorMessage {
    const NAME: &'static str = "CoordinatorMessage";
}

pub struct CoordinatorChannel {
    descriptor: OwnedFd,
}

impl CoordinatorChannel {
    #[must_use]
    pub const fn new(descriptor: OwnedFd) -> Self {
        Self { descriptor }
    }

    /// # Errors
    /// Returns an error for a closed, malformed, oversized, or unknown message.
    pub fn receive(&self) -> io::Result<SupervisorEvent> {
        receive_message(&self.descriptor)
    }

    /// # Errors
    /// Returns an error when exact State Anchor acquisition cannot be requested.
    pub fn mount_state_anchor(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::MountStateAnchor)
    }

    /// # Errors
    /// Returns an error when the exact daemon acknowledgment cannot be delivered.
    pub fn acknowledge_state_anchor(
        &self,
        acknowledgment: ProcessOwnershipCommitAck,
    ) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::StateAnchorStagedAck(acknowledgment),
        )
    }

    /// # Errors
    /// Returns an error when generation-local Git routing cannot be requested.
    pub fn start_git_routing(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::StartGitRouting)
    }

    /// # Errors
    /// Returns an error when the exact router acknowledgment cannot be delivered.
    pub fn acknowledge_git_routing(
        &self,
        acknowledgment: ProcessOwnershipCommitAck,
    ) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::GitRoutingStagedAck(acknowledgment),
        )
    }

    /// # Errors
    /// Returns an error when the RS-owned sidecar install path cannot be released.
    pub fn start_sidecar(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::StartSidecar)
    }

    /// # Errors
    /// Returns an error when the exact typed acknowledgment cannot be delivered.
    pub fn acknowledge_staged(&self, acknowledgment: ProcessOwnershipCommitAck) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::StagedAck(acknowledgment),
        )
    }

    /// # Errors
    /// Returns an error when the exact helper acknowledgment cannot be delivered.
    pub fn acknowledge_helper(&self, acknowledgment: ProcessOwnershipCommitAck) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::HelperStagedAck(acknowledgment),
        )
    }

    /// # Errors
    /// Returns an error when accepted attestation cannot release sandbox acquisition.
    pub fn acknowledge_network_attestation(&self, request: NetworkProbeRequest) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::NetworkAttestationAccepted(request),
        )
    }

    /// # Errors
    /// Returns an error when durable network readiness cannot release sandbox acquisition.
    pub fn acknowledge_network_readiness(&self) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::NetworkReadinessCommitted,
        )
    }

    /// # Errors
    /// Returns an error when the durable sandbox commit cannot be acknowledged.
    pub fn acknowledge_sandbox_committed(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::SandboxCommitted)
    }

    /// # Errors
    /// Returns an error when the exact retained-runsc action cannot be requested.
    pub fn signal_attached_runsc(&self, signal: AttachedRunscSignal) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::SignalAttachedRunsc(signal),
        )
    }

    /// # Errors
    /// Returns an error when ordered sandbox stop cannot be requested.
    pub fn stop(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::Stop)
    }

    /// # Errors
    /// Returns an error when ordered sandbox stop cannot be requested.
    pub fn stop_sandbox(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::StopSandbox)
    }

    /// # Errors
    /// Returns an error when exact generation-local Git routing cannot be stopped.
    pub fn stop_git_routing(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::StopGitRouting)
    }

    /// # Errors
    /// Returns an error when ordered sidecar stop cannot be requested.
    pub fn stop_sidecar(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::StopSidecar)
    }

    /// # Errors
    /// Returns an error when exact State Anchor unmount cannot be requested.
    pub fn unmount_state_anchor(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::UnmountStateAnchor)
    }

    /// # Errors
    /// Returns an error when exact namespace-local absence observation cannot be requested.
    pub fn observe_state_anchor_absence(
        &self,
        user_namespace: NamespaceIdentity,
        mount_namespace: NamespaceIdentity,
        mount: MountIdentity,
    ) -> io::Result<()> {
        send_message(
            &self.descriptor,
            &CoordinatorMessage::ObserveStateAnchorAbsence {
                user_namespace,
                mount_namespace,
                mount,
            },
        )
    }

    /// # Errors
    /// Returns an error when graceful Runtime Supervisor exit cannot be requested.
    pub fn exit(&self) -> io::Result<()> {
        send_message(&self.descriptor, &CoordinatorMessage::Exit)
    }
}

pub struct RuntimeSupervisorLaunch {
    target: TargetCommand,
    handoff_file: Option<File>,
    ownership: ProcessOwnershipContext,
    policy_file: Option<File>,
    policy_digest: Option<DerivedPolicyDigest>,
    configuration_fingerprint: Option<ConfigurationFingerprint>,
    network_sidecar_id: Option<NetworkSidecarId>,
    runsc_command: RunscCommand,
    state_anchor: Option<StateAnchorLaunchAuthority>,
    git_routing: Option<RuntimeGitRoutingLaunch>,
    user_namespace_map: Option<UserNamespaceMap>,
}

#[derive(Clone, Copy)]
pub struct RuntimeNetworkLaunch<'policy> {
    pub policy: &'policy CanonicalDerivedNetworkPolicy,
    pub configuration_fingerprint: ConfigurationFingerprint,
    pub network_sidecar_id: NetworkSidecarId,
}

pub struct RuntimeGitRoutingLaunch {
    workspace: File,
    workspace_label: PathBuf,
    isolated: File,
    mountpoint: File,
    mountpoint_label: PathBuf,
    git_common: File,
    git_common_label: PathBuf,
    transient_hooks: File,
    transient_hooks_label: PathBuf,
    transient_config: File,
    transient_config_label: PathBuf,
    global_excludes: Option<PathBuf>,
    isolate_ignored: bool,
}

pub struct RuntimeGitRoutingAuthority {
    pub workspace: File,
    pub workspace_label: PathBuf,
    pub isolated: File,
    pub mountpoint: File,
    pub mountpoint_label: PathBuf,
    pub git_common: File,
    pub git_common_label: PathBuf,
    pub transient_hooks: File,
    pub transient_hooks_label: PathBuf,
    pub transient_config: File,
    pub transient_config_label: PathBuf,
    pub global_excludes: Option<PathBuf>,
    pub isolate_ignored: bool,
}

impl RuntimeGitRoutingLaunch {
    /// # Errors
    /// Refuses non-directory authorities or descriptors that cannot survive Runtime Supervisor
    /// launch.
    pub fn new(authority: RuntimeGitRoutingAuthority) -> io::Result<Self> {
        for authority in [
            &authority.workspace,
            &authority.isolated,
            &authority.mountpoint,
            &authority.git_common,
            &authority.transient_hooks,
        ] {
            if !FileType::from_raw_mode(fstat(authority)?.st_mode).is_dir() {
                return Err(io::Error::other("Git-routing authority is not a directory"));
            }
            fcntl_setfd(authority, FdFlags::empty())?;
        }
        if !FileType::from_raw_mode(fstat(&authority.transient_config)?.st_mode).is_file() {
            return Err(io::Error::other(
                "Git-routing transient config authority is not a regular file",
            ));
        }
        fcntl_setfd(&authority.transient_config, FdFlags::empty())?;
        Ok(Self {
            workspace: authority.workspace,
            workspace_label: authority.workspace_label,
            isolated: authority.isolated,
            mountpoint: authority.mountpoint,
            mountpoint_label: authority.mountpoint_label,
            git_common: authority.git_common,
            git_common_label: authority.git_common_label,
            transient_hooks: authority.transient_hooks,
            transient_hooks_label: authority.transient_hooks_label,
            transient_config: authority.transient_config,
            transient_config_label: authority.transient_config_label,
            global_excludes: authority.global_excludes,
            isolate_ignored: authority.isolate_ignored,
        })
    }
}

impl RuntimeSupervisorLaunch {
    /// # Errors
    /// Refuses any command other than the deterministic foreground attached-runsc form.
    pub fn from_runsc_command(
        ownership: ProcessOwnershipContext,
        command: &RunscCommand,
    ) -> io::Result<Self> {
        if ownership.slot != HostProcessOwnershipSlot::SandboxRuntimeSupervisor
            || command.program() != OsStr::new("runsc")
            || !command
                .arguments()
                .iter()
                .any(|argument| argument == OsStr::new("run"))
            || command
                .arguments()
                .iter()
                .any(|argument| matches!(argument.to_str(), Some("-detach" | "--detach" | "-d")))
        {
            return Err(io::Error::other(
                "Runtime Supervisor requires foreground attached runsc",
            ));
        }
        let mut launch = Self {
            target: TargetCommand::new("/usr/bin/false").with_lifetime_pipe_parent_edge(),
            handoff_file: None,
            ownership,
            policy_file: None,
            policy_digest: None,
            configuration_fingerprint: None,
            network_sidecar_id: None,
            runsc_command: command.clone(),
            state_anchor: None,
            git_routing: None,
            user_namespace_map: None,
        };
        launch.refresh_target()?;
        Ok(launch)
    }

    /// Builds the RS-owned network path with one inherited sealed policy document.
    ///
    /// # Errors
    /// Refuses an invalid runsc form or a policy descriptor that cannot be sealed and inherited.
    pub fn from_networked_runsc_command(
        ownership: ProcessOwnershipContext,
        command: &RunscCommand,
        network: RuntimeNetworkLaunch<'_>,
    ) -> io::Result<Self> {
        let mut launch = Self::from_runsc_command(ownership, command)?;
        let policy_file = sealed_memfd("lazaret-network-policy", network.policy.canonical_bytes())?;
        fcntl_setfd(policy_file.as_fd(), FdFlags::empty())?;
        launch.policy_file = Some(policy_file);
        launch.policy_digest = Some(network.policy.digest());
        launch.configuration_fingerprint = Some(network.configuration_fingerprint);
        launch.network_sidecar_id = Some(network.network_sidecar_id);
        launch.refresh_target()?;
        Ok(launch)
    }

    /// Adds exact inherited State Anchor authorities without beginning physical acquisition.
    /// # Errors
    /// Refuses an invalid or unsealable State Anchor handoff.
    pub fn with_state_anchor(mut self, authority: StateAnchorLaunchAuthority) -> io::Result<Self> {
        self.state_anchor = Some(authority);
        self.refresh_target()?;
        Ok(self)
    }

    /// # Errors
    /// Refuses an invalid or unsealable Git-routing handoff.
    pub fn with_git_routing(mut self, authority: RuntimeGitRoutingLaunch) -> io::Result<Self> {
        self.git_routing = Some(authority);
        self.refresh_target()?;
        Ok(self)
    }

    /// Adds the canonical caller-configured user namespace required by the Runtime Supervisor.
    ///
    /// # Errors
    /// Refuses a non-canonical rootless map.
    pub fn with_user_namespace_map(mut self, map: UserNamespaceMap) -> io::Result<Self> {
        map.validate()?;
        self.user_namespace_map = Some(map);
        Ok(self)
    }

    fn refresh_target(&mut self) -> io::Result<()> {
        let encoded = build_runtime_handoff(self)?;
        let handoff = sealed_memfd("lazaret-runtime-handoff", &encoded)?;
        fcntl_setfd(&handoff, FdFlags::empty())?;
        self.target = TargetCommand::new("/proc/self/exe")
            .argument(PRIVATE_RUNTIME_ROLE)
            .argument(handoff.as_raw_fd().to_string())
            .with_lifetime_pipe_parent_edge();
        self.handoff_file = Some(handoff);
        Ok(())
    }

    #[must_use]
    pub const fn target(&self) -> &TargetCommand {
        &self.target
    }

    #[must_use]
    pub const fn ownership(&self) -> ProcessOwnershipContext {
        self.ownership
    }

    #[must_use]
    pub const fn network_sidecar_id(&self) -> Option<NetworkSidecarId> {
        self.network_sidecar_id
    }

    #[must_use]
    pub const fn has_git_routing(&self) -> bool {
        self.git_routing.is_some()
    }

    #[must_use]
    pub const fn runsc_command(&self) -> &RunscCommand {
        &self.runsc_command
    }

    #[must_use]
    pub const fn user_namespace_map(&self) -> Option<&UserNamespaceMap> {
        self.user_namespace_map.as_ref()
    }

    /// # Errors
    /// Refuses a Runtime Supervisor target without the lifetime-pipe parent edge.
    pub fn child_standard_io(
        &self,
        private_control: OwnedFd,
        lifetime_pipe: OwnedFd,
    ) -> io::Result<ChildStandardIo> {
        if self.target.parent_edge() != ParentEdge::LifetimePipe {
            return Err(io::Error::other(
                "Runtime Supervisor launch requires the lifetime-pipe parent edge",
            ));
        }
        Ok(ChildStandardIo::private_control_with_lifetime_pipe(
            private_control,
            lifetime_pipe,
        ))
    }
}

/// Runs the Runtime Supervisor's private early role before ordinary binary dispatch.
///
/// # Errors
/// Returns a fail-closed parse, acquisition, capability, channel, or process error.
#[must_use]
pub fn dispatch_private_runtime_role(arguments: &[OsString]) -> Option<io::Result<()>> {
    if arguments
        .first()
        .is_none_or(|argument| argument != PRIVATE_RUNTIME_ROLE)
    {
        return None;
    }
    let result = parse_runtime_handoff(&arguments[1..]).and_then(|parsed| run_supervisor(&parsed));
    if let Err(error) = &result {
        let _ = send_message(
            std::io::stdout(),
            &SupervisorEvent::Refused(error.to_string()),
        );
    }
    Some(result)
}

struct RuntimeArguments {
    ownership: ProcessOwnershipContext,
    state_anchor: Option<RuntimeStateAnchorArguments>,
    git_routing: Option<RuntimeGitRoutingArguments>,
    network: Option<RuntimeNetworkArguments>,
    runsc_program: OsString,
    runsc_arguments: Vec<OsString>,
}

struct RuntimeGitRoutingArguments {
    workspace_fd: i32,
    workspace_label: PathBuf,
    isolated_fd: i32,
    mountpoint_fd: i32,
    mountpoint_label: PathBuf,
    git_common_fd: i32,
    git_common_label: PathBuf,
    transient_hooks_fd: i32,
    transient_hooks_label: PathBuf,
    transient_config_fd: i32,
    transient_config_label: PathBuf,
    global_excludes: Option<PathBuf>,
    isolate_ignored: bool,
}

struct RuntimeStateAnchorArguments {
    lower_fd: i32,
    lower_label: PathBuf,
    persistent_root_fd: i32,
    persistent_anchor_fd: i32,
    persistent_anchor_label: PathBuf,
    runtime_root_fd: i32,
    runtime_anchor_fd: i32,
    runtime_anchor_label: PathBuf,
    expected_lower: (u64, u64),
    persistent_destinations: Vec<PathBuf>,
}

struct RuntimeNetworkArguments {
    policy_fd: i32,
    policy_digest: DerivedPolicyDigest,
    configuration_fingerprint: ConfigurationFingerprint,
    network_sidecar_id: NetworkSidecarId,
}

mod handoff;
mod runtime;
mod startup;

#[cfg(test)]
mod tests;