use std::ffi::{OsStr, OsString}; use std::fs::File; use std::io; use std::os::fd::{AsFd, AsRawFd, OwnedFd}; use std::os::unix::fs::FileExt; use std::path::PathBuf; use std::time::{Duration, Instant}; use lazaret_core::codec::{decode, encode}; use lazaret_core::identity::{ ConfigurationFingerprint, CoordinatorId, DerivedPolicyDigest, DomainId, GenerationId, MountIdentity, NamespaceIdentity, NetworkSidecarId, }; use lazaret_core::network::CanonicalDerivedNetworkPolicy; use lazaret_core::records::{ HostProcessOwnershipSlot, NetworkReadinessAttestation, ProcessOwnershipCommitAck, StagedProcessOwnership, }; use lazaret_linux::capability; use lazaret_linux::filesystem::sealed_memfd; use lazaret_linux::process::{ ChildStandardIo, ParentEdge, ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, current_process_identity, }; use lazaret_state_anchor::{ RuntimeStateAnchorAbsenceAuthority, RuntimeStateAnchorAuthority, RuntimeStateAnchorInherited, RuntimeStateAnchorMount, StateAnchorLaunchAuthority, }; use rustix::event::{PollFd, PollFlags, Timespec, poll}; use rustix::fs::{MemfdFlags, memfd_create}; use rustix::io::{FdFlags, fcntl_setfd, read, write}; use rustix::process::Signal; use rustix::thread::CapabilitySet; use serde::{Deserialize, Serialize}; use crate::network::{ NetworkAttestationContext, NetworkProbeRequest, PreparedNetwork, enter_generation_filesystem_namespaces, prepare_network, probe_current_network, }; use crate::sandbox::RunscCommand; const PRIVATE_ROLE_ARGUMENT: &str = "--lazaret-runtime-supervisor-role"; const MAX_CONTROL_MESSAGE: usize = 16 * 1024; const READINESS_TIMEOUT: Duration = Duration::from_secs(30); const TERMINATION_GRACE: Duration = Duration::from_secs(2); #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum AttachedRunscSignal { Stop, Continue, Kill, } impl AttachedRunscSignal { const fn signal(self) -> Signal { match self { Self::Stop => Signal::STOP, Self::Continue => Signal::CONT, Self::Kill => Signal::KILL, } } } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum SupervisorEvent { StateAnchorStaged(StagedProcessOwnership), StateAnchorReady { staged: StagedProcessOwnership, user_namespace: lazaret_core::identity::NamespaceIdentity, mount_namespace: lazaret_core::identity::NamespaceIdentity, mount: lazaret_core::identity::MountIdentity, }, SidecarStarted, HelperStaged(StagedProcessOwnership), HelperReady(StagedProcessOwnership), NetworkAttestation(NetworkReadinessAttestation), NetworkProbe(lazaret_core::records::NetworkReachabilityProbe), HelperExited, Staged(StagedProcessOwnership), Ready { staged: StagedProcessOwnership, sentry_pid_inventory: u32, }, SignalApplied(AttachedRunscSignal), SandboxStopped, SidecarStopped, StateAnchorUnmounted { mount: lazaret_core::identity::MountIdentity, }, StateAnchorAbsenceObserved { user_namespace: NamespaceIdentity, mount_namespace: NamespaceIdentity, mount: MountIdentity, observed: MountIdentity, }, Exited, AttachedRunscExited, Stopped, Refused(String), } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] enum CoordinatorMessage { MountStateAnchor, StateAnchorStagedAck(ProcessOwnershipCommitAck), StartSidecar, HelperStagedAck(ProcessOwnershipCommitAck), NetworkAttestationAccepted(NetworkProbeRequest), NetworkReadinessCommitted, StagedAck(ProcessOwnershipCommitAck), SandboxCommitted, SignalAttachedRunsc(AttachedRunscSignal), StopSandbox, StopSidecar, UnmountStateAnchor, ObserveStateAnchorAbsence { user_namespace: NamespaceIdentity, mount_namespace: NamespaceIdentity, mount: MountIdentity, }, Exit, Stop, } pub struct CoordinatorChannel { descriptor: OwnedFd, } impl CoordinatorChannel { #[must_use] pub const fn new(descriptor: OwnedFd) -> Self { Self { descriptor } } /// # Errors /// Returns an error for a closed, malformed, oversized, or unknown message. pub fn receive(&self) -> io::Result { receive_message(&self.descriptor) } /// # Errors /// Returns an error when exact State Anchor acquisition cannot be requested. pub fn mount_state_anchor(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::MountStateAnchor) } /// # Errors /// Returns an error when the exact daemon acknowledgment cannot be delivered. pub fn acknowledge_state_anchor( &self, acknowledgment: ProcessOwnershipCommitAck, ) -> io::Result<()> { send_message( &self.descriptor, &CoordinatorMessage::StateAnchorStagedAck(acknowledgment), ) } /// # Errors /// Returns an error when the RS-owned sidecar install path cannot be released. pub fn start_sidecar(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::StartSidecar) } /// # Errors /// Returns an error when the exact typed acknowledgment cannot be delivered. pub fn acknowledge_staged(&self, acknowledgment: ProcessOwnershipCommitAck) -> io::Result<()> { send_message( &self.descriptor, &CoordinatorMessage::StagedAck(acknowledgment), ) } /// # Errors /// Returns an error when the exact helper acknowledgment cannot be delivered. pub fn acknowledge_helper(&self, acknowledgment: ProcessOwnershipCommitAck) -> io::Result<()> { send_message( &self.descriptor, &CoordinatorMessage::HelperStagedAck(acknowledgment), ) } /// # Errors /// Returns an error when accepted attestation cannot release sandbox acquisition. pub fn acknowledge_network_attestation(&self, request: NetworkProbeRequest) -> io::Result<()> { send_message( &self.descriptor, &CoordinatorMessage::NetworkAttestationAccepted(request), ) } /// # Errors /// Returns an error when durable network readiness cannot release sandbox acquisition. pub fn acknowledge_network_readiness(&self) -> io::Result<()> { send_message( &self.descriptor, &CoordinatorMessage::NetworkReadinessCommitted, ) } /// # Errors /// Returns an error when the durable sandbox commit cannot be acknowledged. pub fn acknowledge_sandbox_committed(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::SandboxCommitted) } /// # Errors /// Returns an error when the exact retained-runsc action cannot be requested. pub fn signal_attached_runsc(&self, signal: AttachedRunscSignal) -> io::Result<()> { send_message( &self.descriptor, &CoordinatorMessage::SignalAttachedRunsc(signal), ) } /// # Errors /// Returns an error when ordered sandbox stop cannot be requested. pub fn stop(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::Stop) } /// # Errors /// Returns an error when ordered sandbox stop cannot be requested. pub fn stop_sandbox(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::StopSandbox) } /// # Errors /// Returns an error when ordered sidecar stop cannot be requested. pub fn stop_sidecar(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::StopSidecar) } /// # Errors /// Returns an error when exact State Anchor unmount cannot be requested. pub fn unmount_state_anchor(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::UnmountStateAnchor) } /// # Errors /// Returns an error when exact namespace-local absence observation cannot be requested. pub fn observe_state_anchor_absence( &self, user_namespace: NamespaceIdentity, mount_namespace: NamespaceIdentity, mount: MountIdentity, ) -> io::Result<()> { send_message( &self.descriptor, &CoordinatorMessage::ObserveStateAnchorAbsence { user_namespace, mount_namespace, mount, }, ) } /// # Errors /// Returns an error when graceful Runtime Supervisor exit cannot be requested. pub fn exit(&self) -> io::Result<()> { send_message(&self.descriptor, &CoordinatorMessage::Exit) } } pub struct RuntimeSupervisorLaunch { target: TargetCommand, ownership: ProcessOwnershipContext, policy_file: Option, policy_digest: Option, configuration_fingerprint: Option, network_sidecar_id: Option, runsc_command: RunscCommand, state_anchor: Option, } #[derive(Clone, Copy)] pub struct RuntimeNetworkLaunch<'policy> { pub policy: &'policy CanonicalDerivedNetworkPolicy, pub configuration_fingerprint: ConfigurationFingerprint, pub network_sidecar_id: NetworkSidecarId, } impl RuntimeSupervisorLaunch { /// # Errors /// Refuses any command other than the deterministic foreground attached-runsc form. pub fn from_runsc_command( ownership: ProcessOwnershipContext, command: &RunscCommand, ) -> io::Result { if ownership.slot != HostProcessOwnershipSlot::SandboxRuntimeSupervisor || command.program() != OsStr::new("runsc") || !command .arguments() .iter() .any(|argument| argument == OsStr::new("run")) || command .arguments() .iter() .any(|argument| matches!(argument.to_str(), Some("-detach" | "--detach" | "-d"))) { return Err(io::Error::other( "Runtime Supervisor requires foreground attached runsc", )); } let mut target = TargetCommand::new("/proc/self/exe") .argument(PRIVATE_ROLE_ARGUMENT) .argument("--domain") .argument(hex(ownership.domain_id.as_bytes())) .argument("--generation") .argument(ownership.generation_id.get().to_string()) .argument("--coordinator") .argument(hex(ownership.coordinator_id.as_bytes())) .argument("--runsc") .argument(command.program()); for argument in command.arguments() { target = target.argument(argument); } Ok(Self { target: target.with_lifetime_pipe_parent_edge(), ownership, policy_file: None, policy_digest: None, configuration_fingerprint: None, network_sidecar_id: None, runsc_command: command.clone(), state_anchor: None, }) } /// Builds the RS-owned network path with one inherited sealed policy document. /// /// # Errors /// Refuses an invalid runsc form or a policy descriptor that cannot be sealed and inherited. pub fn from_networked_runsc_command( ownership: ProcessOwnershipContext, command: &RunscCommand, network: RuntimeNetworkLaunch<'_>, ) -> io::Result { let runtime_supervisor_executable = std::env::current_exe()?; let mut launch = Self::from_runsc_command(ownership, command)?; let policy_file = sealed_memfd("lazaret-network-policy", network.policy.canonical_bytes())?; fcntl_setfd(policy_file.as_fd(), FdFlags::empty())?; let policy_fd = policy_file.as_raw_fd(); let target = std::mem::replace( &mut launch.target, TargetCommand::new("/usr/bin/false").with_lifetime_pipe_parent_edge(), ); launch.target = TargetCommand::new(runtime_supervisor_executable) .argument(PRIVATE_ROLE_ARGUMENT) .argument("--domain") .argument(hex(ownership.domain_id.as_bytes())) .argument("--generation") .argument(ownership.generation_id.get().to_string()) .argument("--coordinator") .argument(hex(ownership.coordinator_id.as_bytes())) .argument("--network-policy-fd") .argument(policy_fd.to_string()) .argument("--network-policy-digest") .argument(hex(network.policy.digest().as_bytes())) .argument("--configuration-fingerprint") .argument(hex(network.configuration_fingerprint.as_bytes())) .argument("--network-sidecar") .argument(hex(network.network_sidecar_id.as_bytes())) .argument("--runsc") .argument(command.program()); for argument in command.arguments() { launch.target = launch.target.argument(argument); } launch.target = launch.target.with_lifetime_pipe_parent_edge(); drop(target); launch.policy_file = Some(policy_file); launch.policy_digest = Some(network.policy.digest()); launch.configuration_fingerprint = Some(network.configuration_fingerprint); launch.network_sidecar_id = Some(network.network_sidecar_id); Ok(launch) } /// Adds exact inherited State Anchor authorities without beginning physical acquisition. #[must_use] pub fn with_state_anchor(mut self, authority: StateAnchorLaunchAuthority) -> Self { self.state_anchor = Some(authority); self.rebuild_target(); self } fn rebuild_target(&mut self) { let mut target = TargetCommand::new("/proc/self/exe") .argument(PRIVATE_ROLE_ARGUMENT) .argument("--domain") .argument(hex(self.ownership.domain_id.as_bytes())) .argument("--generation") .argument(self.ownership.generation_id.get().to_string()) .argument("--coordinator") .argument(hex(self.ownership.coordinator_id.as_bytes())); if let Some(authority) = &self.state_anchor { let (device, inode) = authority.expected_lower(); target = target .argument("--state-lower-fd") .argument(authority.lower_fd().to_string()) .argument("--state-lower-label") .argument(authority.lower_label()) .argument("--state-persistent-root-fd") .argument(authority.persistent_root_fd().to_string()) .argument("--state-persistent-fd") .argument(authority.persistent_anchor_fd().to_string()) .argument("--state-persistent-label") .argument(authority.persistent_anchor_label()) .argument("--state-runtime-root-fd") .argument(authority.runtime_root_fd().to_string()) .argument("--state-runtime-fd") .argument(authority.runtime_anchor_fd().to_string()) .argument("--state-runtime-label") .argument(authority.runtime_anchor_label()) .argument("--state-lower-device") .argument(device.to_string()) .argument("--state-lower-inode") .argument(inode.to_string()) .argument("--state-persistent-count") .argument(authority.persistent_destinations().len().to_string()); for destination in authority.persistent_destinations() { target = target .argument("--state-persistent-destination") .argument(destination); } } if let (Some(policy_file), Some(digest), Some(fingerprint), Some(sidecar)) = ( &self.policy_file, self.policy_digest, self.configuration_fingerprint, self.network_sidecar_id, ) { target = target .argument("--network-policy-fd") .argument(policy_file.as_raw_fd().to_string()) .argument("--network-policy-digest") .argument(hex(digest.as_bytes())) .argument("--configuration-fingerprint") .argument(hex(fingerprint.as_bytes())) .argument("--network-sidecar") .argument(hex(sidecar.as_bytes())); } target = target .argument("--runsc") .argument(self.runsc_command.program()); for argument in self.runsc_command.arguments() { target = target.argument(argument); } self.target = target.with_lifetime_pipe_parent_edge(); } #[must_use] pub const fn target(&self) -> &TargetCommand { &self.target } #[must_use] pub const fn ownership(&self) -> ProcessOwnershipContext { self.ownership } #[must_use] pub const fn network_sidecar_id(&self) -> Option { self.network_sidecar_id } #[must_use] pub const fn runsc_command(&self) -> &RunscCommand { &self.runsc_command } /// # Errors /// Refuses a Runtime Supervisor target without the lifetime-pipe parent edge. pub fn child_standard_io( &self, private_control: OwnedFd, lifetime_pipe: OwnedFd, ) -> io::Result { if self.target.parent_edge() != ParentEdge::LifetimePipe { return Err(io::Error::other( "Runtime Supervisor launch requires the lifetime-pipe parent edge", )); } Ok(ChildStandardIo::private_control_with_lifetime_pipe( private_control, lifetime_pipe, )) } } /// Runs the Runtime Supervisor's private early role before ordinary binary dispatch. /// /// # Errors /// Returns a fail-closed parse, acquisition, capability, channel, or process error. #[must_use] pub fn dispatch_private_runtime_role(arguments: &[OsString]) -> Option> { if arguments .first() .is_none_or(|argument| argument != PRIVATE_ROLE_ARGUMENT) { return None; } let result = parse_runtime_arguments(&arguments[1..]).and_then(|parsed| run_supervisor(&parsed)); if let Err(error) = &result { let _ = send_message( std::io::stdout(), &SupervisorEvent::Refused(error.to_string()), ); } Some(result) } struct RuntimeArguments { ownership: ProcessOwnershipContext, state_anchor: Option, network: Option, runsc_program: OsString, runsc_arguments: Vec, } struct RuntimeStateAnchorArguments { lower_fd: i32, lower_label: PathBuf, persistent_root_fd: i32, persistent_anchor_fd: i32, persistent_anchor_label: PathBuf, runtime_root_fd: i32, runtime_anchor_fd: i32, runtime_anchor_label: PathBuf, expected_lower: (u64, u64), persistent_destinations: Vec, } struct RuntimeNetworkArguments { policy_fd: i32, policy_digest: DerivedPolicyDigest, configuration_fingerprint: ConfigurationFingerprint, network_sidecar_id: NetworkSidecarId, } fn parse_runtime_arguments(arguments: &[OsString]) -> io::Result { let mut arguments = arguments.iter().peekable(); let domain = parse_fixed_hex(named(&mut arguments, "--domain")?)?; let generation = named(&mut arguments, "--generation")? .to_str() .and_then(|value| value.parse().ok()) .ok_or_else(|| io::Error::other("Runtime Supervisor generation is invalid"))?; let coordinator = parse_fixed_hex(named(&mut arguments, "--coordinator")?)?; let state_anchor = if arguments .peek() .is_some_and(|value| *value == "--state-lower-fd") { let lower_fd = parse_number(named(&mut arguments, "--state-lower-fd")?)?; let lower_label = PathBuf::from(named(&mut arguments, "--state-lower-label")?); let persistent_root_fd = parse_number(named(&mut arguments, "--state-persistent-root-fd")?)?; let persistent_anchor_fd = parse_number(named(&mut arguments, "--state-persistent-fd")?)?; let persistent_anchor_label = PathBuf::from(named(&mut arguments, "--state-persistent-label")?); let runtime_root_fd = parse_number(named(&mut arguments, "--state-runtime-root-fd")?)?; let runtime_anchor_fd = parse_number(named(&mut arguments, "--state-runtime-fd")?)?; let runtime_anchor_label = PathBuf::from(named(&mut arguments, "--state-runtime-label")?); let expected_lower = ( parse_number(named(&mut arguments, "--state-lower-device")?)?, parse_number(named(&mut arguments, "--state-lower-inode")?)?, ); let count: usize = parse_number(named(&mut arguments, "--state-persistent-count")?)?; let persistent_destinations = (0..count) .map(|_| named(&mut arguments, "--state-persistent-destination").map(PathBuf::from)) .collect::>>()?; Some(RuntimeStateAnchorArguments { lower_fd, lower_label, persistent_root_fd, persistent_anchor_fd, persistent_anchor_label, runtime_root_fd, runtime_anchor_fd, runtime_anchor_label, expected_lower, persistent_destinations, }) } else { None }; let network = if arguments .peek() .is_some_and(|value| *value == "--network-policy-fd") { Some(parse_network_arguments(&mut arguments)?) } else { None }; if arguments.next().is_none_or(|value| value != "--runsc") { return Err(io::Error::other( "Runtime Supervisor runsc separator is missing", )); } let program = arguments .next() .ok_or_else(|| io::Error::other("Runtime Supervisor runsc program is missing"))?; if program != "runsc" { return Err(io::Error::other("Runtime Supervisor target is not runsc")); } Ok(RuntimeArguments { ownership: ProcessOwnershipContext { domain_id: DomainId::new(domain), generation_id: GenerationId::new(generation), coordinator_id: CoordinatorId::new(coordinator), slot: HostProcessOwnershipSlot::SandboxAttachedRunsc, }, state_anchor, network, runsc_program: (*program).clone(), runsc_arguments: arguments.cloned().collect(), }) } fn parse_number(value: &OsStr) -> io::Result where T: std::str::FromStr, { value .to_str() .and_then(|value| value.parse().ok()) .ok_or_else(|| io::Error::other("Runtime Supervisor numeric argument is invalid")) } fn parse_network_arguments<'a>( arguments: &mut impl Iterator, ) -> io::Result { let policy_fd = named(arguments, "--network-policy-fd")? .to_str() .and_then(|value| value.parse().ok()) .filter(|value| *value >= 0) .ok_or_else(|| io::Error::other("Runtime Supervisor policy descriptor is invalid"))?; let policy_digest = DerivedPolicyDigest::new(parse_fixed_hex(named( arguments, "--network-policy-digest", )?)?); let configuration_fingerprint = ConfigurationFingerprint::new(parse_fixed_hex(named( arguments, "--configuration-fingerprint", )?)?); let network_sidecar_id = NetworkSidecarId::new(parse_fixed_hex(named(arguments, "--network-sidecar")?)?); Ok(RuntimeNetworkArguments { policy_fd, policy_digest, configuration_fingerprint, network_sidecar_id, }) } fn named<'a>( arguments: &mut impl Iterator, name: &str, ) -> io::Result<&'a OsString> { if arguments.next().is_none_or(|value| value != name) { return Err(io::Error::other(format!( "Runtime Supervisor argument {name} is missing" ))); } arguments .next() .ok_or_else(|| io::Error::other(format!("Runtime Supervisor argument {name} has no value"))) } fn parse_fixed_hex(value: &OsStr) -> io::Result<[u8; 32]> { let value = value .to_str() .ok_or_else(|| io::Error::other("Runtime Supervisor identity is not UTF-8"))?; if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { return Err(io::Error::other("Runtime Supervisor identity is invalid")); } let mut bytes = [0_u8; 32]; for (index, byte) in bytes.iter_mut().enumerate() { *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16) .map_err(|_| io::Error::other("Runtime Supervisor identity is invalid"))?; } Ok(bytes) } fn run_supervisor(arguments: &RuntimeArguments) -> io::Result<()> { require_lifetime_held()?; let namespaces = enter_generation_filesystem_namespaces()?; let control = std::io::stdout(); let parent = current_process_identity()?; let mut state_anchor = prepare_state_anchor(arguments, parent, namespaces, &control)?; let mut state_anchor_absence = None; let result = run_supervisor_with_state( arguments, parent, &control, namespaces, &mut state_anchor, &mut state_anchor_absence, ); let cleanup = state_anchor .take() .map(RuntimeStateAnchorMount::unmount) .map(|result| result.map(drop)); match (result, cleanup) { (Err(error), Some(Err(cleanup))) => Err(io::Error::new( error.kind(), format!("{error}; exact State Anchor cleanup failed: {cleanup}"), )), (Err(error), _) => Err(error), (Ok(()), Some(Err(error))) => Err(io::Error::other(error)), (Ok(()), _) => Ok(()), } } fn prepare_state_anchor( arguments: &RuntimeArguments, parent: lazaret_core::identity::PidfsProcessIdentityV1, namespaces: ( lazaret_core::identity::NamespaceIdentity, lazaret_core::identity::NamespaceIdentity, ), control: &std::io::Stdout, ) -> io::Result> { let Some(state) = &arguments.state_anchor else { return Ok(None); }; let command: CoordinatorMessage = receive_message(control)?; if command != CoordinatorMessage::MountStateAnchor { return Err(io::Error::other( "Runtime Supervisor received no State Anchor mount command", )); } let authority = RuntimeStateAnchorAuthority::from_inherited(RuntimeStateAnchorInherited { lower_fd: state.lower_fd, lower_label: &state.lower_label, persistent_root_fd: state.persistent_root_fd, persistent_anchor_fd: state.persistent_anchor_fd, persistent_anchor_label: &state.persistent_anchor_label, runtime_root_fd: state.runtime_root_fd, runtime_anchor_fd: state.runtime_anchor_fd, runtime_anchor_label: &state.runtime_anchor_label, expected_lower: state.expected_lower, }) .map_err(io::Error::other)?; let ownership = ProcessOwnershipContext { slot: HostProcessOwnershipSlot::StateAnchorDaemon, ..arguments.ownership }; let mount = authority .acquire( parent, ownership, |staged| send_message(control, &SupervisorEvent::StateAnchorStaged(staged)), |staged| match receive_message(control)? { CoordinatorMessage::StateAnchorStagedAck(acknowledgment) if acknowledgment.acknowledges(&staged) => { Ok(acknowledgment) } _ => Err(io::Error::other( "Runtime Supervisor received a mismatched State Anchor acknowledgment", )), }, |readiness| { send_message( control, &SupervisorEvent::StateAnchorReady { staged: readiness.staged(), user_namespace: namespaces.0, mount_namespace: namespaces.1, mount: readiness.mount(), }, ) }, ) .map_err(io::Error::other)?; mount .require_persistent_destinations(&state.persistent_destinations) .map_err(io::Error::other)?; Ok(Some(mount)) } fn run_supervisor_with_state( arguments: &RuntimeArguments, parent: lazaret_core::identity::PidfsProcessIdentityV1, control: &std::io::Stdout, namespaces: (NamespaceIdentity, NamespaceIdentity), state_anchor: &mut Option, state_anchor_absence: &mut Option, ) -> io::Result<()> { if arguments.state_anchor.is_some() { let message: CoordinatorMessage = receive_message(control)?; if message != CoordinatorMessage::StartSidecar { return Err(io::Error::other( "Runtime Supervisor received no sidecar start command", )); } send_message(control, &SupervisorEvent::SidecarStarted)?; } let network = match &arguments.network { Some(network) => Some(prepare_runtime_network( arguments, network, parent, control, )?), None => None, }; let pid_file = File::from(memfd_create("lazaret-runsc-pid", MemfdFlags::empty())?); fcntl_setfd(pid_file.as_fd(), FdFlags::empty())?; let runsc = runsc_target(arguments, pid_file.as_raw_fd())?; let delete = runsc_delete_target(arguments)?; let staged = std::cell::Cell::new(None); let process = match child_broker().acquire_with_stdio( parent, arguments.ownership, &runsc, attached_runsc_standard_io(&runsc)?, |ownership| { staged.set(Some(ownership)); send_message(control, &SupervisorEvent::Staged(ownership)) }, |ownership| match receive_message(control)? { CoordinatorMessage::StagedAck(acknowledgment) if acknowledgment.acknowledges(&ownership) => { Ok(acknowledgment) } _ => Err(io::Error::other( "Runtime Supervisor received a mismatched staged acknowledgment", )), }, ) { Ok(process) => process, Err(error) => return fail_network(network, error), }; if let Err(error) = fcntl_setfd(pid_file.as_fd(), FdFlags::CLOEXEC) { return fail_with_network_cleanup(process, network, error.into()); } let Some(staged) = staged.get() else { return fail_with_network_cleanup( process, network, io::Error::other("attached runsc ownership was not staged"), ); }; let sentry_pid_inventory = match wait_for_runsc_ready(&pid_file, &process) { Ok(pid) => pid, Err(error) => return fail_with_network_cleanup(process, network, error), }; if let Err(error) = capability::irreversibly_drop(CapabilitySet::NET_ADMIN) { return fail_with_network_cleanup(process, network, error); } if let Err(error) = capability::require_regain_refused(CapabilitySet::NET_ADMIN) { return fail_with_network_cleanup(process, network, error); } if let Err(error) = send_message( control, &SupervisorEvent::Ready { staged, sentry_pid_inventory, }, ) { return fail_with_network_cleanup(process, network, error); } let committed: CoordinatorMessage = match receive_message(control) { Ok(committed) => committed, Err(error) => return fail_with_network_cleanup(process, network, error), }; if committed != CoordinatorMessage::SandboxCommitted { return fail_with_network_cleanup( process, network, io::Error::other("sandbox commit acknowledgment was not received"), ); } supervise( control, process, network, namespaces, state_anchor, state_anchor_absence, &delete, ) } fn prepare_runtime_network( arguments: &RuntimeArguments, network: &RuntimeNetworkArguments, runtime_supervisor: lazaret_core::identity::PidfsProcessIdentityV1, control: &std::io::Stdout, ) -> io::Result { let policy_file = File::open(format!("/proc/self/fd/{}", network.policy_fd))?; let staged = std::cell::Cell::new(None); let prepared = prepare_network( &policy_file, network.policy_digest, NetworkAttestationContext { domain_id: arguments.ownership.domain_id, generation_id: arguments.ownership.generation_id, configuration_fingerprint: network.configuration_fingerprint, network_sidecar_id: network.network_sidecar_id, runtime_supervisor, coordinator_id: arguments.ownership.coordinator_id, }, |ownership| { staged.set(Some(ownership)); send_message(control, &SupervisorEvent::HelperStaged(ownership)) }, |ownership| match receive_message(control)? { CoordinatorMessage::HelperStagedAck(acknowledgment) if acknowledgment.acknowledges(&ownership) => { Ok(acknowledgment) } _ => Err(io::Error::other( "Runtime Supervisor received a mismatched helper acknowledgment", )), }, )?; let staged = staged .get() .ok_or_else(|| io::Error::other("connectivity helper ownership was not staged"))?; if let Err(error) = send_message(control, &SupervisorEvent::HelperReady(staged)) { return fail_network(Some(prepared), error); } if let Err(error) = send_message( control, &SupervisorEvent::NetworkAttestation(prepared.attestation().clone()), ) { return fail_network(Some(prepared), error); } let message: CoordinatorMessage = match receive_message(control) { Ok(message) => message, Err(error) => return fail_network(Some(prepared), error), }; let CoordinatorMessage::NetworkAttestationAccepted(request) = message else { return fail_network( Some(prepared), io::Error::other("network attestation acceptance was not received"), ); }; let probe = match probe_current_network(&request) { Ok(probe) => probe, Err(error) => return fail_network(Some(prepared), error), }; if let Err(error) = send_message(control, &SupervisorEvent::NetworkProbe(probe)) { return fail_network(Some(prepared), error); } let committed: CoordinatorMessage = match receive_message(control) { Ok(message) => message, Err(error) => return fail_network(Some(prepared), error), }; if committed != CoordinatorMessage::NetworkReadinessCommitted { return fail_network( Some(prepared), io::Error::other("durable network readiness was not acknowledged"), ); } Ok(prepared) } fn runsc_target(arguments: &RuntimeArguments, pid_file: i32) -> io::Result { let run = arguments .runsc_arguments .iter() .position(|argument| argument == "run") .ok_or_else(|| io::Error::other("attached runsc command has no run operation"))?; let mut target = TargetCommand::new(arguments.runsc_program.clone()); for argument in &arguments.runsc_arguments[..=run] { target = target.argument(argument); } target = target.argument(format!("-pid-file=/proc/self/fd/{pid_file}")); for argument in &arguments.runsc_arguments[run + 1..] { target = target.argument(argument); } Ok(target.with_parent_death_kill_edge()) } fn runsc_delete_target(arguments: &RuntimeArguments) -> io::Result { let run = arguments .runsc_arguments .iter() .position(|argument| argument == "run") .ok_or_else(|| io::Error::other("attached runsc command has no run operation"))?; let container = arguments .runsc_arguments .last() .filter(|container| { !container.is_empty() && !container.as_encoded_bytes().starts_with(b"-") }) .ok_or_else(|| io::Error::other("attached runsc command has no container ID"))?; let mut target = TargetCommand::new(arguments.runsc_program.clone()); for argument in &arguments.runsc_arguments[..run] { target = target.argument(argument); } Ok(target .argument("delete") .argument("--force") .argument(container)) } fn attached_runsc_standard_io(target: &TargetCommand) -> io::Result { if target.parent_edge() != ParentEdge::ParentDeathKill { return Err(io::Error::other( "attached runsc requires the parent-death KILL edge", )); } Ok(ChildStandardIo::silenced()) } fn wait_for_runsc_ready(pid_file: &File, process: &RetainedProcess) -> io::Result { let deadline = Instant::now() + READINESS_TIMEOUT; let mut bytes = [0_u8; 32]; loop { require_lifetime_held()?; if process.has_exited()? { return Err(io::Error::other("attached runsc exited before readiness")); } let length = pid_file.read_at(&mut bytes, 0)?; if length > 0 { return std::str::from_utf8(&bytes[..length]) .ok() .map(str::trim) .and_then(|value| value.parse().ok()) .filter(|pid| *pid > 0) .ok_or_else(|| io::Error::other("runsc pid-file readiness is malformed")); } if Instant::now() >= deadline { return Err(io::Error::other("attached runsc readiness timed out")); } std::thread::sleep(Duration::from_millis(5)); } } fn supervise( control: &std::io::Stdout, process: RetainedProcess, mut network: Option, namespaces: (NamespaceIdentity, NamespaceIdentity), state_anchor: &mut Option, state_anchor_absence: &mut Option, delete: &TargetCommand, ) -> io::Result<()> { let mut process = Some(process); let mut state_anchor = StateAnchorSupervision { namespaces, mount: state_anchor, absence: state_anchor_absence, }; loop { let exited = match process .as_ref() .map_or(Ok(false), RetainedProcess::has_exited) { Ok(exited) => exited, Err(error) => return fail_active_runtime(process, network, error), }; if exited { child_broker().reap( process .take() .ok_or_else(|| io::Error::other("attached runsc handle is absent"))?, )?; child_broker().run_transient(delete)?; stop_network(network)?; send_message(control, &SupervisorEvent::AttachedRunscExited)?; return Ok(()); } if network .as_ref() .is_some_and(|network| network.helper_exited().unwrap_or(true)) { network .take() .ok_or_else(|| io::Error::other("connectivity helper handle is absent"))? .reap_exited()?; send_message(control, &SupervisorEvent::HelperExited)?; } let lease_lost = match lifetime_lost() { Ok(lost) => lost, Err(error) => return fail_active_runtime(process, network, error), }; if lease_lost { if let Some(process) = process.take() { terminate_and_reap(process)?; child_broker().run_transient(delete)?; } stop_network(network)?; return Ok(()); } let mut descriptors = [PollFd::new(control, PollFlags::IN)]; if let Err(error) = poll( &mut descriptors, Some(&Timespec { tv_sec: 0, tv_nsec: 10_000_000, }), ) { return fail_active_runtime(process, network, error.into()); } let events = descriptors[0].revents(); if events.intersects(PollFlags::HUP | PollFlags::ERR) { return fail_active_runtime( process, network, io::Error::new( io::ErrorKind::UnexpectedEof, "runtime control channel closed", ), ); } if events.intersects(PollFlags::IN) { let message = match receive_message(control) { Ok(message) => message, Err(error) => return fail_active_runtime(process, network, error), }; match handle_supervisor_message( control, &message, &mut process, &mut network, &mut state_anchor, delete, ) { Ok(true) => return Ok(()), Ok(false) => {} Err(error) => return fail_active_runtime(process, network, error), } } } } struct StateAnchorSupervision<'a> { namespaces: (NamespaceIdentity, NamespaceIdentity), mount: &'a mut Option, absence: &'a mut Option, } fn handle_supervisor_message( control: &std::io::Stdout, message: &CoordinatorMessage, process: &mut Option, network: &mut Option, state_anchor: &mut StateAnchorSupervision<'_>, delete: &TargetCommand, ) -> io::Result { match message { CoordinatorMessage::SignalAttachedRunsc(signal) => { process .as_ref() .ok_or_else(|| io::Error::other("attached runsc is already stopped"))? .send_signal(signal.signal())?; send_message(control, &SupervisorEvent::SignalApplied(*signal))?; } CoordinatorMessage::StopSandbox => { terminate_and_reap( process .take() .ok_or_else(|| io::Error::other("attached runsc is already stopped"))?, )?; child_broker().run_transient(delete)?; send_message(control, &SupervisorEvent::SandboxStopped)?; } CoordinatorMessage::StopSidecar => { if process.is_some() { return Err(io::Error::other("sidecar stop preceded sandbox stop")); } stop_network(network.take())?; send_message(control, &SupervisorEvent::SidecarStopped)?; } CoordinatorMessage::UnmountStateAnchor | CoordinatorMessage::ObserveStateAnchorAbsence { .. } => { handle_state_anchor_message( control, message, process.is_some() || network.is_some(), state_anchor, )?; } CoordinatorMessage::Exit => { if process.is_some() || network.is_some() || state_anchor.mount.is_some() { return Err(io::Error::other( "Runtime Supervisor exit preceded resource cleanup", )); } send_message(control, &SupervisorEvent::Exited)?; return Ok(true); } CoordinatorMessage::Stop => { if let Some(process) = process.take() { terminate_and_reap(process)?; } stop_network(network.take())?; send_message(control, &SupervisorEvent::Stopped)?; return Ok(true); } CoordinatorMessage::HelperStagedAck(_) | CoordinatorMessage::MountStateAnchor | CoordinatorMessage::StateAnchorStagedAck(_) | CoordinatorMessage::StartSidecar | CoordinatorMessage::NetworkAttestationAccepted(_) | CoordinatorMessage::NetworkReadinessCommitted | CoordinatorMessage::StagedAck(_) | CoordinatorMessage::SandboxCommitted => { return Err(io::Error::other("unexpected duplicate coordinator message")); } } Ok(false) } fn handle_state_anchor_message( control: &std::io::Stdout, message: &CoordinatorMessage, runtime_resources_present: bool, state: &mut StateAnchorSupervision<'_>, ) -> io::Result<()> { match message { CoordinatorMessage::UnmountStateAnchor => { if runtime_resources_present { return Err(io::Error::other( "State Anchor unmount preceded runtime resource stop", )); } let mount = state .mount .as_ref() .ok_or_else(|| io::Error::other("State Anchor is absent"))? .mount_identity(); let absence = state .mount .take() .ok_or_else(|| io::Error::other("State Anchor is absent"))? .unmount() .map_err(io::Error::other)?; *state.absence = Some(absence); send_message(control, &SupervisorEvent::StateAnchorUnmounted { mount }) } CoordinatorMessage::ObserveStateAnchorAbsence { user_namespace, mount_namespace, mount, } => { if (*user_namespace, *mount_namespace) != state.namespaces { return send_message( control, &SupervisorEvent::Refused( "State Anchor absence request namespace authority mismatched".into(), ), ); } match state.absence.as_ref().map_or_else( || { Err(lazaret_state_anchor::StateAnchorError::InvalidLayout( "State Anchor absence authority is absent", )) }, |authority| authority.observe_absence(*mount), ) { Ok(observed) => send_message( control, &SupervisorEvent::StateAnchorAbsenceObserved { user_namespace: state.namespaces.0, mount_namespace: state.namespaces.1, mount: *mount, observed, }, ), Err(error) => send_message(control, &SupervisorEvent::Refused(error.to_string())), } } _ => Err(io::Error::other("message is not a State Anchor command")), } } fn fail_active_runtime( process: Option, network: Option, error: io::Error, ) -> io::Result<()> { if let Some(process) = process { fail_with_network_cleanup(process, network, error) } else { let _ = stop_network(network); Err(error) } } fn terminate_and_reap(process: RetainedProcess) -> io::Result<()> { let term_error = process.send_signal(Signal::TERM).err(); let deadline = Instant::now() + TERMINATION_GRACE; while !process.has_exited()? && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(5)); } if !process.has_exited()? { process.send_signal(Signal::KILL)?; } child_broker().reap(process)?; if let Some(error) = term_error { Err(error) } else { Ok(()) } } fn fail_with_network_cleanup( process: RetainedProcess, network: Option, error: io::Error, ) -> io::Result<()> { let runsc_cleanup = terminate_and_reap(process); let network_cleanup = stop_network(network); match (runsc_cleanup, network_cleanup) { (Ok(()), Ok(())) => Err(error), (runsc, network) => Err(io::Error::new( error.kind(), format!("{error}; exact runtime cleanup failed: runsc={runsc:?}, network={network:?}"), )), } } fn fail_network(network: Option, error: io::Error) -> io::Result { match stop_network(network) { Ok(()) => Err(error), Err(cleanup) => Err(io::Error::new( error.kind(), format!("{error}; exact connectivity-helper cleanup failed: {cleanup}"), )), } } fn stop_network(network: Option) -> io::Result<()> { match network { Some(network) => network.stop(), None => Ok(()), } } fn require_lifetime_held() -> io::Result<()> { if lifetime_lost()? { Err(io::Error::other("Host Coordinator lifetime lease is lost")) } else { Ok(()) } } fn lifetime_lost() -> io::Result { let stderr = std::io::stderr(); let mut descriptors = [PollFd::new(&stderr, PollFlags::IN)]; poll(&mut descriptors, Some(&Timespec::default()))?; Ok(descriptors[0] .revents() .intersects(PollFlags::IN | PollFlags::HUP | PollFlags::ERR)) } fn send_message(descriptor: impl AsFd, message: &T) -> io::Result<()> { let encoded = encode(message).map_err(|error| io::Error::other(error.to_string()))?; if encoded.len() > MAX_CONTROL_MESSAGE { return Err(io::Error::other("runtime control message is oversized")); } let length = u32::try_from(encoded.len()) .map_err(|_| io::Error::other("runtime control message is oversized"))? .to_be_bytes(); send_all(descriptor.as_fd(), &length)?; send_all(descriptor.as_fd(), &encoded) } fn receive_message(descriptor: impl AsFd) -> io::Result where T: serde::de::DeserializeOwned + Serialize, { let mut length = [0_u8; 4]; receive_exact(descriptor.as_fd(), &mut length)?; let length = usize::try_from(u32::from_be_bytes(length)) .map_err(|_| io::Error::other("runtime control length is invalid"))?; if length > MAX_CONTROL_MESSAGE { return Err(io::Error::other("runtime control message is oversized")); } let mut encoded = vec![0_u8; length]; receive_exact(descriptor.as_fd(), &mut encoded)?; decode(&encoded).map_err(|error| io::Error::other(error.to_string())) } fn send_all(descriptor: impl AsFd + Copy, mut bytes: &[u8]) -> io::Result<()> { while !bytes.is_empty() { let sent = write(descriptor, bytes)?; if sent == 0 { return Err(io::Error::new( io::ErrorKind::WriteZero, "runtime control channel closed", )); } bytes = &bytes[sent..]; } Ok(()) } fn receive_exact(descriptor: impl AsFd + Copy, mut bytes: &mut [u8]) -> io::Result<()> { while !bytes.is_empty() { let received = read(descriptor, &mut *bytes)?; if received == 0 { return Err(io::Error::new( io::ErrorKind::UnexpectedEof, "runtime control channel closed", )); } bytes = &mut bytes[received..]; } Ok(()) } fn hex(bytes: &[u8]) -> String { use std::fmt::Write as _; bytes.iter().fold(String::new(), |mut output, byte| { write!(output, "{byte:02x}").expect("writing to a String cannot fail"); output }) } #[cfg(test)] mod tests { use std::ffi::OsString; use lazaret_core::identity::{CoordinatorId, DomainId, GenerationId}; use lazaret_core::records::HostProcessOwnershipSlot; use lazaret_linux::ipc::private_seqpacket_pair; use lazaret_linux::process::ChildStandardIoPolicy; use rustix::pipe::{PipeFlags, pipe_with}; use super::*; #[test] fn private_stream_framing_round_trips_typed_events() { let (coordinator, supervisor) = private_seqpacket_pair().unwrap(); let expected = SupervisorEvent::Refused("refused".into()); send_message(&supervisor, &expected).unwrap(); assert_eq!( CoordinatorChannel::new(coordinator).receive().unwrap(), expected ); } #[test] fn coordinator_channel_preserves_phased_resource_commands() { let (coordinator, supervisor) = private_seqpacket_pair().unwrap(); let channel = CoordinatorChannel::new(coordinator); channel.mount_state_anchor().unwrap(); assert_eq!( receive_message::(&supervisor).unwrap(), CoordinatorMessage::MountStateAnchor ); channel.start_sidecar().unwrap(); assert_eq!( receive_message::(&supervisor).unwrap(), CoordinatorMessage::StartSidecar ); channel.stop_sandbox().unwrap(); assert_eq!( receive_message::(&supervisor).unwrap(), CoordinatorMessage::StopSandbox ); channel.stop_sidecar().unwrap(); assert_eq!( receive_message::(&supervisor).unwrap(), CoordinatorMessage::StopSidecar ); channel.unmount_state_anchor().unwrap(); assert_eq!( receive_message::(&supervisor).unwrap(), CoordinatorMessage::UnmountStateAnchor ); let user_namespace = NamespaceIdentity { device: lazaret_core::identity::DeviceId::new(11), inode: lazaret_core::identity::InodeId::new(12), }; let mount_namespace = NamespaceIdentity { device: lazaret_core::identity::DeviceId::new(13), inode: lazaret_core::identity::InodeId::new(14), }; let mount = MountIdentity { mount_id: lazaret_core::identity::MountId::new(15), device: lazaret_core::identity::DeviceId::new(16), inode: lazaret_core::identity::InodeId::new(17), }; channel .observe_state_anchor_absence(user_namespace, mount_namespace, mount) .unwrap(); assert_eq!( receive_message::(&supervisor).unwrap(), CoordinatorMessage::ObserveStateAnchorAbsence { user_namespace, mount_namespace, mount, } ); channel.exit().unwrap(); assert_eq!( receive_message::(&supervisor).unwrap(), CoordinatorMessage::Exit ); } #[test] fn runtime_supervisor_and_runsc_keep_distinct_edge_and_stdio_policies() { let ownership = ProcessOwnershipContext { domain_id: DomainId::new([1; 32]), generation_id: GenerationId::new(7), coordinator_id: CoordinatorId::new([2; 32]), slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor, }; let launch = RuntimeSupervisorLaunch { target: TargetCommand::new("/proc/self/exe").with_lifetime_pipe_parent_edge(), ownership, policy_file: None, policy_digest: None, configuration_fingerprint: None, runsc_command: RunscCommand::new( "runsc", [OsString::from("run"), OsString::from("container")], [0; 32], ), network_sidecar_id: None, state_anchor: None, }; let (_coordinator_control, supervisor_control) = private_seqpacket_pair().unwrap(); let (supervisor_lifetime, _coordinator_lifetime) = pipe_with(PipeFlags::CLOEXEC).unwrap(); assert_eq!(launch.target().parent_edge(), ParentEdge::LifetimePipe); assert_eq!( launch .child_standard_io(supervisor_control, supervisor_lifetime) .unwrap() .policy(), ChildStandardIoPolicy::PrivateControlWithLifetimePipe ); let runsc_arguments = RuntimeArguments { ownership: ProcessOwnershipContext { slot: HostProcessOwnershipSlot::SandboxAttachedRunsc, ..ownership }, state_anchor: None, network: None, runsc_program: "runsc".into(), runsc_arguments: ["-root=/runtime", "run", "container"] .map(OsString::from) .into(), }; let runsc = runsc_target(&runsc_arguments, 9).unwrap(); assert_eq!(runsc.parent_edge(), ParentEdge::ParentDeathKill); assert_eq!( attached_runsc_standard_io(&runsc).unwrap().policy(), ChildStandardIoPolicy::Silenced ); let delete = runsc_delete_target(&runsc_arguments).unwrap(); assert_eq!(delete.program(), "runsc"); assert_eq!( delete.arguments(), ["-root=/runtime", "delete", "--force", "container"] .map(OsString::from) .as_slice() ); } #[test] fn runtime_role_parser_preserves_exact_ownership_and_runsc_arguments() { let domain = [1_u8; 32]; let coordinator = [2_u8; 32]; let arguments = [ "--domain".into(), hex(&domain).into(), "--generation".into(), "7".into(), "--coordinator".into(), hex(&coordinator).into(), "--runsc".into(), "runsc".into(), "-root=/runtime".into(), "run".into(), "container".into(), ]; let parsed = parse_runtime_arguments(&arguments).unwrap(); assert_eq!(parsed.ownership.domain_id, DomainId::new(domain)); assert_eq!(parsed.ownership.generation_id, GenerationId::new(7)); assert_eq!( parsed.ownership.coordinator_id, CoordinatorId::new(coordinator) ); assert_eq!( parsed.ownership.slot, HostProcessOwnershipSlot::SandboxAttachedRunsc ); assert_eq!(parsed.runsc_program, OsString::from("runsc")); assert_eq!( parsed.runsc_arguments, ["-root=/runtime", "run", "container"].map(OsString::from) ); } #[test] fn runtime_role_parser_refuses_non_runsc_target() { let arguments = [ "--domain".into(), hex(&[1_u8; 32]).into(), "--generation".into(), "7".into(), "--coordinator".into(), hex(&[2_u8; 32]).into(), "--runsc".into(), "foreign-runtime".into(), ]; assert!(parse_runtime_arguments(&arguments).is_err()); } }