ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Implement supervised rootless network readiness

Browse source
This commit is contained in:
ms2k 2026-07-15 23:26:37 +09:00
commit ba6a075839
1 changed files with 2917 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

351
crates/lazaret-runtime-supervisor/src/supervision.rs
View file

@ -6,11 +6,17 @@ use std::os::unix::fs::FileExt;
use std::time::{Duration, Instant};
use lazaret_core::codec::{decode, encode};
use lazaret_core::identity::{CoordinatorId, DomainId, GenerationId};
use lazaret_core::identity::{
ConfigurationFingerprint, CoordinatorId, DerivedPolicyDigest, DomainId, GenerationId,
NetworkSidecarId,
};
use lazaret_core::network::CanonicalDerivedNetworkPolicy;
use lazaret_core::records::{
HostProcessOwnershipSlot, ProcessOwnershipCommitAck, StagedProcessOwnership,
HostProcessOwnershipSlot, NetworkReadinessAttestation, ProcessOwnershipCommitAck,
StagedProcessOwnership,
};
use lazaret_linux::capability;
use lazaret_linux::filesystem::sealed_memfd;
use lazaret_linux::process::{
ChildStandardIo, ParentEdge, ProcessOwnershipContext, RetainedProcess, TargetCommand,
child_broker, current_process_identity,
@ -22,6 +28,10 @@ use rustix::process::Signal;
use rustix::thread::CapabilitySet;
use serde::{Deserialize, Serialize};
use crate::network::{
NetworkAttestationContext, NetworkProbeRequest, PreparedNetwork, prepare_network,
probe_current_network,
};
use crate::sandbox::RunscCommand;
const PRIVATE_ROLE_ARGUMENT: &str = "--lazaret-runtime-supervisor-role";
@ -48,6 +58,11 @@ impl AttachedRunscSignal {
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum SupervisorEvent {
HelperStaged(StagedProcessOwnership),
HelperReady(StagedProcessOwnership),
NetworkAttestation(NetworkReadinessAttestation),
NetworkProbe(lazaret_core::records::NetworkReachabilityProbe),
HelperExited,
Staged(StagedProcessOwnership),
Ready {
staged: StagedProcessOwnership,
@ -59,8 +74,11 @@ pub enum SupervisorEvent {
Refused(String),
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
enum CoordinatorMessage {
HelperStagedAck(ProcessOwnershipCommitAck),
NetworkAttestationAccepted(NetworkProbeRequest),
NetworkReadinessCommitted,
StagedAck(ProcessOwnershipCommitAck),
SandboxCommitted,
SignalAttachedRunsc(AttachedRunscSignal),
@ -92,6 +110,33 @@ impl CoordinatorChannel {
)
}
/// # Errors
/// Returns an error when the exact helper acknowledgment cannot be delivered.
pub fn acknowledge_helper(&self, acknowledgment: ProcessOwnershipCommitAck) -> io::Result<()> {
send_message(
&self.descriptor,
&CoordinatorMessage::HelperStagedAck(acknowledgment),
)
}
/// # Errors
/// Returns an error when accepted attestation cannot release sandbox acquisition.
pub fn acknowledge_network_attestation(&self, request: NetworkProbeRequest) -> io::Result<()> {
send_message(
&self.descriptor,
&CoordinatorMessage::NetworkAttestationAccepted(request),
)
}
/// # Errors
/// Returns an error when durable network readiness cannot release sandbox acquisition.
pub fn acknowledge_network_readiness(&self) -> io::Result<()> {
send_message(
&self.descriptor,
&CoordinatorMessage::NetworkReadinessCommitted,
)
}
/// # Errors
/// Returns an error when the durable sandbox commit cannot be acknowledged.
pub fn acknowledge_sandbox_committed(&self) -> io::Result<()> {
@ -117,6 +162,14 @@ impl CoordinatorChannel {
pub struct RuntimeSupervisorLaunch {
target: TargetCommand,
ownership: ProcessOwnershipContext,
_policy_file: Option<File>,
network_sidecar_id: Option<NetworkSidecarId>,
}
pub struct RuntimeNetworkLaunch<'policy> {
pub policy: &'policy CanonicalDerivedNetworkPolicy,
pub configuration_fingerprint: ConfigurationFingerprint,
pub network_sidecar_id: NetworkSidecarId,
}
impl RuntimeSupervisorLaunch {
@ -157,9 +210,57 @@ impl RuntimeSupervisorLaunch {
Ok(Self {
target: target.with_lifetime_pipe_parent_edge(),
ownership,
_policy_file: None,
network_sidecar_id: None,
})
}
/// Builds the RS-owned network path with one inherited sealed policy document.
///
/// # Errors
/// Refuses an invalid runsc form or a policy descriptor that cannot be sealed and inherited.
pub fn from_networked_runsc_command(
ownership: ProcessOwnershipContext,
command: &RunscCommand,
network: RuntimeNetworkLaunch<'_>,
) -> io::Result<Self> {
let runtime_supervisor_executable = std::env::current_exe()?;
let mut launch = Self::from_runsc_command(ownership, command)?;
let policy_file = sealed_memfd("lazaret-network-policy", network.policy.canonical_bytes())?;
fcntl_setfd(policy_file.as_fd(), FdFlags::empty())?;
let policy_fd = policy_file.as_raw_fd();
let target = std::mem::replace(
&mut launch.target,
TargetCommand::new("/usr/bin/false").with_lifetime_pipe_parent_edge(),
);
launch.target = TargetCommand::new(runtime_supervisor_executable)
.argument(PRIVATE_ROLE_ARGUMENT)
.argument("--domain")
.argument(hex(ownership.domain_id.as_bytes()))
.argument("--generation")
.argument(ownership.generation_id.get().to_string())
.argument("--coordinator")
.argument(hex(ownership.coordinator_id.as_bytes()))
.argument("--network-policy-fd")
.argument(policy_fd.to_string())
.argument("--network-policy-digest")
.argument(hex(network.policy.digest().as_bytes()))
.argument("--configuration-fingerprint")
.argument(hex(network.configuration_fingerprint.as_bytes()))
.argument("--network-sidecar")
.argument(hex(network.network_sidecar_id.as_bytes()))
.argument("--runsc")
.argument(command.program());
for argument in command.arguments() {
launch.target = launch.target.argument(argument);
}
launch.target = launch.target.with_lifetime_pipe_parent_edge();
drop(target);
launch._policy_file = Some(policy_file);
launch.network_sidecar_id = Some(network.network_sidecar_id);
Ok(launch)
}
#[must_use]
pub const fn target(&self) -> &TargetCommand {
&self.target
@ -170,6 +271,11 @@ impl RuntimeSupervisorLaunch {
self.ownership
}
#[must_use]
pub const fn network_sidecar_id(&self) -> Option<NetworkSidecarId> {
self.network_sidecar_id
}
/// # Errors
/// Refuses a Runtime Supervisor target without the lifetime-pipe parent edge.
pub fn child_standard_io(
@ -214,18 +320,34 @@ pub fn dispatch_private_runtime_role(arguments: &[OsString]) -> Option<io::Resul
struct RuntimeArguments {
ownership: ProcessOwnershipContext,
network: Option<RuntimeNetworkArguments>,
runsc_program: OsString,
runsc_arguments: Vec<OsString>,
}
struct RuntimeNetworkArguments {
policy_fd: i32,
policy_digest: DerivedPolicyDigest,
configuration_fingerprint: ConfigurationFingerprint,
network_sidecar_id: NetworkSidecarId,
}
fn parse_runtime_arguments(arguments: &[OsString]) -> io::Result<RuntimeArguments> {
let mut arguments = arguments.iter();
let mut arguments = arguments.iter().peekable();
let domain = parse_fixed_hex(named(&mut arguments, "--domain")?)?;
let generation = named(&mut arguments, "--generation")?
.to_str()
.and_then(|value| value.parse().ok())
.ok_or_else(|| io::Error::other("Runtime Supervisor generation is invalid"))?;
let coordinator = parse_fixed_hex(named(&mut arguments, "--coordinator")?)?;
let network = if arguments
.peek()
.is_some_and(|value| *value == "--network-policy-fd")
{
Some(parse_network_arguments(&mut arguments)?)
} else {
None
};
if arguments.next().is_none_or(|value| value != "--runsc") {
return Err(io::Error::other(
"Runtime Supervisor runsc separator is missing",
@ -244,11 +366,38 @@ fn parse_runtime_arguments(arguments: &[OsString]) -> io::Result<RuntimeArgument
coordinator_id: CoordinatorId::new(coordinator),
slot: HostProcessOwnershipSlot::SandboxAttachedRunsc,
},
network,
runsc_program: (*program).clone(),
runsc_arguments: arguments.cloned().collect(),
})
}
fn parse_network_arguments<'a>(
arguments: &mut impl Iterator<Item = &'a OsString>,
) -> io::Result<RuntimeNetworkArguments> {
let policy_fd = named(arguments, "--network-policy-fd")?
.to_str()
.and_then(|value| value.parse().ok())
.filter(|value| *value >= 0)
.ok_or_else(|| io::Error::other("Runtime Supervisor policy descriptor is invalid"))?;
let policy_digest = DerivedPolicyDigest::new(parse_fixed_hex(named(
arguments,
"--network-policy-digest",
)?)?);
let configuration_fingerprint = ConfigurationFingerprint::new(parse_fixed_hex(named(
arguments,
"--configuration-fingerprint",
)?)?);
let network_sidecar_id =
NetworkSidecarId::new(parse_fixed_hex(named(arguments, "--network-sidecar")?)?);
Ok(RuntimeNetworkArguments {
policy_fd,
policy_digest,
configuration_fingerprint,
network_sidecar_id,
})
}
fn named<'a>(
arguments: &mut impl Iterator<Item = &'a OsString>,
name: &str,
@ -282,11 +431,17 @@ fn run_supervisor(arguments: &RuntimeArguments) -> io::Result<()> {
require_lifetime_held()?;
let control = std::io::stdout();
let parent = current_process_identity()?;
let network = match &arguments.network {
Some(network) => Some(prepare_runtime_network(
arguments, network, parent, &control,
)?),
None => None,
};
let pid_file = File::from(memfd_create("lazaret-runsc-pid", MemfdFlags::empty())?);
fcntl_setfd(pid_file.as_fd(), FdFlags::empty())?;
let runsc = runsc_target(arguments, pid_file.as_raw_fd())?;
let staged = std::cell::Cell::new(None);
let process = child_broker().acquire_with_stdio(
let process = match child_broker().acquire_with_stdio(
parent,
arguments.ownership,
&runsc,
@ -305,25 +460,29 @@ fn run_supervisor(arguments: &RuntimeArguments) -> io::Result<()> {
"Runtime Supervisor received a mismatched staged acknowledgment",
)),
},
)?;
) {
Ok(process) => process,
Err(error) => return fail_network(network, error),
};
if let Err(error) = fcntl_setfd(pid_file.as_fd(), FdFlags::CLOEXEC) {
return fail_with_exact_cleanup(process, error.into());
return fail_with_network_cleanup(process, network, error.into());
}
let Some(staged) = staged.get() else {
return fail_with_exact_cleanup(
return fail_with_network_cleanup(
process,
network,
io::Error::other("attached runsc ownership was not staged"),
);
};
let sentry_pid_inventory = match wait_for_runsc_ready(&pid_file, &process) {
Ok(pid) => pid,
Err(error) => return fail_with_exact_cleanup(process, error),
Err(error) => return fail_with_network_cleanup(process, network, error),
};
if let Err(error) = capability::irreversibly_drop(CapabilitySet::NET_ADMIN) {
return fail_with_exact_cleanup(process, error);
return fail_with_network_cleanup(process, network, error);
}
if let Err(error) = capability::require_regain_refused(CapabilitySet::NET_ADMIN) {
return fail_with_exact_cleanup(process, error);
return fail_with_network_cleanup(process, network, error);
}
if let Err(error) = send_message(
&control,
@ -332,19 +491,99 @@ fn run_supervisor(arguments: &RuntimeArguments) -> io::Result<()> {
sentry_pid_inventory,
},
) {
return fail_with_exact_cleanup(process, error);
return fail_with_network_cleanup(process, network, error);
}
let committed: CoordinatorMessage = match receive_message(&control) {
Ok(committed) => committed,
Err(error) => return fail_with_exact_cleanup(process, error),
Err(error) => return fail_with_network_cleanup(process, network, error),
};
if committed != CoordinatorMessage::SandboxCommitted {
return fail_with_exact_cleanup(
return fail_with_network_cleanup(
process,
network,
io::Error::other("sandbox commit acknowledgment was not received"),
);
}
supervise(&control, process)
supervise(&control, process, network)
}
fn prepare_runtime_network(
arguments: &RuntimeArguments,
network: &RuntimeNetworkArguments,
runtime_supervisor: lazaret_core::identity::PidfsProcessIdentityV1,
control: &std::io::Stdout,
) -> io::Result<PreparedNetwork> {
let policy_file = File::open(format!("/proc/self/fd/{}", network.policy_fd))?;
let staged = std::cell::Cell::new(None);
let prepared = prepare_network(
&policy_file,
network.policy_digest,
NetworkAttestationContext {
domain_id: arguments.ownership.domain_id,
generation_id: arguments.ownership.generation_id,
configuration_fingerprint: network.configuration_fingerprint,
network_sidecar_id: network.network_sidecar_id,
runtime_supervisor,
coordinator_id: arguments.ownership.coordinator_id,
},
|ownership| {
staged.set(Some(ownership));
send_message(control, &SupervisorEvent::HelperStaged(ownership))
},
|ownership| match receive_message(control)? {
CoordinatorMessage::HelperStagedAck(acknowledgment)
if acknowledgment.acknowledges(&ownership) =>
{
Ok(acknowledgment)
}
_ => Err(io::Error::other(
"Runtime Supervisor received a mismatched helper acknowledgment",
)),
},
)?;
let staged = staged
.get()
.ok_or_else(|| io::Error::other("connectivity helper ownership was not staged"))?;
if let Err(error) = send_message(control, &SupervisorEvent::HelperReady(staged)) {
return fail_network(Some(prepared), error);
}
if let Err(error) = send_message(
control,
&SupervisorEvent::NetworkAttestation(prepared.attestation().clone()),
) {
return fail_network(Some(prepared), error);
}
let message: CoordinatorMessage = match receive_message(control) {
Ok(message) => message,
Err(error) => return fail_network(Some(prepared), error),
};
let request = match message {
CoordinatorMessage::NetworkAttestationAccepted(request) => request,
_ => {
return fail_network(
Some(prepared),
io::Error::other("network attestation acceptance was not received"),
);
}
};
let probe = match probe_current_network(&request) {
Ok(probe) => probe,
Err(error) => return fail_network(Some(prepared), error),
};
if let Err(error) = send_message(control, &SupervisorEvent::NetworkProbe(probe)) {
return fail_network(Some(prepared), error);
}
let committed: CoordinatorMessage = match receive_message(control) {
Ok(message) => message,
Err(error) => return fail_network(Some(prepared), error),
};
if committed != CoordinatorMessage::NetworkReadinessCommitted {
return fail_network(
Some(prepared),
io::Error::other("durable network readiness was not acknowledged"),
);
}
Ok(prepared)
}
fn runsc_target(arguments: &RuntimeArguments, pid_file: i32) -> io::Result<TargetCommand> {
@ -397,23 +636,39 @@ fn wait_for_runsc_ready(pid_file: &File, process: &RetainedProcess) -> io::Resul
}
}
fn supervise(control: &std::io::Stdout, process: RetainedProcess) -> io::Result<()> {
fn supervise(
control: &std::io::Stdout,
process: RetainedProcess,
mut network: Option<PreparedNetwork>,
) -> io::Result<()> {
loop {
let exited = match process.has_exited() {
Ok(exited) => exited,
Err(error) => return fail_with_exact_cleanup(process, error),
Err(error) => return fail_with_network_cleanup(process, network, error),
};
if exited {
child_broker().reap(process)?;
stop_network(network)?;
send_message(control, &SupervisorEvent::AttachedRunscExited)?;
return Ok(());
}
if network
.as_ref()
.is_some_and(|network| network.helper_exited().unwrap_or(true))
{
network
.take()
.ok_or_else(|| io::Error::other("connectivity helper handle is absent"))?
.reap_exited()?;
send_message(control, &SupervisorEvent::HelperExited)?;
}
let lease_lost = match lifetime_lost() {
Ok(lost) => lost,
Err(error) => return fail_with_exact_cleanup(process, error),
Err(error) => return fail_with_network_cleanup(process, network, error),
};
if lease_lost {
terminate_and_reap(process)?;
stop_network(network)?;
return Ok(());
}
let mut descriptors = [PollFd::new(control, PollFlags::IN)];
@ -424,12 +679,13 @@ fn supervise(control: &std::io::Stdout, process: RetainedProcess) -> io::Result<
tv_nsec: 10_000_000,
}),
) {
return fail_with_exact_cleanup(process, error.into());
return fail_with_network_cleanup(process, network, error.into());
}
let events = descriptors[0].revents();
if events.intersects(PollFlags::HUP | PollFlags::ERR) {
return fail_with_exact_cleanup(
return fail_with_network_cleanup(
process,
network,
io::Error::new(
io::ErrorKind::UnexpectedEof,
"runtime control channel closed",
@ -439,27 +695,33 @@ fn supervise(control: &std::io::Stdout, process: RetainedProcess) -> io::Result<
if events.intersects(PollFlags::IN) {
let message = match receive_message(control) {
Ok(message) => message,
Err(error) => return fail_with_exact_cleanup(process, error),
Err(error) => return fail_with_network_cleanup(process, network, error),
};
match message {
CoordinatorMessage::SignalAttachedRunsc(signal) => {
if let Err(error) = process.send_signal(signal.signal()) {
return fail_with_exact_cleanup(process, error);
return fail_with_network_cleanup(process, network, error);
}
if let Err(error) =
send_message(control, &SupervisorEvent::SignalApplied(signal))
{
return fail_with_exact_cleanup(process, error);
return fail_with_network_cleanup(process, network, error);
}
}
CoordinatorMessage::Stop => {
terminate_and_reap(process)?;
stop_network(network)?;
send_message(control, &SupervisorEvent::Stopped)?;
return Ok(());
}
CoordinatorMessage::StagedAck(_) | CoordinatorMessage::SandboxCommitted => {
return fail_with_exact_cleanup(
CoordinatorMessage::HelperStagedAck(_)
| CoordinatorMessage::NetworkAttestationAccepted(_)
| CoordinatorMessage::NetworkReadinessCommitted
| CoordinatorMessage::StagedAck(_)
| CoordinatorMessage::SandboxCommitted => {
return fail_with_network_cleanup(
process,
network,
io::Error::other("unexpected duplicate coordinator message"),
);
}
@ -485,17 +747,39 @@ fn terminate_and_reap(process: RetainedProcess) -> io::Result<()> {
}
}
fn fail_with_exact_cleanup(process: RetainedProcess, error: io::Error) -> io::Result<()> {
let cleanup = terminate_and_reap(process);
match cleanup {
fn fail_with_network_cleanup(
process: RetainedProcess,
network: Option<PreparedNetwork>,
error: io::Error,
) -> io::Result<()> {
let runsc_cleanup = terminate_and_reap(process);
let network_cleanup = stop_network(network);
match (runsc_cleanup, network_cleanup) {
(Ok(()), Ok(())) => Err(error),
(runsc, network) => Err(io::Error::new(
error.kind(),
format!("{error}; exact runtime cleanup failed: runsc={runsc:?}, network={network:?}"),
)),
}
}
fn fail_network<T>(network: Option<PreparedNetwork>, error: io::Error) -> io::Result<T> {
match stop_network(network) {
Ok(()) => Err(error),
Err(cleanup) => Err(io::Error::new(
error.kind(),
format!("{error}; exact attached-runsc cleanup failed: {cleanup}"),
format!("{error}; exact connectivity-helper cleanup failed: {cleanup}"),
)),
}
}
fn stop_network(network: Option<PreparedNetwork>) -> io::Result<()> {
match network {
Some(network) => network.stop(),
None => Ok(()),
}
}
fn require_lifetime_held() -> io::Result<()> {
if lifetime_lost()? {
Err(io::Error::other("Host Coordinator lifetime lease is lost"))
@ -584,7 +868,7 @@ mod tests {
use lazaret_core::identity::{CoordinatorId, DomainId, GenerationId};
use lazaret_core::records::HostProcessOwnershipSlot;
use lazaret_linux::ipc::private_stream_pair;
use lazaret_linux::ipc::private_seqpacket_pair;
use lazaret_linux::process::ChildStandardIoPolicy;
use rustix::pipe::{PipeFlags, pipe_with};
@ -592,7 +876,7 @@ mod tests {
#[test]
fn private_stream_framing_round_trips_typed_events() {
let (coordinator, supervisor) = private_stream_pair().unwrap();
let (coordinator, supervisor) = private_seqpacket_pair().unwrap();
let expected = SupervisorEvent::Refused("refused".into());
send_message(&supervisor, &expected).unwrap();
assert_eq!(
@ -612,8 +896,10 @@ mod tests {
let launch = RuntimeSupervisorLaunch {
target: TargetCommand::new("/proc/self/exe").with_lifetime_pipe_parent_edge(),
ownership,
_policy_file: None,
network_sidecar_id: None,
};
let (_coordinator_control, supervisor_control) = private_stream_pair().unwrap();
let (_coordinator_control, supervisor_control) = private_seqpacket_pair().unwrap();
let (supervisor_lifetime, _coordinator_lifetime) = pipe_with(PipeFlags::CLOEXEC).unwrap();
assert_eq!(launch.target().parent_edge(), ParentEdge::LifetimePipe);
assert_eq!(
@ -629,6 +915,7 @@ mod tests {
slot: HostProcessOwnershipSlot::SandboxAttachedRunsc,
..ownership
},
network: None,
runsc_program: "runsc".into(),
runsc_arguments: ["-root=/runtime", "run", "container"]
.map(OsString::from)