ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Decompose oversized Rust files by responsibility

Browse source
This commit is contained in:
ms2k 2026-07-19 03:02:44 +09:00
commit ee9201b057
3 changed files with 18648 additions and 18513 deletions
Download patch file Download diff file Expand all files Collapse all files

609
crates/lazaret-runtime-supervisor/src/supervision/runtime.rs Normal file
View file

@ -0,0 +1,609 @@
fn runsc_target(arguments: &RuntimeArguments, pid_file: i32) -> io::Result<TargetCommand> {
let run = arguments
.runsc_arguments
.iter()
.position(|argument| argument == "run")
.ok_or_else(|| io::Error::other("attached runsc command has no run operation"))?;
let mut target = TargetCommand::new(arguments.runsc_program.clone());
for argument in &arguments.runsc_arguments[..=run] {
target = target.argument(argument);
}
target = target.argument(format!("-pid-file=/proc/self/fd/{pid_file}"));
for argument in &arguments.runsc_arguments[run + 1..] {
target = target.argument(argument);
}
Ok(target.with_parent_death_kill_edge())
}
fn runsc_delete_target(arguments: &RuntimeArguments) -> io::Result<TargetCommand> {
let run = arguments
.runsc_arguments
.iter()
.position(|argument| argument == "run")
.ok_or_else(|| io::Error::other("attached runsc command has no run operation"))?;
let container = arguments
.runsc_arguments
.last()
.filter(|container| {
!container.is_empty() && !container.as_encoded_bytes().starts_with(b"-")
})
.ok_or_else(|| io::Error::other("attached runsc command has no container ID"))?;
let mut target = TargetCommand::new(arguments.runsc_program.clone());
for argument in &arguments.runsc_arguments[..run] {
target = target.argument(argument);
}
Ok(target
.argument("delete")
.argument("--force")
.argument(container))
}
fn attached_runsc_standard_io(target: &TargetCommand) -> io::Result<ChildStandardIo> {
if target.parent_edge() != ParentEdge::ParentDeathKill {
return Err(io::Error::other(
"attached runsc requires the parent-death KILL edge",
));
}
Ok(ChildStandardIo::silenced())
}
fn wait_for_runsc_ready(pid_file: &File, process: &RetainedProcess) -> io::Result<u32> {
let deadline = Instant::now() + READINESS_TIMEOUT;
let mut bytes = [0_u8; 32];
loop {
require_lifetime_held()?;
if process.has_exited()? {
return Err(io::Error::other("attached runsc exited before readiness"));
}
let length = pid_file.read_at(&mut bytes, 0)?;
if length > 0 {
return std::str::from_utf8(&bytes[..length])
.ok()
.map(str::trim)
.and_then(|value| value.parse().ok())
.filter(|pid| *pid > 0)
.ok_or_else(|| io::Error::other("runsc pid-file readiness is malformed"));
}
if Instant::now() >= deadline {
return Err(io::Error::other("attached runsc readiness timed out"));
}
std::thread::sleep(Duration::from_millis(5));
}
}
#[allow(clippy::too_many_arguments, clippy::too_many_lines)]
fn supervise(
control: &std::io::Stdout,
process: RetainedProcess,
mut git_routing: Option<RuntimeGitRouting>,
mut network: Option<PreparedNetwork>,
namespaces: (NamespaceIdentity, NamespaceIdentity),
state_anchor: &mut Option<RuntimeStateAnchorMount>,
state_anchor_absence: &mut Option<RuntimeStateAnchorAbsenceAuthority>,
delete: &TargetCommand,
) -> io::Result<()> {
let mut process = Some(process);
let mut state_anchor = StateAnchorSupervision {
namespaces,
mount: state_anchor,
absence: state_anchor_absence,
};
loop {
let exited = match process
.as_ref()
.map_or(Ok(false), RetainedProcess::has_exited)
{
Ok(exited) => exited,
Err(error) => return fail_active_runtime(process, git_routing, network, error),
};
if exited {
let runsc = process
.take()
.ok_or_else(|| io::Error::other("attached runsc handle is absent"))?;
cleanup_stopped_runtime(runsc, git_routing, network, delete, false)?;
send_message(control, &SupervisorEvent::AttachedRunscExited)?;
return Ok(());
}
if git_routing
.as_ref()
.is_some_and(|routing| routing.process.has_exited().unwrap_or(true))
{
return fail_active_runtime(
process,
git_routing,
network,
io::Error::other("Git router exited while the sandbox was active"),
);
}
if network
.as_ref()
.is_some_and(|network| network.helper_exited().unwrap_or(true))
{
network
.take()
.ok_or_else(|| io::Error::other("connectivity helper handle is absent"))?
.reap_exited()?;
send_message(control, &SupervisorEvent::HelperExited)?;
}
let lease_lost = match lifetime_lost() {
Ok(lost) => lost,
Err(error) => return fail_active_runtime(process, git_routing, network, error),
};
if lease_lost {
if let Some(process) = process.take() {
cleanup_stopped_runtime(process, git_routing, network, delete, true)?;
} else {
cleanup_runtime_components(git_routing, network)?;
}
return Ok(());
}
let mut descriptors = [PollFd::new(control, PollFlags::IN)];
if let Err(error) = poll(
&mut descriptors,
Some(&Timespec {
tv_sec: 0,
tv_nsec: 10_000_000,
}),
) {
return fail_active_runtime(process, git_routing, network, error.into());
}
let events = descriptors[0].revents();
if events.intersects(PollFlags::HUP | PollFlags::ERR) {
return fail_active_runtime(
process,
git_routing,
network,
io::Error::new(
io::ErrorKind::UnexpectedEof,
"runtime control channel closed",
),
);
}
if events.intersects(PollFlags::IN) {
let message = match receive_message(control) {
Ok(message) => message,
Err(error) => {
return fail_active_runtime(process, git_routing, network, error);
}
};
match handle_supervisor_message(
control,
&message,
&mut process,
&mut git_routing,
&mut network,
&mut state_anchor,
delete,
) {
Ok(true) => return Ok(()),
Ok(false) => {}
Err(error) => {
return fail_active_runtime(process, git_routing, network, error);
}
}
}
}
}
struct StateAnchorSupervision<'a> {
namespaces: (NamespaceIdentity, NamespaceIdentity),
mount: &'a mut Option<RuntimeStateAnchorMount>,
absence: &'a mut Option<RuntimeStateAnchorAbsenceAuthority>,
}
fn handle_supervisor_message(
control: &std::io::Stdout,
message: &CoordinatorMessage,
process: &mut Option<RetainedProcess>,
git_routing: &mut Option<RuntimeGitRouting>,
network: &mut Option<PreparedNetwork>,
state_anchor: &mut StateAnchorSupervision<'_>,
delete: &TargetCommand,
) -> io::Result<bool> {
match message {
CoordinatorMessage::SignalAttachedRunsc(signal) => {
process
.as_ref()
.ok_or_else(|| io::Error::other("attached runsc is already stopped"))?
.send_signal(signal.signal())?;
send_message(control, &SupervisorEvent::SignalApplied(*signal))?;
}
CoordinatorMessage::StopSandbox => {
terminate_and_reap(
process
.take()
.ok_or_else(|| io::Error::other("attached runsc is already stopped"))?,
)?;
child_broker().run_transient(delete)?;
send_message(control, &SupervisorEvent::SandboxStopped)?;
}
CoordinatorMessage::StopGitRouting => {
if process.is_some() {
return Err(io::Error::other("Git-routing stop preceded sandbox stop"));
}
let routing = git_routing
.take()
.ok_or_else(|| io::Error::other("Git routing is already stopped"))?;
let mount = routing.mount;
stop_git_routing(Some(routing))?;
send_message(control, &SupervisorEvent::GitRoutingStopped { mount })?;
}
CoordinatorMessage::StopSidecar => {
if process.is_some() {
return Err(io::Error::other("sidecar stop preceded sandbox stop"));
}
stop_network(network.take())?;
send_message(control, &SupervisorEvent::SidecarStopped)?;
}
CoordinatorMessage::UnmountStateAnchor
| CoordinatorMessage::ObserveStateAnchorAbsence { .. } => {
handle_state_anchor_message(
control,
message,
process.is_some() || git_routing.is_some() || network.is_some(),
state_anchor,
)?;
}
CoordinatorMessage::Exit => {
if process.is_some()
|| git_routing.is_some()
|| network.is_some()
|| state_anchor.mount.is_some()
{
return Err(io::Error::other(
"Runtime Supervisor exit preceded resource cleanup",
));
}
send_message(control, &SupervisorEvent::Exited)?;
return Ok(true);
}
CoordinatorMessage::Stop => {
if let Some(process) = process.take() {
terminate_and_reap(process)?;
}
stop_network(network.take())?;
stop_git_routing(git_routing.take())?;
send_message(control, &SupervisorEvent::Stopped)?;
return Ok(true);
}
CoordinatorMessage::HelperStagedAck(_)
| CoordinatorMessage::MountStateAnchor
| CoordinatorMessage::StateAnchorStagedAck(_)
| CoordinatorMessage::StartGitRouting
| CoordinatorMessage::GitRoutingStagedAck(_)
| CoordinatorMessage::StartSidecar
| CoordinatorMessage::NetworkAttestationAccepted(_)
| CoordinatorMessage::NetworkReadinessCommitted
| CoordinatorMessage::StagedAck(_)
| CoordinatorMessage::SandboxCommitted => {
return Err(io::Error::other("unexpected duplicate coordinator message"));
}
}
Ok(false)
}
fn handle_state_anchor_message(
control: &std::io::Stdout,
message: &CoordinatorMessage,
runtime_resources_present: bool,
state: &mut StateAnchorSupervision<'_>,
) -> io::Result<()> {
match message {
CoordinatorMessage::UnmountStateAnchor => {
if runtime_resources_present {
return Err(io::Error::other(
"State Anchor unmount preceded runtime resource stop",
));
}
let mount = state
.mount
.as_ref()
.ok_or_else(|| io::Error::other("State Anchor is absent"))?
.mount_identity();
let absence = state
.mount
.take()
.ok_or_else(|| io::Error::other("State Anchor is absent"))?
.unmount()
.map_err(io::Error::other)?;
*state.absence = Some(absence);
send_message(control, &SupervisorEvent::StateAnchorUnmounted { mount })
}
CoordinatorMessage::ObserveStateAnchorAbsence {
user_namespace,
mount_namespace,
mount,
} => {
if (*user_namespace, *mount_namespace) != state.namespaces {
return send_message(
control,
&SupervisorEvent::Refused(
"State Anchor absence request namespace authority mismatched".into(),
),
);
}
match state.absence.as_ref().map_or_else(
|| {
Err(lazaret_state_anchor::StateAnchorError::InvalidLayout(
"State Anchor absence authority is absent",
))
},
|authority| authority.observe_absence(*mount),
) {
Ok(observed) => send_message(
control,
&SupervisorEvent::StateAnchorAbsenceObserved {
user_namespace: state.namespaces.0,
mount_namespace: state.namespaces.1,
mount: *mount,
observed,
},
),
Err(error) => send_message(control, &SupervisorEvent::Refused(error.to_string())),
}
}
_ => Err(io::Error::other("message is not a State Anchor command")),
}
}
fn fail_active_runtime(
process: Option<RetainedProcess>,
git_routing: Option<RuntimeGitRouting>,
network: Option<PreparedNetwork>,
error: io::Error,
) -> io::Result<()> {
if let Some(process) = process {
fail_with_runtime_cleanup(process, git_routing, network, error)
} else {
fail_runtime_components(git_routing, network, error)
}
}
fn terminate_and_reap(process: RetainedProcess) -> io::Result<()> {
let term_error = process.send_signal(Signal::TERM).err();
let deadline = Instant::now() + TERMINATION_GRACE;
while !process.has_exited()? && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(5));
}
if !process.has_exited()? {
process.send_signal(Signal::KILL)?;
}
child_broker().reap(process)?;
if let Some(error) = term_error {
Err(error)
} else {
Ok(())
}
}
fn fail_with_runtime_cleanup(
process: RetainedProcess,
git_routing: Option<RuntimeGitRouting>,
network: Option<PreparedNetwork>,
error: io::Error,
) -> io::Result<()> {
let runsc_cleanup = terminate_and_reap(process);
let network_cleanup = stop_network(network);
let routing_cleanup = stop_git_routing(git_routing);
match (runsc_cleanup, routing_cleanup, network_cleanup) {
(Ok(()), Ok(()), Ok(())) => Err(error),
(runsc, routing, network) => Err(io::Error::new(
error.kind(),
format!(
"{error}; exact runtime cleanup failed: runsc={runsc:?}, Git-routing={routing:?}, network={network:?}"
),
)),
}
}
fn cleanup_stopped_runtime(
process: RetainedProcess,
git_routing: Option<RuntimeGitRouting>,
network: Option<PreparedNetwork>,
delete: &TargetCommand,
terminate: bool,
) -> io::Result<()> {
let runsc_cleanup = if terminate {
terminate_and_reap(process)
} else {
child_broker().reap(process)
};
let delete_cleanup = child_broker().run_transient(delete).map(drop);
let network_cleanup = stop_network(network);
let routing_cleanup = stop_git_routing(git_routing);
let failures = [
("attached runsc", runsc_cleanup),
("runsc container", delete_cleanup),
("network", network_cleanup),
("Git routing", routing_cleanup),
]
.into_iter()
.filter_map(|(component, result)| result.err().map(|error| format!("{component}: {error}")))
.collect::<Vec<_>>();
if failures.is_empty() {
Ok(())
} else {
Err(io::Error::other(failures.join("; ")))
}
}
fn fail_runtime_components<T>(
git_routing: Option<RuntimeGitRouting>,
network: Option<PreparedNetwork>,
error: io::Error,
) -> io::Result<T> {
match cleanup_runtime_components(git_routing, network) {
Ok(()) => Err(error),
Err(cleanup) => Err(io::Error::new(
error.kind(),
format!("{error}; exact component cleanup failed: {cleanup}"),
)),
}
}
fn cleanup_runtime_components(
git_routing: Option<RuntimeGitRouting>,
network: Option<PreparedNetwork>,
) -> io::Result<()> {
let routing_cleanup = stop_git_routing(git_routing);
let network_cleanup = stop_network(network);
match (routing_cleanup, network_cleanup) {
(Ok(()), Ok(())) => Ok(()),
(routing, network) => Err(io::Error::other(format!(
"Git-routing={routing:?}, network={network:?}"
))),
}
}
fn stop_git_routing(routing: Option<RuntimeGitRouting>) -> io::Result<()> {
let Some(routing) = routing else {
return Ok(());
};
let RuntimeGitRouting {
process,
shutdown,
sentry,
mountpoint,
mount,
} = routing;
let sentry_cleanup = sentry.unmount();
drop(shutdown);
let process_cleanup = wait_and_reap(process);
let mount_cleanup = inherited_directory_at(&mountpoint)
.and_then(mount_identity)
.and_then(|observed| {
if observed == mount {
Err(io::Error::other(
"Git-routing mount remains after router shutdown",
))
} else {
Ok(())
}
});
let failures = [
("Git Sentry", sentry_cleanup),
("router process", process_cleanup),
("router mount", mount_cleanup),
]
.into_iter()
.filter_map(|(component, result)| result.err().map(|error| format!("{component}: {error}")))
.collect::<Vec<_>>();
if !failures.is_empty() {
return Err(io::Error::other(failures.join("; ")));
}
Ok(())
}
fn wait_and_reap(process: RetainedProcess) -> io::Result<()> {
let deadline = Instant::now() + TERMINATION_GRACE;
while !process.has_exited()? && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(5));
}
if !process.has_exited()? {
return terminate_and_reap(process).and_then(|()| {
Err(io::Error::other(
"Git router did not exit after its shutdown channel closed",
))
});
}
child_broker().reap(process)
}
fn fail_network<T>(network: Option<PreparedNetwork>, error: io::Error) -> io::Result<T> {
match stop_network(network) {
Ok(()) => Err(error),
Err(cleanup) => Err(io::Error::new(
error.kind(),
format!("{error}; exact connectivity-helper cleanup failed: {cleanup}"),
)),
}
}
fn stop_network(network: Option<PreparedNetwork>) -> io::Result<()> {
match network {
Some(network) => network.stop(),
None => Ok(()),
}
}
fn require_lifetime_held() -> io::Result<()> {
if lifetime_lost()? {
Err(io::Error::other("Host Coordinator lifetime lease is lost"))
} else {
Ok(())
}
}
fn lifetime_lost() -> io::Result<bool> {
let stderr = std::io::stderr();
let mut descriptors = [PollFd::new(&stderr, PollFlags::IN)];
poll(&mut descriptors, Some(&Timespec::default()))?;
Ok(descriptors[0]
.revents()
.intersects(PollFlags::IN | PollFlags::HUP | PollFlags::ERR))
}
fn send_message<T: Serialize>(descriptor: impl AsFd, message: &T) -> io::Result<()> {
let encoded = encode(message).map_err(|error| io::Error::other(error.to_string()))?;
if encoded.len() > MAX_CONTROL_MESSAGE {
return Err(io::Error::other("runtime control message is oversized"));
}
let length = u32::try_from(encoded.len())
.map_err(|_| io::Error::other("runtime control message is oversized"))?
.to_be_bytes();
send_all(descriptor.as_fd(), &length)?;
send_all(descriptor.as_fd(), &encoded)
}
fn receive_message<T>(descriptor: impl AsFd) -> io::Result<T>
where
T: serde::de::DeserializeOwned + Serialize,
{
let mut length = [0_u8; 4];
receive_exact(descriptor.as_fd(), &mut length)?;
let length = usize::try_from(u32::from_be_bytes(length))
.map_err(|_| io::Error::other("runtime control length is invalid"))?;
if length > MAX_CONTROL_MESSAGE {
return Err(io::Error::other("runtime control message is oversized"));
}
let mut encoded = vec![0_u8; length];
receive_exact(descriptor.as_fd(), &mut encoded)?;
decode(&encoded).map_err(|error| io::Error::other(error.to_string()))
}
fn send_all(descriptor: impl AsFd + Copy, mut bytes: &[u8]) -> io::Result<()> {
while !bytes.is_empty() {
let sent = write(descriptor, bytes)?;
if sent == 0 {
return Err(io::Error::new(
io::ErrorKind::WriteZero,
"runtime control channel closed",
));
}
bytes = &bytes[sent..];
}
Ok(())
}
fn receive_exact(descriptor: impl AsFd + Copy, mut bytes: &mut [u8]) -> io::Result<()> {
while !bytes.is_empty() {
let received = read(descriptor, &mut *bytes)?;
if received == 0 {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"runtime control channel closed",
));
}
bytes = &mut bytes[received..];
}
Ok(())
}
fn hex(bytes: &[u8]) -> String {
use std::fmt::Write as _;
bytes.iter().fold(String::new(), |mut output, byte| {
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
output
})
}

653
crates/lazaret-runtime-supervisor/src/supervision/startup.rs Normal file
View file

@ -0,0 +1,653 @@
fn run_supervisor(arguments: &RuntimeArguments) -> io::Result<()> {
require_lifetime_held()?;
let namespaces = enter_generation_filesystem_namespaces()?;
if arguments.network.is_none() {
enter_generation_network_namespace()?;
}
let control = std::io::stdout();
let parent = current_process_identity()?;
let mut state_anchor = prepare_state_anchor(arguments, parent, namespaces, &control)?;
let mut state_anchor_absence = None;
let result = run_supervisor_with_state(
arguments,
parent,
&control,
namespaces,
&mut state_anchor,
&mut state_anchor_absence,
);
let cleanup = state_anchor
.take()
.map(RuntimeStateAnchorMount::unmount)
.map(|result| result.map(drop));
match (result, cleanup) {
(Err(error), Some(Err(cleanup))) => Err(io::Error::new(
error.kind(),
format!("{error}; exact State Anchor cleanup failed: {cleanup}"),
)),
(Err(error), _) => Err(error),
(Ok(()), Some(Err(error))) => Err(io::Error::other(error)),
(Ok(()), _) => Ok(()),
}
}
fn prepare_state_anchor(
arguments: &RuntimeArguments,
parent: lazaret_core::identity::PidfsProcessIdentityV1,
namespaces: (
lazaret_core::identity::NamespaceIdentity,
lazaret_core::identity::NamespaceIdentity,
),
control: &std::io::Stdout,
) -> io::Result<Option<RuntimeStateAnchorMount>> {
let Some(state) = &arguments.state_anchor else {
return Ok(None);
};
let command: CoordinatorMessage = receive_message(control)?;
if command != CoordinatorMessage::MountStateAnchor {
return Err(io::Error::other(
"Runtime Supervisor received no State Anchor mount command",
));
}
let authority = RuntimeStateAnchorAuthority::from_inherited(RuntimeStateAnchorInherited {
lower_fd: state.lower_fd,
lower_label: &state.lower_label,
persistent_root_fd: state.persistent_root_fd,
persistent_anchor_fd: state.persistent_anchor_fd,
persistent_anchor_label: &state.persistent_anchor_label,
runtime_root_fd: state.runtime_root_fd,
runtime_anchor_fd: state.runtime_anchor_fd,
runtime_anchor_label: &state.runtime_anchor_label,
expected_lower: state.expected_lower,
})
.map_err(io::Error::other)?;
let ownership = ProcessOwnershipContext {
slot: HostProcessOwnershipSlot::StateAnchorDaemon,
..arguments.ownership
};
let mount = authority
.acquire(
parent,
ownership,
|staged| send_message(control, &SupervisorEvent::StateAnchorStaged(staged)),
|staged| match receive_message(control)? {
CoordinatorMessage::StateAnchorStagedAck(acknowledgment)
if acknowledgment.acknowledges(&staged) =>
{
Ok(acknowledgment)
}
_ => Err(io::Error::other(
"Runtime Supervisor received a mismatched State Anchor acknowledgment",
)),
},
|readiness| {
send_message(
control,
&SupervisorEvent::StateAnchorReady {
staged: readiness.staged(),
user_namespace: namespaces.0,
mount_namespace: namespaces.1,
mount: readiness.mount(),
},
)
},
)
.map_err(io::Error::other)?;
mount
.require_persistent_destinations(&state.persistent_destinations)
.map_err(io::Error::other)?;
Ok(Some(mount))
}
#[allow(clippy::too_many_lines)]
fn run_supervisor_with_state(
arguments: &RuntimeArguments,
parent: lazaret_core::identity::PidfsProcessIdentityV1,
control: &std::io::Stdout,
namespaces: (NamespaceIdentity, NamespaceIdentity),
state_anchor: &mut Option<RuntimeStateAnchorMount>,
state_anchor_absence: &mut Option<RuntimeStateAnchorAbsenceAuthority>,
) -> io::Result<()> {
let git_routing = match &arguments.git_routing {
Some(routing) => Some(prepare_git_routing(arguments, routing, parent, control)?),
None => None,
};
if arguments.state_anchor.is_some() {
let message: CoordinatorMessage = receive_message(control)?;
if message != CoordinatorMessage::StartSidecar {
return Err(io::Error::other(
"Runtime Supervisor received no sidecar start command",
));
}
send_message(control, &SupervisorEvent::SidecarStarted)?;
}
let network = match &arguments.network {
Some(network) => Some(prepare_runtime_network(
arguments, network, parent, control,
)?),
None => None,
};
if let Some(routing) = &git_routing {
let observed = inherited_directory_at(&routing.mountpoint).and_then(mount_identity);
match observed {
Ok(observed) if observed == routing.mount => {}
Ok(_) => {
return fail_runtime_components(
git_routing,
network,
io::Error::other("Git-routing mount changed before runsc launch"),
);
}
Err(error) => return fail_runtime_components(git_routing, network, error),
}
}
let pid_file = File::from(memfd_create("lazaret-runsc-pid", MemfdFlags::empty())?);
fcntl_setfd(pid_file.as_fd(), FdFlags::empty())?;
let runsc = runsc_target(arguments, pid_file.as_raw_fd())?;
let delete = runsc_delete_target(arguments)?;
let staged = std::cell::Cell::new(None);
let process = match child_broker().acquire_with_stdio(
parent,
arguments.ownership,
&runsc,
attached_runsc_standard_io(&runsc)?,
|ownership| {
staged.set(Some(ownership));
send_message(control, &SupervisorEvent::Staged(ownership))
},
|ownership| match receive_message(control)? {
CoordinatorMessage::StagedAck(acknowledgment)
if acknowledgment.acknowledges(&ownership) =>
{
Ok(acknowledgment)
}
_ => Err(io::Error::other(
"Runtime Supervisor received a mismatched staged acknowledgment",
)),
},
) {
Ok(process) => process,
Err(error) => return fail_runtime_components(git_routing, network, error),
};
if let Err(error) = fcntl_setfd(pid_file.as_fd(), FdFlags::CLOEXEC) {
return fail_with_runtime_cleanup(process, git_routing, network, error.into());
}
let Some(staged) = staged.get() else {
return fail_with_runtime_cleanup(
process,
git_routing,
network,
io::Error::other("attached runsc ownership was not staged"),
);
};
let sentry_pid_inventory = match wait_for_runsc_ready(&pid_file, &process) {
Ok(pid) => pid,
Err(error) => return fail_with_runtime_cleanup(process, git_routing, network, error),
};
if let Err(error) = capability::irreversibly_drop(CapabilitySet::NET_ADMIN) {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
if let Err(error) = capability::require_regain_refused(CapabilitySet::NET_ADMIN) {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
if let Err(error) = send_message(
control,
&SupervisorEvent::Ready {
staged,
sentry_pid_inventory,
},
) {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
let committed: CoordinatorMessage = match receive_message(control) {
Ok(committed) => committed,
Err(error) => return fail_with_runtime_cleanup(process, git_routing, network, error),
};
if committed != CoordinatorMessage::SandboxCommitted {
return fail_with_runtime_cleanup(
process,
git_routing,
network,
io::Error::other("sandbox commit acknowledgment was not received"),
);
}
supervise(
control,
process,
git_routing,
network,
namespaces,
state_anchor,
state_anchor_absence,
&delete,
)
}
struct RuntimeGitRouting {
process: RetainedProcess,
shutdown: File,
sentry: GitSentry,
mountpoint: PathBuf,
mount: MountIdentity,
}
#[allow(clippy::too_many_lines)]
fn prepare_git_routing(
arguments: &RuntimeArguments,
routing: &RuntimeGitRoutingArguments,
parent: lazaret_core::identity::PidfsProcessIdentityV1,
control: &std::io::Stdout,
) -> io::Result<RuntimeGitRouting> {
let message: CoordinatorMessage = receive_message(control)?;
if message != CoordinatorMessage::StartGitRouting {
return Err(io::Error::other(
"Runtime Supervisor received no Git-routing start command",
));
}
let workspace = inherited_directory(routing.workspace_fd)
.map_err(|error| git_routing_error("opening workspace authority", &error))?;
let isolated = inherited_directory(routing.isolated_fd)
.map_err(|error| git_routing_error("opening isolated-state authority", &error))?;
let mountpoint_authority = inherited_directory(routing.mountpoint_fd)
.map_err(|error| git_routing_error("opening mountpoint authority", &error))?;
let underlying_mount = mount_identity(&mountpoint_authority)
.map_err(|error| git_routing_error("reading mountpoint identity", &error))?;
for descriptor in [&workspace, &isolated] {
fcntl_setfd(descriptor, FdFlags::empty()).map_err(|error| {
git_routing_error(
"making router authority inheritable",
&io::Error::from(error),
)
})?;
}
let (shutdown_reader, shutdown_writer) = pipe_with(PipeFlags::CLOEXEC).map_err(|error| {
git_routing_error("creating router shutdown channel", &io::Error::from(error))
})?;
let threads = std::thread::available_parallelism()
.map_or(1, std::num::NonZero::get)
.min(8);
let target = TargetCommand::new("/proc/self/exe")
.argument(PRIVATE_ROUTER_ROLE)
.argument("--workspace-fd")
.argument(workspace.as_raw_fd().to_string())
.argument("--workspace-label")
.argument(&routing.workspace_label)
.argument("--isolated-fd")
.argument(isolated.as_raw_fd().to_string())
.argument("--mountpoint")
.argument(&routing.mountpoint_label)
.argument("--global-excludes")
.argument(
routing
.global_excludes
.as_deref()
.unwrap_or_else(|| std::path::Path::new("")),
)
.argument("--isolate-ignored")
.argument(if routing.isolate_ignored {
"true"
} else {
"false"
})
.argument("--threads")
.argument(threads.to_string())
.with_target_stdin(File::from(shutdown_reader))
.map_err(|error| git_routing_error("attaching router shutdown channel", &error))?;
let ownership = ProcessOwnershipContext {
slot: HostProcessOwnershipSlot::GitRoutingDaemon,
..arguments.ownership
};
let acquired = child_broker()
.acquire_captured(
parent,
ownership,
&target,
|staged| send_message(control, &SupervisorEvent::GitRoutingStaged(staged)),
|staged| match receive_message(control)? {
CoordinatorMessage::GitRoutingStagedAck(acknowledgment)
if acknowledgment.acknowledges(&staged) =>
{
Ok(acknowledgment)
}
_ => Err(io::Error::other(
"Runtime Supervisor received a mismatched Git-routing acknowledgment",
)),
},
)
.map_err(|error| git_routing_error("acquiring router process", &error));
let mut restore_error = None;
for descriptor in [&workspace, &isolated] {
if let Err(error) = fcntl_setfd(descriptor, FdFlags::CLOEXEC)
&& restore_error.is_none()
{
let error = io::Error::from(error);
restore_error = Some(git_routing_error(
"restoring router authority close-on-exec",
&error,
));
}
}
let mut captured = match (acquired, restore_error) {
(Ok(captured), None) => captured,
(Ok(captured), Some(error)) => {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
(Err(error), None) => return Err(error),
(Err(error), Some(restore)) => {
return Err(io::Error::new(
error.kind(),
format!("{error}; restoring Git-routing descriptor flags failed: {restore}"),
));
}
};
if let Err(error) = wait_for_router_ready(&mut captured)
.map_err(|error| git_routing_error("waiting for router readiness", &error))
{
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
let mounted_workspace = match inherited_directory_at(&routing.mountpoint_label)
.map_err(|error| git_routing_error("opening routed workspace", &error))
{
Ok(mounted_workspace) => mounted_workspace,
Err(error) => {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
};
let mount = match mount_identity(&mounted_workspace)
.map_err(|error| git_routing_error("reading routed mount identity", &error))
{
Ok(mount) => mount,
Err(error) => {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
};
if mount == underlying_mount {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
io::Error::other("Git router reported ready without its exact mount"),
);
}
let git_common = match reopen_exact_directory(routing.git_common_fd, &routing.git_common_label)
.map_err(|error| git_routing_error("opening Git common-directory authority", &error))
{
Ok(git_common) => git_common,
Err(error) => {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
};
let transient_hooks =
match reopen_exact_directory(routing.transient_hooks_fd, &routing.transient_hooks_label)
.map_err(|error| git_routing_error("opening transient hooks authority", &error))
{
Ok(transient_hooks) => transient_hooks,
Err(error) => {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
};
let transient_config = match reopen_exact_regular_file(
routing.transient_config_fd,
&routing.transient_config_label,
)
.map_err(|error| git_routing_error("opening transient config authority", &error))
{
Ok(transient_config) => transient_config,
Err(error) => {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
};
let sentry = match GitSentry::install_from_authorities(
mounted_workspace,
git_common,
transient_hooks,
routing.transient_hooks_label.clone(),
transient_config,
routing.transient_config_label.clone(),
)
.map_err(|error| git_routing_error("installing Git Sentry", &error))
{
Ok(sentry) => sentry,
Err(error) => {
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
};
let staged = StagedProcessOwnership::v1(
arguments.ownership.domain_id,
arguments.ownership.generation_id,
arguments.ownership.coordinator_id,
HostProcessOwnershipSlot::GitRoutingDaemon,
captured.process().identity(),
);
if let Err(error) = send_message(control, &SupervisorEvent::GitRoutingReady { staged, mount }) {
let _ = sentry.unmount();
return fail_router_before_commit(
captured.into_process(),
File::from(shutdown_writer),
error,
);
}
Ok(RuntimeGitRouting {
process: captured.into_process(),
shutdown: File::from(shutdown_writer),
sentry,
mountpoint: routing.mountpoint_label.clone(),
mount,
})
}
fn git_routing_error(action: &str, error: &io::Error) -> io::Error {
io::Error::new(error.kind(), format!("{action}: {error}"))
}
fn inherited_directory(descriptor: i32) -> io::Result<File> {
if descriptor < 0 {
return Err(io::Error::other(
"Runtime Supervisor inherited directory descriptor is invalid",
));
}
inherited_directory_at(PathBuf::from(format!("/proc/self/fd/{descriptor}")))
}
fn reopen_exact_directory(descriptor: i32, label: &std::path::Path) -> io::Result<File> {
reopen_exact(descriptor, label, OFlags::RDONLY | OFlags::DIRECTORY, true)
}
fn reopen_exact_regular_file(descriptor: i32, label: &std::path::Path) -> io::Result<File> {
reopen_exact(descriptor, label, OFlags::RDWR, false)
}
fn reopen_exact(
descriptor: i32,
label: &std::path::Path,
flags: OFlags,
directory: bool,
) -> io::Result<File> {
if descriptor < 0 {
return Err(io::Error::other(
"Runtime Supervisor inherited authority descriptor is invalid",
));
}
let inherited = File::open(format!("/proc/self/fd/{descriptor}"))?;
let expected = fstat(&inherited)?;
let reopened = File::from(openat2(
rustix::fs::CWD,
label,
flags | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
)?);
let observed = fstat(&reopened)?;
let expected_type = FileType::from_raw_mode(expected.st_mode);
let observed_type = FileType::from_raw_mode(observed.st_mode);
if expected.st_dev != observed.st_dev
|| expected.st_ino != observed.st_ino
|| expected_type != observed_type
|| observed_type.is_dir() != directory
{
return Err(io::Error::other(
"namespace-local authority differs from its inherited identity",
));
}
Ok(reopened)
}
fn inherited_directory_at(path: impl AsRef<std::path::Path>) -> io::Result<File> {
let file = File::open(path)?;
if !FileType::from_raw_mode(fstat(&file)?.st_mode).is_dir() {
return Err(io::Error::other(
"Runtime Supervisor inherited authority is not a directory",
));
}
Ok(file)
}
fn wait_for_router_ready(process: &mut lazaret_linux::process::CapturedProcess) -> io::Result<()> {
let deadline = Instant::now() + READINESS_TIMEOUT;
let mut output = Vec::new();
loop {
let mut buffer = [0_u8; 64];
match process.streams().0.read(&mut buffer) {
Ok(0) if process.process().has_exited()? => {
return Err(io::Error::other("Git router exited before readiness"));
}
Ok(0) => {}
Ok(length) => {
output.extend_from_slice(&buffer[..length]);
if output == b"ready\n" {
return Ok(());
}
if output.len() > 64 || !b"ready\n".starts_with(&output) {
return Err(io::Error::other("Git router readiness is malformed"));
}
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {}
Err(error) => return Err(error),
}
if process.process().has_exited()? {
return Err(io::Error::other("Git router exited before readiness"));
}
if Instant::now() >= deadline {
return Err(io::Error::other("Git router readiness timed out"));
}
std::thread::sleep(Duration::from_millis(5));
}
}
fn fail_router_before_commit<T>(
process: RetainedProcess,
shutdown: File,
error: io::Error,
) -> io::Result<T> {
drop(shutdown);
let _ = wait_and_reap(process);
Err(error)
}
fn prepare_runtime_network(
arguments: &RuntimeArguments,
network: &RuntimeNetworkArguments,
runtime_supervisor: lazaret_core::identity::PidfsProcessIdentityV1,
control: &std::io::Stdout,
) -> io::Result<PreparedNetwork> {
let policy_file = File::open(format!("/proc/self/fd/{}", network.policy_fd))?;
let staged = std::cell::Cell::new(None);
let prepared = prepare_network(
&policy_file,
network.policy_digest,
NetworkAttestationContext {
domain_id: arguments.ownership.domain_id,
generation_id: arguments.ownership.generation_id,
configuration_fingerprint: network.configuration_fingerprint,
network_sidecar_id: network.network_sidecar_id,
runtime_supervisor,
coordinator_id: arguments.ownership.coordinator_id,
},
|ownership| {
staged.set(Some(ownership));
send_message(control, &SupervisorEvent::HelperStaged(ownership))
},
|ownership| match receive_message(control)? {
CoordinatorMessage::HelperStagedAck(acknowledgment)
if acknowledgment.acknowledges(&ownership) =>
{
Ok(acknowledgment)
}
_ => Err(io::Error::other(
"Runtime Supervisor received a mismatched helper acknowledgment",
)),
},
)?;
let staged = staged
.get()
.ok_or_else(|| io::Error::other("connectivity helper ownership was not staged"))?;
if let Err(error) = send_message(control, &SupervisorEvent::HelperReady(staged)) {
return fail_network(Some(prepared), error);
}
if let Err(error) = send_message(
control,
&SupervisorEvent::NetworkAttestation(prepared.attestation().clone()),
) {
return fail_network(Some(prepared), error);
}
let message: CoordinatorMessage = match receive_message(control) {
Ok(message) => message,
Err(error) => return fail_network(Some(prepared), error),
};
let CoordinatorMessage::NetworkAttestationAccepted(request) = message else {
return fail_network(
Some(prepared),
io::Error::other("network attestation acceptance was not received"),
);
};
let probe = match probe_current_network(&request) {
Ok(probe) => probe,
Err(error) => return fail_network(Some(prepared), error),
};
if let Err(error) = send_message(control, &SupervisorEvent::NetworkProbe(probe)) {
return fail_network(Some(prepared), error);
}
let committed: CoordinatorMessage = match receive_message(control) {
Ok(message) => message,
Err(error) => return fail_network(Some(prepared), error),
};
if committed != CoordinatorMessage::NetworkReadinessCommitted {
return fail_network(
Some(prepared),
io::Error::other("durable network readiness was not acknowledged"),
);
}
Ok(prepared)
}

261
crates/lazaret-runtime-supervisor/src/supervision/tests.rs Normal file
View file

@ -0,0 +1,261 @@
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use lazaret_core::identity::{CoordinatorId, DomainId, GenerationId};
use lazaret_core::records::HostProcessOwnershipSlot;
use lazaret_linux::ipc::private_seqpacket_pair;
use lazaret_linux::process::ChildStandardIoPolicy;
use rustix::pipe::{PipeFlags, pipe_with};
use super::*;
#[test]
fn private_stream_framing_round_trips_typed_events() {
let (coordinator, supervisor) = private_seqpacket_pair().unwrap();
let expected = SupervisorEvent::Refused("refused".into());
send_message(&supervisor, &expected).unwrap();
assert_eq!(
CoordinatorChannel::new(coordinator).receive().unwrap(),
expected
);
}
#[test]
fn coordinator_channel_preserves_phased_resource_commands() {
let (coordinator, supervisor) = private_seqpacket_pair().unwrap();
let channel = CoordinatorChannel::new(coordinator);
channel.mount_state_anchor().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::MountStateAnchor
);
channel.start_git_routing().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::StartGitRouting
);
channel.start_sidecar().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::StartSidecar
);
channel.stop_sandbox().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::StopSandbox
);
channel.stop_git_routing().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::StopGitRouting
);
channel.stop_sidecar().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::StopSidecar
);
channel.unmount_state_anchor().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::UnmountStateAnchor
);
let user_namespace = NamespaceIdentity {
device: lazaret_core::identity::DeviceId::new(11),
inode: lazaret_core::identity::InodeId::new(12),
};
let mount_namespace = NamespaceIdentity {
device: lazaret_core::identity::DeviceId::new(13),
inode: lazaret_core::identity::InodeId::new(14),
};
let mount = MountIdentity {
mount_id: lazaret_core::identity::MountId::new(15),
device: lazaret_core::identity::DeviceId::new(16),
inode: lazaret_core::identity::InodeId::new(17),
};
channel
.observe_state_anchor_absence(user_namespace, mount_namespace, mount)
.unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::ObserveStateAnchorAbsence {
user_namespace,
mount_namespace,
mount,
}
);
channel.exit().unwrap();
assert_eq!(
receive_message::<CoordinatorMessage>(&supervisor).unwrap(),
CoordinatorMessage::Exit
);
}
#[test]
fn runtime_supervisor_and_runsc_keep_distinct_edge_and_stdio_policies() {
let ownership = ProcessOwnershipContext {
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(7),
coordinator_id: CoordinatorId::new([2; 32]),
slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
};
let launch = RuntimeSupervisorLaunch {
target: TargetCommand::new("/proc/self/exe").with_lifetime_pipe_parent_edge(),
ownership,
policy_file: None,
policy_digest: None,
configuration_fingerprint: None,
runsc_command: RunscCommand::new(
"runsc",
[OsString::from("run"), OsString::from("container")],
[0; 32],
),
network_sidecar_id: None,
state_anchor: None,
git_routing: None,
};
let (_coordinator_control, supervisor_control) = private_seqpacket_pair().unwrap();
let (supervisor_lifetime, _coordinator_lifetime) = pipe_with(PipeFlags::CLOEXEC).unwrap();
assert_eq!(launch.target().parent_edge(), ParentEdge::LifetimePipe);
assert_eq!(
launch
.child_standard_io(supervisor_control, supervisor_lifetime)
.unwrap()
.policy(),
ChildStandardIoPolicy::PrivateControlWithLifetimePipe
);
let runsc_arguments = RuntimeArguments {
ownership: ProcessOwnershipContext {
slot: HostProcessOwnershipSlot::SandboxAttachedRunsc,
..ownership
},
state_anchor: None,
git_routing: None,
network: None,
runsc_program: "runsc".into(),
runsc_arguments: ["-root=/runtime", "run", "container"]
.map(OsString::from)
.into(),
};
let runsc = runsc_target(&runsc_arguments, 9).unwrap();
assert_eq!(runsc.parent_edge(), ParentEdge::ParentDeathKill);
assert_eq!(
attached_runsc_standard_io(&runsc).unwrap().policy(),
ChildStandardIoPolicy::Silenced
);
let delete = runsc_delete_target(&runsc_arguments).unwrap();
assert_eq!(delete.program(), "runsc");
assert_eq!(
delete.arguments(),
["-root=/runtime", "delete", "--force", "container"]
.map(OsString::from)
.as_slice()
);
}
#[test]
fn runtime_role_parser_preserves_exact_ownership_and_runsc_arguments() {
let domain = [1_u8; 32];
let coordinator = [2_u8; 32];
let arguments = [
"--domain".into(),
hex(&domain).into(),
"--generation".into(),
"7".into(),
"--coordinator".into(),
hex(&coordinator).into(),
"--runsc".into(),
"runsc".into(),
"-root=/runtime".into(),
"run".into(),
"container".into(),
];
let parsed = parse_runtime_arguments(&arguments).unwrap();
assert_eq!(parsed.ownership.domain_id, DomainId::new(domain));
assert_eq!(parsed.ownership.generation_id, GenerationId::new(7));
assert_eq!(
parsed.ownership.coordinator_id,
CoordinatorId::new(coordinator)
);
assert_eq!(
parsed.ownership.slot,
HostProcessOwnershipSlot::SandboxAttachedRunsc
);
assert_eq!(parsed.runsc_program, OsString::from("runsc"));
assert_eq!(
parsed.runsc_arguments,
["-root=/runtime", "run", "container"].map(OsString::from)
);
}
#[test]
fn runtime_role_parser_refuses_non_runsc_target() {
let arguments = [
"--domain".into(),
hex(&[1_u8; 32]).into(),
"--generation".into(),
"7".into(),
"--coordinator".into(),
hex(&[2_u8; 32]).into(),
"--runsc".into(),
"foreign-runtime".into(),
];
assert!(parse_runtime_arguments(&arguments).is_err());
}
#[test]
fn runtime_role_parser_preserves_git_routing_authorities() {
let arguments = [
"--domain".into(),
hex(&[1_u8; 32]).into(),
"--generation".into(),
"7".into(),
"--coordinator".into(),
hex(&[2_u8; 32]).into(),
"--git-workspace-fd".into(),
"3".into(),
"--git-workspace-label".into(),
"/workspace".into(),
"--git-isolated-fd".into(),
"4".into(),
"--git-mountpoint-fd".into(),
"5".into(),
"--git-mountpoint-label".into(),
"/runtime/workspace".into(),
"--git-common-fd".into(),
"6".into(),
"--git-common-label".into(),
"/workspace/.git".into(),
"--git-hooks-fd".into(),
"7".into(),
"--git-hooks-label".into(),
"/runtime/hooks".into(),
"--git-config-fd".into(),
"8".into(),
"--git-config-label".into(),
"/runtime/config".into(),
"--git-global-excludes".into(),
"/home/developer/.config/git/ignore".into(),
"--git-isolate-ignored".into(),
"true".into(),
"--runsc".into(),
"runsc".into(),
"run".into(),
"container".into(),
];
let parsed = parse_runtime_arguments(&arguments).unwrap();
let routing = parsed.git_routing.unwrap();
assert_eq!(routing.workspace_fd, 3);
assert_eq!(routing.isolated_fd, 4);
assert_eq!(routing.mountpoint_fd, 5);
assert_eq!(routing.git_common_fd, 6);
assert_eq!(routing.transient_hooks_fd, 7);
assert_eq!(routing.transient_config_fd, 8);
assert!(routing.isolate_ignored);
let mut negative = arguments;
negative[7] = "-1".into();
assert!(parse_runtime_arguments(&negative).is_err());
}
}