lazaret/crates/lazaret-runtime-supervisor/src/supervision/handoff.rs

660 lines
24 KiB
Rust
Raw Normal View History

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
enum RuntimeDescriptorKindV1 {
Directory,
RegularFile,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
struct RuntimeDescriptorV1 {
number: i32,
device: u64,
inode: u64,
kind: RuntimeDescriptorKindV1,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
struct RuntimeStateAnchorHandoffV1 {
lower: RuntimeDescriptorV1,
lower_label: Vec<u8>,
persistent_root: RuntimeDescriptorV1,
persistent_anchor: RuntimeDescriptorV1,
persistent_anchor_label: Vec<u8>,
runtime_root: RuntimeDescriptorV1,
runtime_anchor: RuntimeDescriptorV1,
runtime_anchor_label: Vec<u8>,
expected_lower: (u64, u64),
persistent_destinations: Vec<Vec<u8>>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
struct RuntimeGitRoutingHandoffV1 {
workspace: RuntimeDescriptorV1,
workspace_label: Vec<u8>,
isolated: RuntimeDescriptorV1,
mountpoint: RuntimeDescriptorV1,
mountpoint_label: Vec<u8>,
git_common: RuntimeDescriptorV1,
git_common_label: Vec<u8>,
transient_hooks: RuntimeDescriptorV1,
transient_hooks_label: Vec<u8>,
transient_config: RuntimeDescriptorV1,
transient_config_label: Vec<u8>,
global_excludes: Option<Vec<u8>>,
isolate_ignored: bool,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
struct RuntimeNetworkHandoffV1 {
policy: RuntimeDescriptorV1,
policy_digest: DerivedPolicyDigest,
configuration_fingerprint: ConfigurationFingerprint,
network_sidecar_id: NetworkSidecarId,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
struct RuntimeSupervisorHandoffV1 {
domain_id: DomainId,
generation_id: GenerationId,
coordinator_id: CoordinatorId,
state_anchor: Option<RuntimeStateAnchorHandoffV1>,
git_routing: Option<RuntimeGitRoutingHandoffV1>,
network: Option<RuntimeNetworkHandoffV1>,
runsc_program: Vec<u8>,
runsc_arguments: Vec<Vec<u8>>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
enum RuntimeSupervisorHandoff {
V1(RuntimeSupervisorHandoffV1),
}
impl lazaret_core::records::SchemaType for RuntimeSupervisorHandoff {
const NAME: &'static str = "RuntimeSupervisorHandoff";
}
fn descriptor(
number: i32,
kind: RuntimeDescriptorKindV1,
) -> io::Result<RuntimeDescriptorV1> {
if number < 0 {
return Err(io::Error::other("Runtime Supervisor descriptor is invalid"));
}
let file = File::open(format!("/proc/self/fd/{number}"))?;
let status = fstat(&file)?;
let actual = FileType::from_raw_mode(status.st_mode);
let expected = match kind {
RuntimeDescriptorKindV1::Directory => actual.is_dir(),
RuntimeDescriptorKindV1::RegularFile => actual.is_file(),
};
if !expected {
return Err(io::Error::other(
"Runtime Supervisor descriptor has the wrong role type",
));
}
Ok(RuntimeDescriptorV1 {
number,
device: status.st_dev,
inode: status.st_ino,
kind,
})
}
fn build_runtime_handoff(launch: &RuntimeSupervisorLaunch) -> io::Result<Vec<u8>> {
let state_anchor = launch
.state_anchor
.as_ref()
.map(build_state_anchor_handoff)
.transpose()?;
let git_routing = launch
.git_routing
.as_ref()
.map(build_git_routing_handoff)
.transpose()?;
let network = build_network_handoff(launch)?;
let handoff = RuntimeSupervisorHandoff::V1(RuntimeSupervisorHandoffV1 {
domain_id: launch.ownership.domain_id,
generation_id: launch.ownership.generation_id,
coordinator_id: launch.ownership.coordinator_id,
state_anchor,
git_routing,
network,
runsc_program: launch.runsc_command.program().as_bytes().to_vec(),
runsc_arguments: launch
.runsc_command
.arguments()
.iter()
.map(|argument| argument.as_bytes().to_vec())
.collect(),
});
let encoded = encode(&handoff).map_err(io::Error::other)?;
if encoded.len() > MAX_RUNTIME_HANDOFF_BYTES {
return Err(io::Error::other(
"Runtime Supervisor handoff is oversized",
));
}
Ok(encoded)
}
fn build_state_anchor_handoff(
authority: &StateAnchorLaunchAuthority,
) -> io::Result<RuntimeStateAnchorHandoffV1> {
Ok(RuntimeStateAnchorHandoffV1 {
lower: descriptor(
authority.lower_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
lower_label: authority.lower_label().as_os_str().as_bytes().to_vec(),
persistent_root: descriptor(
authority.persistent_root_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
persistent_anchor: descriptor(
authority.persistent_anchor_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
persistent_anchor_label: authority
.persistent_anchor_label()
.as_os_str()
.as_bytes()
.to_vec(),
runtime_root: descriptor(
authority.runtime_root_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
runtime_anchor: descriptor(
authority.runtime_anchor_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
runtime_anchor_label: authority
.runtime_anchor_label()
.as_os_str()
.as_bytes()
.to_vec(),
expected_lower: authority.expected_lower(),
persistent_destinations: authority
.persistent_destinations()
.iter()
.map(|path| path.as_os_str().as_bytes().to_vec())
.collect(),
})
}
fn build_git_routing_handoff(
authority: &RuntimeGitRoutingLaunch,
) -> io::Result<RuntimeGitRoutingHandoffV1> {
Ok(RuntimeGitRoutingHandoffV1 {
workspace: descriptor(
authority.workspace.as_raw_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
workspace_label: authority.workspace_label.as_os_str().as_bytes().to_vec(),
isolated: descriptor(
authority.isolated.as_raw_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
mountpoint: descriptor(
authority.mountpoint.as_raw_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
mountpoint_label: authority.mountpoint_label.as_os_str().as_bytes().to_vec(),
git_common: descriptor(
authority.git_common.as_raw_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
git_common_label: authority.git_common_label.as_os_str().as_bytes().to_vec(),
transient_hooks: descriptor(
authority.transient_hooks.as_raw_fd(),
RuntimeDescriptorKindV1::Directory,
)?,
transient_hooks_label: authority
.transient_hooks_label
.as_os_str()
.as_bytes()
.to_vec(),
transient_config: descriptor(
authority.transient_config.as_raw_fd(),
RuntimeDescriptorKindV1::RegularFile,
)?,
transient_config_label: authority
.transient_config_label
.as_os_str()
.as_bytes()
.to_vec(),
global_excludes: authority
.global_excludes
.as_ref()
.map(|path| path.as_os_str().as_bytes().to_vec()),
isolate_ignored: authority.isolate_ignored,
})
}
fn build_network_handoff(
launch: &RuntimeSupervisorLaunch,
) -> io::Result<Option<RuntimeNetworkHandoffV1>> {
let network = launch
.policy_file
.as_ref()
.map(|policy| -> io::Result<RuntimeNetworkHandoffV1> {
Ok(RuntimeNetworkHandoffV1 {
policy: descriptor(
policy.as_raw_fd(),
RuntimeDescriptorKindV1::RegularFile,
)?,
policy_digest: launch
.policy_digest
.ok_or_else(|| io::Error::other("network policy digest is missing"))?,
configuration_fingerprint: launch.configuration_fingerprint.ok_or_else(|| {
io::Error::other("network configuration fingerprint is missing")
})?,
network_sidecar_id: launch
.network_sidecar_id
.ok_or_else(|| io::Error::other("network sidecar identity is missing"))?,
})
})
.transpose()?;
if network.is_none()
&& (launch.policy_digest.is_some()
|| launch.configuration_fingerprint.is_some()
|| launch.network_sidecar_id.is_some())
{
return Err(io::Error::other(
"Runtime Supervisor network handoff is partial",
));
}
Ok(network)
}
fn parse_runtime_handoff(arguments: &[OsString]) -> io::Result<RuntimeArguments> {
let [descriptor] = arguments else {
return Err(io::Error::other(
"Runtime Supervisor requires one handoff descriptor",
));
};
let handoff_descriptor = parse_descriptor(descriptor)?;
let mut file = File::open(format!("/proc/self/fd/{handoff_descriptor}"))?;
require_write_sealed(&file)?;
let length = usize::try_from(file.metadata()?.len())
.ok()
.filter(|length| *length > 0 && *length <= MAX_RUNTIME_HANDOFF_BYTES)
.ok_or_else(|| io::Error::other("Runtime Supervisor handoff size is invalid"))?;
let mut encoded = vec![0_u8; length];
file.read_exact(&mut encoded)?;
let RuntimeSupervisorHandoff::V1(handoff) =
decode::<RuntimeSupervisorHandoff>(&encoded).map_err(io::Error::other)?;
validate_runtime_handoff(handoff_descriptor, &handoff)?;
if handoff.runsc_program.as_slice() != b"runsc" {
return Err(io::Error::other("Runtime Supervisor target is not runsc"));
}
if !handoff
.runsc_arguments
.iter()
.any(|argument| argument.as_slice() == b"run")
|| handoff.runsc_arguments.iter().any(|argument| {
matches!(
argument.as_slice(),
b"-detach" | b"--detach" | b"-d"
)
})
{
return Err(io::Error::other(
"Runtime Supervisor requires foreground attached runsc",
));
}
Ok(RuntimeArguments {
ownership: ProcessOwnershipContext {
domain_id: handoff.domain_id,
generation_id: handoff.generation_id,
coordinator_id: handoff.coordinator_id,
slot: HostProcessOwnershipSlot::SandboxAttachedRunsc,
},
state_anchor: handoff.state_anchor.map(runtime_state_anchor_arguments),
git_routing: handoff.git_routing.map(runtime_git_routing_arguments),
network: handoff.network.map(|network| RuntimeNetworkArguments {
policy_fd: network.policy.number,
policy_digest: network.policy_digest,
configuration_fingerprint: network.configuration_fingerprint,
network_sidecar_id: network.network_sidecar_id,
}),
runsc_program: OsString::from_vec(handoff.runsc_program),
runsc_arguments: handoff
.runsc_arguments
.into_iter()
.map(OsString::from_vec)
.collect(),
})
}
fn parse_descriptor(value: &OsStr) -> io::Result<i32> {
value
.to_str()
.and_then(|value| value.parse().ok())
.filter(|descriptor| *descriptor >= 0)
.ok_or_else(|| io::Error::other("Runtime Supervisor descriptor is invalid"))
}
fn validate_runtime_handoff(
handoff_descriptor: i32,
handoff: &RuntimeSupervisorHandoffV1,
) -> io::Result<()> {
let mut numbers = std::collections::BTreeSet::from([handoff_descriptor]);
let mut descriptors = Vec::new();
if let Some(state) = &handoff.state_anchor {
descriptors.extend([
state.lower,
state.persistent_root,
state.persistent_anchor,
state.runtime_root,
state.runtime_anchor,
]);
if state.expected_lower != (state.lower.device, state.lower.inode) {
return Err(io::Error::other(
"State Anchor lower identity differs from its descriptor slot",
));
}
}
if let Some(git) = &handoff.git_routing {
descriptors.extend([
git.workspace,
git.isolated,
git.mountpoint,
git.git_common,
git.transient_hooks,
git.transient_config,
]);
}
if let Some(network) = &handoff.network {
descriptors.push(network.policy);
}
for expected in descriptors {
if !numbers.insert(expected.number) {
return Err(io::Error::other(
"Runtime Supervisor descriptor roles are duplicated",
));
}
let actual = descriptor(expected.number, expected.kind)?;
if actual != expected {
return Err(io::Error::other(
"Runtime Supervisor descriptor role identity changed",
));
}
if handoff
.network
.as_ref()
.is_some_and(|network| network.policy == expected)
{
let file = File::open(format!("/proc/self/fd/{}", expected.number))?;
require_write_sealed(file)?;
}
}
Ok(())
}
fn runtime_state_anchor_arguments(
state: RuntimeStateAnchorHandoffV1,
) -> RuntimeStateAnchorArguments {
RuntimeStateAnchorArguments {
lower_fd: state.lower.number,
lower_label: PathBuf::from(OsString::from_vec(state.lower_label)),
persistent_root_fd: state.persistent_root.number,
persistent_anchor_fd: state.persistent_anchor.number,
persistent_anchor_label: PathBuf::from(OsString::from_vec(
state.persistent_anchor_label,
)),
runtime_root_fd: state.runtime_root.number,
runtime_anchor_fd: state.runtime_anchor.number,
runtime_anchor_label: PathBuf::from(OsString::from_vec(state.runtime_anchor_label)),
expected_lower: state.expected_lower,
persistent_destinations: state
.persistent_destinations
.into_iter()
.map(|path| PathBuf::from(OsString::from_vec(path)))
.collect(),
}
}
fn runtime_git_routing_arguments(
routing: RuntimeGitRoutingHandoffV1,
) -> RuntimeGitRoutingArguments {
RuntimeGitRoutingArguments {
workspace_fd: routing.workspace.number,
workspace_label: PathBuf::from(OsString::from_vec(routing.workspace_label)),
isolated_fd: routing.isolated.number,
mountpoint_fd: routing.mountpoint.number,
mountpoint_label: PathBuf::from(OsString::from_vec(routing.mountpoint_label)),
git_common_fd: routing.git_common.number,
git_common_label: PathBuf::from(OsString::from_vec(routing.git_common_label)),
transient_hooks_fd: routing.transient_hooks.number,
transient_hooks_label: PathBuf::from(OsString::from_vec(routing.transient_hooks_label)),
transient_config_fd: routing.transient_config.number,
transient_config_label: PathBuf::from(OsString::from_vec(routing.transient_config_label)),
global_excludes: routing
.global_excludes
.map(|path| PathBuf::from(OsString::from_vec(path))),
isolate_ignored: routing.isolate_ignored,
}
}
#[cfg(test)]
mod handoff_tests {
use std::io::{Seek, SeekFrom, Write};
use std::os::unix::ffi::OsStrExt;
use rustix::fs::{MemfdFlags, memfd_create};
use sha2::{Digest, Sha256};
use super::*;
fn minimal_handoff(arguments: Vec<Vec<u8>>) -> RuntimeSupervisorHandoff {
RuntimeSupervisorHandoff::V1(RuntimeSupervisorHandoffV1 {
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(2),
coordinator_id: CoordinatorId::new([3; 32]),
state_anchor: None,
git_routing: None,
network: None,
runsc_program: b"runsc".to_vec(),
runsc_arguments: arguments,
})
}
fn sealed_arguments(bytes: &[u8]) -> (File, [OsString; 1]) {
let file = sealed_memfd("runtime-handoff-test", bytes).unwrap();
let arguments = [OsString::from(file.as_raw_fd().to_string())];
(file, arguments)
}
fn parse_record(record: &RuntimeSupervisorHandoff) -> io::Result<RuntimeArguments> {
let encoded = encode(record).unwrap();
let (_file, arguments) = sealed_arguments(&encoded);
parse_runtime_handoff(&arguments)
}
#[test]
fn v1_vector_and_non_utf8_runsc_arguments_are_stable() {
let directory = |number, identity| RuntimeDescriptorV1 {
number,
device: identity,
inode: identity + 1,
kind: RuntimeDescriptorKindV1::Directory,
};
let regular = |number, identity| RuntimeDescriptorV1 {
kind: RuntimeDescriptorKindV1::RegularFile,
..directory(number, identity)
};
let record = RuntimeSupervisorHandoff::V1(RuntimeSupervisorHandoffV1 {
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(2),
coordinator_id: CoordinatorId::new([3; 32]),
state_anchor: Some(RuntimeStateAnchorHandoffV1 {
lower: directory(10, 100),
lower_label: b"/lower".to_vec(),
persistent_root: directory(11, 110),
persistent_anchor: directory(12, 120),
persistent_anchor_label: b"/persistent/state-anchor".to_vec(),
runtime_root: directory(13, 130),
runtime_anchor: directory(14, 140),
runtime_anchor_label: b"/runtime/state-anchor".to_vec(),
expected_lower: (100, 101),
persistent_destinations: vec![b"/var/lib/tool".to_vec()],
}),
git_routing: Some(RuntimeGitRoutingHandoffV1 {
workspace: directory(15, 150),
workspace_label: b"/workspace".to_vec(),
isolated: directory(16, 160),
mountpoint: directory(17, 170),
mountpoint_label: b"/runtime/workspace".to_vec(),
git_common: directory(18, 180),
git_common_label: b"/workspace/.git".to_vec(),
transient_hooks: directory(19, 190),
transient_hooks_label: b"/runtime/hooks".to_vec(),
transient_config: regular(20, 200),
transient_config_label: b"/runtime/config".to_vec(),
global_excludes: Some(b"/home/user/.config/git/ignore".to_vec()),
isolate_ignored: true,
}),
network: Some(RuntimeNetworkHandoffV1 {
policy: regular(21, 210),
policy_digest: DerivedPolicyDigest::new([4; 32]),
configuration_fingerprint: ConfigurationFingerprint::new([5; 32]),
network_sidecar_id: NetworkSidecarId::new([6; 32]),
}),
runsc_program: b"runsc".to_vec(),
runsc_arguments: vec![
b"-root=/runtime".to_vec(),
b"run".to_vec(),
vec![b'c', 0xff],
],
});
let encoded = encode(&record).unwrap();
assert_eq!(
Sha256::digest(&encoded).as_slice(),
[
123, 0, 18, 227, 164, 11, 96, 15, 4, 201, 99, 25, 198, 122, 175, 149, 170,
165, 12, 40, 209, 64, 119, 67, 10, 109, 128, 44, 201, 103, 14, 182,
]
);
assert_eq!(
decode::<RuntimeSupervisorHandoff>(&encoded).unwrap(),
record
);
let parsed = parse_record(&minimal_handoff(vec![
b"-root=/runtime".to_vec(),
b"run".to_vec(),
vec![b'c', 0xff],
]))
.unwrap();
assert_eq!(parsed.runsc_arguments[2].as_bytes(), [b'c', 0xff]);
}
#[test]
fn unknown_trailing_truncated_unsealed_and_oversize_handoffs_refuse() {
let record = minimal_handoff(vec![b"run".to_vec(), b"container".to_vec()]);
let encoded = encode(&record).unwrap();
for malformed in [
vec![1],
{
let mut trailing = encoded.clone();
trailing.push(0);
trailing
},
encoded[..encoded.len() - 1].to_vec(),
] {
let (_file, arguments) = sealed_arguments(&malformed);
assert!(parse_runtime_handoff(&arguments).is_err());
}
let descriptor = memfd_create("unsealed-handoff", MemfdFlags::ALLOW_SEALING).unwrap();
let mut unsealed = File::from(descriptor);
unsealed.write_all(&encoded).unwrap();
unsealed.seek(SeekFrom::Start(0)).unwrap();
let arguments = [OsString::from(unsealed.as_raw_fd().to_string())];
assert!(parse_runtime_handoff(&arguments).is_err());
let oversized = vec![0_u8; MAX_RUNTIME_HANDOFF_BYTES + 1];
let (_file, arguments) = sealed_arguments(&oversized);
assert!(parse_runtime_handoff(&arguments).is_err());
}
#[test]
fn descriptor_identity_role_substitution_and_duplication_refuse() {
let first_tree = tempfile::tempdir().unwrap();
let second_tree = tempfile::tempdir().unwrap();
let first = File::open(first_tree.path()).unwrap();
let second = File::open(second_tree.path()).unwrap();
let first_slot =
descriptor(first.as_raw_fd(), RuntimeDescriptorKindV1::Directory).unwrap();
let second_slot =
descriptor(second.as_raw_fd(), RuntimeDescriptorKindV1::Directory).unwrap();
let mut role_mismatch = state_record(first_slot);
role_mismatch.lower.kind = RuntimeDescriptorKindV1::RegularFile;
assert!(parse_record(&with_state(role_mismatch)).is_err());
let mut identity_drift = state_record(first_slot);
identity_drift.lower.inode = identity_drift.lower.inode.wrapping_add(1);
identity_drift.expected_lower = (
identity_drift.lower.device,
identity_drift.lower.inode,
);
assert!(parse_record(&with_state(identity_drift)).is_err());
let mut substitution = state_record(first_slot);
substitution.lower.number = second_slot.number;
assert!(parse_record(&with_state(substitution)).is_err());
assert!(parse_record(&with_state(state_record(first_slot))).is_err());
}
#[test]
fn partial_network_group_and_nonexact_private_argv_refuse() {
let ownership = ProcessOwnershipContext {
domain_id: DomainId::new([4; 32]),
generation_id: GenerationId::new(5),
coordinator_id: CoordinatorId::new([6; 32]),
slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
};
let launch = RuntimeSupervisorLaunch {
target: TargetCommand::new("/usr/bin/false").with_lifetime_pipe_parent_edge(),
handoff_file: None,
ownership,
policy_file: None,
policy_digest: Some(DerivedPolicyDigest::new([7; 32])),
configuration_fingerprint: None,
network_sidecar_id: None,
runsc_command: RunscCommand::new(
"runsc",
[OsString::from("run"), OsString::from("container")],
[8; 32],
),
state_anchor: None,
git_routing: None,
};
assert!(build_runtime_handoff(&launch).is_err());
assert!(parse_runtime_handoff(&[]).is_err());
assert!(
parse_runtime_handoff(&[OsString::from("1"), OsString::from("2")]).is_err()
);
}
fn state_record(slot: RuntimeDescriptorV1) -> RuntimeStateAnchorHandoffV1 {
RuntimeStateAnchorHandoffV1 {
lower: slot,
lower_label: b"/lower".to_vec(),
persistent_root: slot,
persistent_anchor: slot,
persistent_anchor_label: b"/persistent".to_vec(),
runtime_root: slot,
runtime_anchor: slot,
runtime_anchor_label: b"/runtime".to_vec(),
expected_lower: (slot.device, slot.inode),
persistent_destinations: vec![b"/state".to_vec()],
}
}
fn with_state(state_anchor: RuntimeStateAnchorHandoffV1) -> RuntimeSupervisorHandoff {
let RuntimeSupervisorHandoff::V1(mut handoff) =
minimal_handoff(vec![b"run".to_vec(), b"container".to_vec()]);
handoff.state_anchor = Some(state_anchor);
RuntimeSupervisorHandoff::V1(handoff)
}
}