Replace external reachability with an owned canary

This commit is contained in:
ms2k 2026-07-19 22:39:31 +09:00
commit 31e9ab0f2d
8 changed files with 1409 additions and 1972 deletions

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,3 @@
use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs::File;
@ -14,8 +13,8 @@ pub use lazaret_sandbox_contract::{
CONTROL_ROOT_DESTINATION, CREDENTIALS_DESTINATION, DEV_DESTINATION,
ESCALATION_CLIENT_DESTINATION, HOSTNAME_DESTINATION, HOSTS_DESTINATION,
MAILBOX_REQUESTS_DESTINATION, MAILBOX_RESULTS_DESTINATION, PROC_DESTINATION,
RESOLV_CONF_DESTINATION, RUNTIME_FILE_DESTINATIONS, SANDBOX_INIT_DESTINATION, SYS_DESTINATION,
TERMINFO_ROOT_DESTINATION, WORKSPACE_DESTINATION,
READINESS_DESTINATION, RESOLV_CONF_DESTINATION, RUNTIME_FILE_DESTINATIONS,
SANDBOX_INIT_DESTINATION, SYS_DESTINATION, TERMINFO_ROOT_DESTINATION, WORKSPACE_DESTINATION,
};
use oci_spec::runtime::{
Capabilities, LinuxBuilder, LinuxCapabilitiesBuilder, LinuxNamespaceBuilder,
@ -42,7 +41,6 @@ use validation::{
pub const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
const OCI_VERSION: &str = "1.2.1";
const DEFAULT_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
const MAX_TERMINFO_BYTES: usize = 1_048_576;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
@ -323,7 +321,6 @@ impl SandboxHandoffAuthority<'_> {
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct HostTerminal {
term: String,
color_term: Option<String>,
compiled_entry: Vec<u8>,
}
@ -334,12 +331,12 @@ impl HostTerminal {
/// Invalid names, colors, terminfo headers, aliases, and oversized entries are refused.
pub fn from_compiled_entry(
term: impl Into<String>,
color_term: Option<String>,
color_term: Option<&str>,
compiled_entry: Vec<u8>,
) -> Result<Self, SandboxBuildError> {
let term = term.into();
validate_terminal_name(&term)?;
if color_term.as_deref().is_some_and(|value| {
if color_term.is_some_and(|value| {
value.is_empty() || value.bytes().any(|byte| byte < 0x20 || byte == 0x7f)
}) {
return Err(SandboxBuildError::new(
@ -350,7 +347,6 @@ impl HostTerminal {
validate_compiled_terminfo(&term, &compiled_entry)?;
Ok(Self {
term,
color_term,
compiled_entry,
})
}
@ -359,7 +355,6 @@ impl HostTerminal {
#[derive(Debug)]
pub struct StagedHostTerminal {
term: String,
color_term: Option<String>,
compiled_entry: Vec<u8>,
relative_source: PathBuf,
identity: FileIdentity,
@ -441,7 +436,6 @@ pub fn stage_host_terminal(
.join(bucket)
.join(&terminal.term),
term: terminal.term,
color_term: terminal.color_term,
compiled_entry: terminal.compiled_entry,
identity,
})

View file

@ -1,8 +1,8 @@
use super::{
CREDENTIALS_DESTINATION, DEV_DESTINATION, ESCALATION_CLIENT_DESTINATION,
MAILBOX_REQUESTS_DESTINATION, MAILBOX_RESULTS_DESTINATION, Mount, MountBuilder,
PROC_DESTINATION, Path, PathBuf, SANDBOX_INIT_DESTINATION, SYS_DESTINATION, SandboxBuildError,
oci_error, validate_absolute_normalized,
PROC_DESTINATION, Path, PathBuf, READINESS_DESTINATION, SANDBOX_INIT_DESTINATION,
SYS_DESTINATION, SandboxBuildError, oci_error, validate_absolute_normalized,
};
pub(super) fn standard_mounts() -> Result<Vec<Mount>, SandboxBuildError> {
@ -66,6 +66,7 @@ pub(super) fn standard_mounts() -> Result<Vec<Mount>, SandboxBuildError> {
pub(super) fn control_mounts(control_sources: &Path) -> Result<Vec<Mount>, SandboxBuildError> {
[
("credentials", CREDENTIALS_DESTINATION, true, false),
("readiness", READINESS_DESTINATION, false, true),
(
"mailbox/requests",
MAILBOX_REQUESTS_DESTINATION,

View file

@ -1,11 +1,10 @@
use super::{
AtFlags, BTreeMap, Capabilities, ContainerId, DEFAULT_PATH, File, FileIdentity, FileType,
HandoffPathType, LinuxBuilder, LinuxCapabilitiesBuilder, LinuxNamespaceBuilder,
LinuxNamespaceType, Mode, Mount, NETWORK_NAMESPACE_PATH, OFlags, OsString, OwnedFd, Path,
PathBuf, ProcessBuilder, RUNTIME_FILE_DESTINATIONS, Read, ResolveFlags,
ResolvedDomainConfiguration, RootProcess, RunscCommand, RuntimeHandoffSource,
SandboxBuildError, SandboxBuildErrorKind, StagedHostTerminal, StatxFlags,
TERMINFO_ROOT_DESTINATION, UserBuilder, VolumeAccess, WORKSPACE_DESTINATION, bind_mount,
AtFlags, Capabilities, ContainerId, File, FileIdentity, FileType, HandoffPathType,
LinuxBuilder, LinuxCapabilitiesBuilder, LinuxNamespaceBuilder, LinuxNamespaceType, Mode, Mount,
NETWORK_NAMESPACE_PATH, OFlags, OsString, OwnedFd, Path, PathBuf, ProcessBuilder,
RUNTIME_FILE_DESTINATIONS, Read, ResolveFlags, ResolvedDomainConfiguration, RootProcess,
RunscCommand, RuntimeHandoffSource, SandboxBuildError, SandboxBuildErrorKind,
StagedHostTerminal, StatxFlags, UserBuilder, VolumeAccess, WORKSPACE_DESTINATION, bind_mount,
control_mounts, hex_lower, host_path, oci_error, openat2, staging_error, standard_mounts,
statx, validate_absolute_normalized,
};
@ -50,6 +49,7 @@ pub(super) fn handoff_runtime_sources(
container.join("mailbox/results"),
HandoffPathType::Directory,
),
(container.join("readiness"), HandoffPathType::Directory),
(
container.join("bin/lazaret-escalate"),
HandoffPathType::RegularFile,
@ -331,36 +331,10 @@ pub(super) fn open_absolute_directory(path: &Path) -> Result<OwnedFd, SandboxBui
}
pub(super) fn build_process(
configuration: &ResolvedDomainConfiguration,
_configuration: &ResolvedDomainConfiguration,
root_process: RootProcess,
terminal: Option<&StagedHostTerminal>,
_terminal: Option<&StagedHostTerminal>,
) -> Result<oci_spec::runtime::Process, SandboxBuildError> {
let mut environment = BTreeMap::from([("PATH".to_owned(), DEFAULT_PATH.to_owned())]);
for (name, value) in configuration.ephemeral_environment() {
let value = value.expose().to_str().ok_or_else(|| {
SandboxBuildError::new(
SandboxBuildErrorKind::InvalidProcess,
"resolved environment value is not UTF-8",
)
})?;
environment.insert(name.clone(), value.to_owned());
}
if let Some(terminal) = terminal {
environment.insert("TERM".to_owned(), terminal.term.clone());
if let Some(color_term) = &terminal.color_term {
environment.insert("COLORTERM".to_owned(), color_term.clone());
} else {
environment.remove("COLORTERM");
}
environment.insert(
"TERMINFO_DIRS".to_owned(),
format!("{TERMINFO_ROOT_DESTINATION}:"),
);
}
let environment = environment
.into_iter()
.map(|(name, value)| format!("{name}={value}"))
.collect::<Vec<_>>();
let empty = Capabilities::new();
let capabilities = LinuxCapabilitiesBuilder::default()
.bounding(empty.clone())
@ -380,7 +354,7 @@ pub(super) fn build_process(
.terminal(false)
.user(user)
.args(root_process.arguments)
.env(environment)
.env(Vec::<String>::new())
.cwd(PathBuf::from(WORKSPACE_DESTINATION))
.capabilities(capabilities)
.no_new_privileges(true)

View file

@ -32,7 +32,7 @@ use rustix::io::{FdFlags, fcntl_setfd, read, write};
use rustix::process::Signal;
use serde::{Deserialize, Serialize};
use crate::network::{NetworkProbeRequest, PreparedNetwork};
use crate::network::PreparedNetwork;
use crate::sandbox::RunscCommand;
use handoff::{build_runtime_handoff, parse_runtime_handoff};
@ -47,7 +47,6 @@ const TERMINATION_GRACE: Duration = Duration::from_secs(2);
pub const RUNTIME_SUPERVISOR_SERIALIZED_RECORD_TYPE_NAMES: &[&str] = &[
"RuntimeSupervisorHandoff",
"NetworkProbeRequest",
"AttachedRunscSignal",
"SupervisorEvent",
"CoordinatorMessage",
@ -92,7 +91,7 @@ pub enum SupervisorEvent {
HelperStaged(StagedProcessOwnership),
HelperReady(StagedProcessOwnership),
NetworkAttestation(NetworkReadinessAttestation),
NetworkProbe(lazaret_core::records::NetworkReachabilityProbe),
NetworkCanary(lazaret_core::records::NetworkCanaryObservation),
HelperExited,
Staged(StagedProcessOwnership),
Ready {
@ -132,7 +131,7 @@ enum CoordinatorMessage {
GitRoutingStagedAck(ProcessOwnershipCommitAck),
StartSidecar,
HelperStagedAck(ProcessOwnershipCommitAck),
NetworkAttestationAccepted(NetworkProbeRequest),
NetworkAttestationAccepted,
NetworkReadinessCommitted,
StagedAck(ProcessOwnershipCommitAck),
SandboxCommitted,
@ -232,10 +231,10 @@ impl CoordinatorChannel {
/// # Errors
/// Returns an error when accepted attestation cannot release sandbox acquisition.
pub fn acknowledge_network_attestation(&self, request: NetworkProbeRequest) -> io::Result<()> {
pub fn acknowledge_network_attestation(&self) -> io::Result<()> {
send_message(
&self.descriptor,
&CoordinatorMessage::NetworkAttestationAccepted(request),
&CoordinatorMessage::NetworkAttestationAccepted,
)
}

View file

@ -284,7 +284,7 @@ fn handle_supervisor_message(
| CoordinatorMessage::StartGitRouting
| CoordinatorMessage::GitRoutingStagedAck(_)
| CoordinatorMessage::StartSidecar
| CoordinatorMessage::NetworkAttestationAccepted(_)
| CoordinatorMessage::NetworkAttestationAccepted
| CoordinatorMessage::NetworkReadinessCommitted
| CoordinatorMessage::StagedAck(_)
| CoordinatorMessage::SandboxCommitted => {

View file

@ -23,7 +23,7 @@ use rustix::thread::CapabilitySet;
use crate::network::{
NetworkAttestationContext, PreparedNetwork, enter_generation_filesystem_namespaces,
enter_generation_network_namespace, prepare_network, probe_current_network,
enter_generation_network_namespace, prepare_network,
};
use super::runtime::{
@ -159,7 +159,7 @@ fn run_supervisor_with_state(
}
send_message(control, &SupervisorEvent::SidecarStarted)?;
}
let network = match &arguments.network {
let mut network = match &arguments.network {
Some(network) => Some(prepare_runtime_network(
arguments, network, parent, control,
)?),
@ -182,6 +182,14 @@ fn run_supervisor_with_state(
let pid_file = File::from(memfd_create("lazaret-runsc-pid", MemfdFlags::empty())?);
fcntl_setfd(pid_file.as_fd(), FdFlags::empty())?;
let runsc = runsc_target(arguments, pid_file.as_raw_fd())?;
let readiness_control = readiness_control(arguments)?;
let sandbox_control = network
.as_ref()
.map(|prepared| prepared.prepare_sandbox_control(&readiness_control))
.transpose();
if let Err(error) = sandbox_control {
return fail_runtime_components(git_routing, network, error);
}
let delete = runsc_delete_target(arguments)?;
let staged = std::cell::Cell::new(None);
let process = match child_broker().acquire_with_stdio(
@ -222,12 +230,66 @@ fn run_supervisor_with_state(
Ok(pid) => pid,
Err(error) => return fail_with_runtime_cleanup(process, git_routing, network, error),
};
let mut canary_observation = if let Some(prepared) = network.as_mut() {
let initial = match prepared.sandbox_initial_observation(&readiness_control) {
Ok(initial) => initial,
Err(error) => {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
};
let mut observation = match prepared.finish_canary(
[initial[0], initial[1], initial[4], initial[5], initial[6]],
false,
) {
Ok(observation) => observation,
Err(error) => {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
};
observation.allowed_endpoint_authenticated &= initial[2];
observation.denied_endpoint_unreachable &= initial[3];
let post = match prepared.sandbox_post_cleanup_observation(&readiness_control) {
Ok(post) => post,
Err(error) => {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
};
observation.allowed_endpoint_unreachable_after_cleanup = post[0];
observation.tap_present_after_cleanup = post[1];
observation.ipv6_unchanged_after_cleanup = post[2];
Some(observation)
} else {
None
};
if let Err(error) = capability::irreversibly_drop(CapabilitySet::NET_ADMIN) {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
if let Err(error) = capability::require_regain_refused(CapabilitySet::NET_ADMIN) {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
if let Some(observation) = canary_observation.as_mut() {
observation.network_admin_dropped = true;
if let Err(error) = send_message(
control,
&SupervisorEvent::NetworkCanary(observation.clone()),
) {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
let committed: CoordinatorMessage = match receive_message(control) {
Ok(message) => message,
Err(error) => {
return fail_with_runtime_cleanup(process, git_routing, network, error);
}
};
if committed != CoordinatorMessage::NetworkReadinessCommitted {
return fail_with_runtime_cleanup(
process,
git_routing,
network,
io::Error::other("durable network readiness was not acknowledged"),
);
}
}
if let Err(error) = send_message(
control,
&SupervisorEvent::Ready {
@ -261,6 +323,30 @@ fn run_supervisor_with_state(
)
}
fn readiness_control(arguments: &RuntimeArguments) -> io::Result<PathBuf> {
let bundle = arguments
.runsc_arguments
.iter()
.find_map(|argument| {
argument
.to_str()
.and_then(|argument| argument.strip_prefix("-bundle="))
})
.map(PathBuf::from)
.ok_or_else(|| io::Error::other("attached runsc command has no bundle path"))?;
let container = bundle
.file_name()
.ok_or_else(|| io::Error::other("attached runsc bundle has no container identity"))?;
let runtime = bundle
.parent()
.and_then(|path| path.parent())
.ok_or_else(|| io::Error::other("attached runsc bundle has no runtime root"))?;
Ok(runtime
.join("sandbox-control")
.join(container)
.join("readiness"))
}
pub(super) struct RuntimeGitRouting {
pub(super) process: RetainedProcess,
pub(super) shutdown: File,
@ -663,28 +749,11 @@ fn prepare_runtime_network(
Ok(message) => message,
Err(error) => return fail_network(Some(prepared), error),
};
let CoordinatorMessage::NetworkAttestationAccepted(request) = message else {
if message != CoordinatorMessage::NetworkAttestationAccepted {
return fail_network(
Some(prepared),
io::Error::other("network attestation acceptance was not received"),
);
};
let probe = match probe_current_network(&request) {
Ok(probe) => probe,
Err(error) => return fail_network(Some(prepared), error),
};
if let Err(error) = send_message(control, &SupervisorEvent::NetworkProbe(probe)) {
return fail_network(Some(prepared), error);
}
let committed: CoordinatorMessage = match receive_message(control) {
Ok(message) => message,
Err(error) => return fail_network(Some(prepared), error),
};
if committed != CoordinatorMessage::NetworkReadinessCommitted {
return fail_network(
Some(prepared),
io::Error::other("durable network readiness was not acknowledged"),
);
}
Ok(prepared)
}

View file

@ -6,7 +6,7 @@ use lazaret_core::identity::{
PidfsProcessIdentityV1, ProcessStartTimeTicks,
};
use lazaret_core::records::{
HostProcessOwnershipSlot, NetworkObservation, NetworkReachabilityProbe,
HostProcessOwnershipSlot, NetworkCanaryObservation, NetworkObservation,
NetworkReadinessAttestation, NetworkReadinessAttestationFields, PolicyDelivery,
ProcessOwnershipCommitAck, SchemaType, StagedProcessOwnership,
};
@ -15,8 +15,6 @@ use lazaret_linux::process::ChildStandardIoPolicy;
use rustix::pipe::{PipeFlags, pipe_with};
use sha2::{Digest, Sha256};
use crate::network::NetworkProbeRequest;
use super::runtime::{attached_runsc_standard_io, runsc_delete_target, runsc_target};
use super::*;
@ -52,7 +50,7 @@ fn event_tag(event: &SupervisorEvent) -> &'static str {
SupervisorEvent::HelperStaged(_) => "HelperStaged",
SupervisorEvent::HelperReady(_) => "HelperReady",
SupervisorEvent::NetworkAttestation(_) => "NetworkAttestation",
SupervisorEvent::NetworkProbe(_) => "NetworkProbe",
SupervisorEvent::NetworkCanary(_) => "NetworkCanary",
SupervisorEvent::HelperExited => "HelperExited",
SupervisorEvent::Staged(_) => "Staged",
SupervisorEvent::Ready { .. } => "Ready",
@ -77,7 +75,7 @@ fn message_tag(message: &CoordinatorMessage) -> &'static str {
CoordinatorMessage::GitRoutingStagedAck(_) => "GitRoutingStagedAck",
CoordinatorMessage::StartSidecar => "StartSidecar",
CoordinatorMessage::HelperStagedAck(_) => "HelperStagedAck",
CoordinatorMessage::NetworkAttestationAccepted(_) => "NetworkAttestationAccepted",
CoordinatorMessage::NetworkAttestationAccepted => "NetworkAttestationAccepted",
CoordinatorMessage::NetworkReadinessCommitted => "NetworkReadinessCommitted",
CoordinatorMessage::StagedAck(_) => "StagedAck",
CoordinatorMessage::SandboxCommitted => "SandboxCommitted",
@ -97,7 +95,7 @@ fn representative_events(
mount: MountIdentity,
staged: StagedProcessOwnership,
attestation: NetworkReadinessAttestation,
probe: NetworkReachabilityProbe,
canary: NetworkCanaryObservation,
) -> Vec<SupervisorEvent> {
vec![
SupervisorEvent::StateAnchorStaged(staged),
@ -113,7 +111,7 @@ fn representative_events(
SupervisorEvent::HelperStaged(staged),
SupervisorEvent::HelperReady(staged),
SupervisorEvent::NetworkAttestation(attestation),
SupervisorEvent::NetworkProbe(probe),
SupervisorEvent::NetworkCanary(canary),
SupervisorEvent::HelperExited,
SupervisorEvent::Staged(staged),
SupervisorEvent::Ready {
@ -142,7 +140,6 @@ fn representative_messages(
namespace: NamespaceIdentity,
mount: MountIdentity,
acknowledgment: ProcessOwnershipCommitAck,
request: NetworkProbeRequest,
) -> Vec<CoordinatorMessage> {
vec![
CoordinatorMessage::MountStateAnchor,
@ -151,7 +148,7 @@ fn representative_messages(
CoordinatorMessage::GitRoutingStagedAck(acknowledgment),
CoordinatorMessage::StartSidecar,
CoordinatorMessage::HelperStagedAck(acknowledgment),
CoordinatorMessage::NetworkAttestationAccepted(request),
CoordinatorMessage::NetworkAttestationAccepted,
CoordinatorMessage::NetworkReadinessCommitted,
CoordinatorMessage::StagedAck(acknowledgment),
CoordinatorMessage::SandboxCommitted,
@ -182,7 +179,7 @@ fn assert_event_variant_coverage(events: &[SupervisorEvent]) {
"HelperStaged",
"HelperReady",
"NetworkAttestation",
"NetworkProbe",
"NetworkCanary",
"HelperExited",
"Staged",
"Ready",
@ -265,34 +262,39 @@ fn private_control_record_vectors_are_stable() {
routes_attested: true,
},
});
let probe = NetworkReachabilityProbe {
let canary = NetworkCanaryObservation {
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(2),
enforcement_namespace: namespace,
denied_destinations: Vec::new(),
permitted_destinations: Vec::new(),
dns_observations: Vec::new(),
};
let request = NetworkProbeRequest {
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(2),
enforcement_namespace: namespace,
denied: Vec::new(),
permitted: Vec::new(),
dns: Vec::new(),
linux_policy_matches: true,
endpoints_authenticated_before_policy: true,
allowed_endpoint_authenticated: true,
denied_endpoint_unreachable: true,
imported_interface_matches: true,
tap_present: true,
ipv6_absent: true,
network_capabilities_empty: true,
route_mutation_refused: true,
cleanup_shape_matched: true,
responder_closed: true,
peer_namespace_closed: true,
host_interface_deleted: true,
allowed_endpoint_unreachable_after_cleanup: true,
tap_present_after_cleanup: true,
ipv6_unchanged_after_cleanup: true,
network_admin_dropped: true,
};
let signals = [
AttachedRunscSignal::Stop,
AttachedRunscSignal::Continue,
AttachedRunscSignal::Kill,
];
let events = representative_events(namespace, mount, staged, attestation, probe);
let messages = representative_messages(namespace, mount, acknowledgment, request);
let events = representative_events(namespace, mount, staged, attestation, canary);
let messages = representative_messages(namespace, mount, acknowledgment);
assert_eq!(
RUNTIME_SUPERVISOR_SERIALIZED_RECORD_TYPE_NAMES,
&[
"RuntimeSupervisorHandoff",
<NetworkProbeRequest as SchemaType>::NAME,
<AttachedRunscSignal as SchemaType>::NAME,
<SupervisorEvent as SchemaType>::NAME,
<CoordinatorMessage as SchemaType>::NAME,
@ -312,12 +314,12 @@ fn private_control_record_vectors_are_stable() {
251, 52, 93, 195, 13, 90, 91, 33, 97, 9, 204, 249, 127, 100,
],
[
46, 250, 76, 205, 43, 212, 214, 67, 162, 9, 72, 170, 22, 44, 123, 161, 14, 206, 29,
244, 137, 251, 60, 194, 96, 96, 92, 216, 50, 97, 244, 47,
108, 86, 36, 107, 140, 141, 247, 255, 129, 78, 49, 148, 70, 146, 231, 189, 214,
105, 240, 74, 231, 71, 82, 173, 54, 64, 80, 156, 120, 94, 74, 216,
],
[
227, 133, 56, 78, 199, 235, 191, 196, 191, 7, 154, 135, 223, 104, 221, 62, 240,
253, 202, 161, 22, 254, 133, 59, 22, 126, 218, 254, 81, 114, 240, 120,
82, 32, 31, 71, 234, 160, 96, 131, 26, 216, 188, 198, 118, 241, 206, 220, 149, 73,
248, 93, 204, 88, 151, 169, 96, 186, 112, 242, 119, 15, 250, 56,
],
]
);