use std::collections::BTreeMap; use std::fs::{self, File, OpenOptions}; use std::net::{Ipv4Addr, SocketAddrV4, TcpListener, TcpStream, ToSocketAddrs}; use std::os::linux::net::SocketAddrExt; use std::os::unix::ffi::OsStringExt; use std::os::unix::fs::symlink; use std::os::unix::net::{SocketAddr, UnixListener, UnixStream}; use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::thread; use std::time::Duration; use lazaret_core::configuration::{ ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration, }; use lazaret_core::identity::{ BundleDigest, ContainerId, CoordinatorId, GenerationId, HostPath, NetworkSidecarId, SandboxId, }; use lazaret_core::network::{DnsQueryExpectation, ReachabilityExpectation, derive_ipv4_policy}; use lazaret_core::records::{ Domain, DurableDecision, Generation, HostProcessOwnershipSlot, IpAddress, ParticipantRole, ParticipantState, ProcessOwnershipCommitAck, }; use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore}; use lazaret_host_coordinator::network::NetworkCoordinator; use lazaret_host_coordinator::runtime::{ AwaitingAttachedRunsc, CommittedSandboxRuntime, SandboxCoordinator, }; use lazaret_linux::filesystem::try_exclusive_flock; use lazaret_linux::platform::{PlatformObservation, check_platform}; use lazaret_linux::process::{ ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, current_process_identity, }; use lazaret_runtime_supervisor::network::{DnsProbeTarget, NetworkProbeRequest, TcpProbeTarget}; use lazaret_runtime_supervisor::sandbox::{ NETWORK_NAMESPACE_PATH, RootProcess, RunscCommand, SandboxBuildRequest, SandboxBundle, build_sandbox_bundle, production_sandbox_layout, }; use lazaret_runtime_supervisor::supervision::{ AttachedRunscSignal, RuntimeNetworkLaunch, RuntimeSupervisorLaunch, dispatch_private_runtime_role, }; use rustix::process::Signal; use tempfile::TempDir; const SKIP: u8 = 77; const HOST_COORDINATOR_DRIVER: &str = "--runtime-composition-host-coordinator"; const SANDBOX_NETWORK_PROBE: &str = "--sandbox-network-probe"; const NETWORK_MATRIX: &str = "--network-matrix"; const HANDOFF_MATRIX: &str = "--handoff-matrix"; const BENCHMARK_DRIVER: &str = "--runtime-composition-benchmark-command"; const BENCHMARK_GUEST: &str = "--runtime-composition-benchmark-guest"; const BENCHMARK_COMPLETE: &str = "benchmark-complete"; const BENCHMARK_ERROR: &str = "benchmark-error"; fn main() -> ExitCode { let arguments = std::env::args_os().skip(1).collect::>(); if let Some(result) = lazaret_linux::process::dispatch_private_process_role(&arguments) { return exit(result); } if let Some(result) = dispatch_private_runtime_role(&arguments) { return exit(result); } if arguments.first().and_then(|argument| argument.to_str()) == Some(BENCHMARK_GUEST) { return exit(run_benchmark_guest(&arguments[1..])); } if arguments.first().and_then(|argument| argument.to_str()) == Some(BENCHMARK_DRIVER) { return exit(run_benchmark_driver(&arguments[1..])); } if arguments.first().and_then(|argument| argument.to_str()) == Some(HOST_COORDINATOR_DRIVER) { return exit(run_host_coordinator_driver(&arguments[1..])); } if arguments.first().and_then(|argument| argument.to_str()) == Some(SANDBOX_NETWORK_PROBE) { return exit(run_sandbox_network_probe(&arguments[1..])); } if arguments.first().and_then(|argument| argument.to_str()) == Some(NETWORK_MATRIX) { return match run_network_matrix() { Ok(()) => { println!("PASS production network composition matrix"); ExitCode::SUCCESS } Err(MatrixError::Skip(reason)) => { println!("SKIP network composition: {reason}"); ExitCode::from(SKIP) } Err(MatrixError::Failure(error)) => { eprintln!("network composition failed: {error}"); ExitCode::FAILURE } }; } if arguments.first().and_then(|argument| argument.to_str()) == Some(HANDOFF_MATRIX) { return match run_handoff_matrix() { Ok(()) => { println!("PASS final handoff replacement refusal"); ExitCode::SUCCESS } Err(MatrixError::Skip(reason)) => { println!("SKIP final handoff replacement refusal: {reason}"); ExitCode::from(SKIP) } Err(MatrixError::Failure(error)) => { eprintln!("final handoff replacement refusal failed: {error}"); ExitCode::FAILURE } }; } match run_matrix() { Ok(()) => { println!("PASS foreground runtime composition and causal-loss matrix"); ExitCode::SUCCESS } Err(MatrixError::Skip(reason)) => { println!("SKIP foreground runtime composition: {reason}"); ExitCode::from(SKIP) } Err(MatrixError::Failure(error)) => { eprintln!("runtime composition failed: {error}"); ExitCode::FAILURE } } } fn run_benchmark_driver(arguments: &[std::ffi::OsString]) -> std::io::Result<()> { let separator = arguments .iter() .position(|argument| argument == "--") .ok_or_else(|| std::io::Error::other("benchmark command separator is missing"))?; if separator != 8 || arguments.len() == separator + 1 { return Err(std::io::Error::other( "benchmark composition arguments are malformed", )); } let paths = arguments[..separator] .iter() .map(PathBuf::from) .collect::>(); let fixture = Fixture::build_benchmark( &paths[0], &paths[1], &paths[2], &paths[3], &paths[4], &paths[5], &paths[6], &paths[7], &arguments[separator + 1..], ) .map_err(matrix_io)?; let coordinator = SandboxCoordinator::new(&fixture.store); let runtime = fixture.launch_committed().map_err(matrix_io)?; let results = fixture .benchmark_results .as_ref() .ok_or_else(|| std::io::Error::other("benchmark composition result authority is absent"))?; let complete = results.join(BENCHMARK_COMPLETE); let error = results.join(BENCHMARK_ERROR); for _ in 0..24_000 { if complete.is_file() || error.is_file() { break; } thread::sleep(Duration::from_millis(5)); } let outcome = if complete.is_file() { Ok(()) } else if error.is_file() { Err(std::io::Error::other(fs::read_to_string(&error)?)) } else { Err(std::io::Error::other("benchmark command timed out")) }; runtime .stop(&coordinator) .map_err(|error| std::io::Error::other(error.to_string()))?; outcome } #[allow(clippy::disallowed_methods, clippy::disallowed_types)] fn run_benchmark_guest(arguments: &[std::ffi::OsString]) -> std::io::Result<()> { if arguments.len() < 3 { return Err(std::io::Error::other( "benchmark guest arguments are malformed", )); } let results = PathBuf::from(&arguments[0]); let output = std::process::Command::new(&arguments[1]) .args(&arguments[2..]) .output(); match output { Ok(output) if output.status.success() => { fs::write(results.join(BENCHMARK_COMPLETE), b"ok")?; } Ok(output) => fs::write( results.join(BENCHMARK_ERROR), format!( "benchmark command exited with {}: {}", output.status, String::from_utf8_lossy(&output.stderr) ), )?, Err(error) => fs::write(results.join(BENCHMARK_ERROR), error.to_string())?, } loop { thread::park(); } } fn exit(result: std::io::Result<()>) -> ExitCode { match result { Ok(()) => ExitCode::SUCCESS, Err(error) => { eprintln!("private runtime role failed: {error}"); ExitCode::FAILURE } } } enum MatrixError { Skip(String), Failure(String), } impl From for MatrixError { fn from(error: std::io::Error) -> Self { Self::Failure(error.to_string()) } } fn run_matrix() -> Result<(), MatrixError> { require_matrix_prerequisites()?; lost_acknowledgment_refuses_target_work()?; replaced_handoff_authority_refuses_target_work()?; normal_stop_and_stop_resume()?; attached_runsc_loss_stops_guest()?; runtime_supervisor_loss_stops_guest()?; host_coordinator_loss_stops_guest()?; run_network_cases() } fn run_network_matrix() -> Result<(), MatrixError> { require_matrix_prerequisites()?; run_network_cases() } fn run_handoff_matrix() -> Result<(), MatrixError> { require_matrix_prerequisites()?; replaced_handoff_authority_refuses_target_work()?; cross_bundle_handoff_refuses_target_work() } fn run_network_cases() -> Result<(), MatrixError> { failed_probe_blocks_sandbox_start()?; networked_sandbox_view_isolated() } fn require_matrix_prerequisites() -> Result<(), MatrixError> { let release = fs::read_to_string("/proc/sys/kernel/osrelease")?; check_platform(&PlatformObservation { operating_system: std::env::consts::OS, architecture: std::env::consts::ARCH, kernel_release: release.trim(), }) .map_err(|error| MatrixError::Skip(error.to_string()))?; require_runtime_prerequisites() } fn require_runtime_prerequisites() -> Result<(), MatrixError> { let runsc = std::env::var_os("PATH") .into_iter() .flat_map(|path| std::env::split_paths(&path).collect::>()) .map(|directory| directory.join("runsc")) .find(|candidate| candidate.is_file()) .ok_or_else(|| MatrixError::Skip("runsc is not installed on PATH".into()))?; for path in [ runsc, PathBuf::from("/bin/sh"), PathBuf::from("/bin/sleep"), PathBuf::from("/lib64/libc.so.6"), PathBuf::from("/lib64/libgcc_s.so.1"), PathBuf::from("/lib64/libm.so.6"), PathBuf::from("/lib64/libtinfo.so.6"), PathBuf::from("/lib64/ld-linux-x86-64.so.2"), ] { if !path.is_file() { return Err(MatrixError::Skip(format!( "required runtime fixture is absent: {}", path.display() ))); } } let socket_directory = tempfile::tempdir()?; let socket_path = socket_directory.path().join("runsc-control.sock"); match UnixListener::bind(&socket_path) { Ok(listener) => drop(listener), Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => { return Err(MatrixError::Skip(format!( "filesystem Unix sockets required by runsc are denied: {error}" ))); } Err(error) => return Err(error.into()), } Ok(()) } fn lost_acknowledgment_refuses_target_work() -> Result<(), MatrixError> { let fixture = Fixture::new()?; let mut pending = fixture.launch_pending()?; let staged = pending .receive_staged(&SandboxCoordinator::new(&fixture.store)) .map_err(failure)?; let participant = sandbox_participant(&fixture.store); assert_eq!(participant.state(), ParticipantState::Intent); let lazaret_core::records::ParticipantOwnership::Sandbox(ownership) = participant.ownership().unwrap() else { unreachable!(); }; assert_eq!( ownership.attached_runsc.authoritative_pidfs().unwrap(), &staged.identity() ); pending.abort().map_err(failure)?; if fixture.activity.exists() { return Err(failure("lost acknowledgment released sandbox work")); } Ok(()) } fn replaced_handoff_authority_refuses_target_work() -> Result<(), MatrixError> { let fixture = Fixture::new()?; let handoff = fixture .bundle .capture_handoff_authority(&fixture.configuration) .map_err(failure)?; let mut pending = fixture.launch_pending()?; let staged = pending .receive_staged(&SandboxCoordinator::new(&fixture.store)) .map_err(failure)?; let workspace = host_path(fixture.configuration.workspace().canonical_label()); let selected = workspace.with_extension("selected"); fs::rename(&workspace, &selected)?; fs::create_dir(&workspace)?; let marker = workspace.join("replacement-marker"); fs::write(&marker, b"unchanged")?; if pending.acknowledge(staged, &handoff).is_ok() { return Err(failure("replaced handoff authority released sandbox work")); } if fs::read(&marker)? != b"unchanged" || workspace.join("activity").exists() || selected.join("activity").exists() { return Err(failure("refused handoff mutated a workspace tree")); } Ok(()) } fn cross_bundle_handoff_refuses_target_work() -> Result<(), MatrixError> { let fixture = Fixture::new()?; let handoff = fixture .bundle .capture_handoff_authority(&fixture.configuration) .map_err(failure)?; let alternate = fixture.alternate_bundle()?; if fixture.bundle.command().arguments() != alternate.command().arguments() { return Err(failure("cross-bundle fixture changed runsc argv")); } let mut pending = fixture.launch_pending_with_command(alternate.command())?; let staged = pending .receive_staged(&SandboxCoordinator::new(&fixture.store)) .map_err(failure)?; if pending.acknowledge(staged, &handoff).is_ok() || fixture.activity.exists() { return Err(failure("cross-bundle handoff released sandbox work")); } Ok(()) } fn normal_stop_and_stop_resume() -> Result<(), MatrixError> { let fixture = Fixture::new()?; let runtime = fixture.launch_committed()?; wait_for_activity(&fixture.activity)?; runtime .signal_attached_runsc(AttachedRunscSignal::Stop) .map_err(failure)?; thread::sleep(Duration::from_millis(50)); runtime .signal_attached_runsc(AttachedRunscSignal::Continue) .map_err(failure)?; runtime .signal_runtime_supervisor(Signal::STOP) .map_err(failure)?; let stopped_size = fs::metadata(&fixture.activity)?.len(); thread::sleep(Duration::from_millis(50)); if fs::metadata(&fixture.activity)?.len() <= stopped_size { return Err(failure( "stopped Runtime Supervisor incorrectly stopped a present sandbox", )); } runtime .signal_runtime_supervisor(Signal::CONT) .map_err(failure)?; runtime .stop(&SandboxCoordinator::new(&fixture.store)) .map_err(failure)?; assert_eq!( sandbox_participant(&fixture.store).state(), ParticipantState::Clean ); assert_activity_stable(&fixture.activity) } fn attached_runsc_loss_stops_guest() -> Result<(), MatrixError> { let fixture = Fixture::new()?; let runtime = fixture.launch_committed()?; wait_for_activity(&fixture.activity)?; runtime .signal_attached_runsc(AttachedRunscSignal::Kill) .map_err(failure)?; runtime.wait_for_attached_runsc_exit().map_err(failure)?; assert_eq!( sandbox_participant(&fixture.store).state(), ParticipantState::Committed ); assert_activity_stable(&fixture.activity) } fn runtime_supervisor_loss_stops_guest() -> Result<(), MatrixError> { let fixture = Fixture::new()?; let runtime = fixture.launch_committed()?; wait_for_activity(&fixture.activity)?; runtime .signal_runtime_supervisor(Signal::KILL) .map_err(failure)?; runtime .wait_for_runtime_supervisor_exit() .map_err(failure)?; assert_eq!( sandbox_participant(&fixture.store).state(), ParticipantState::Committed ); assert_activity_stable(&fixture.activity) } fn host_coordinator_loss_stops_guest() -> Result<(), MatrixError> { let tree = tempfile::tempdir()?; let ready = tree.path().join("host-coordinator-ready"); let activity = tree.path().join("workspace/activity"); let parent = current_process_identity()?; let target = TargetCommand::new(std::env::current_exe()?) .argument(HOST_COORDINATOR_DRIVER) .argument(tree.path()) .argument(&ready); let mut driver = ExactChild::new(child_broker().acquire( parent, ProcessOwnershipContext { domain_id: lazaret_core::identity::DomainId::new([1; 32]), generation_id: GenerationId::new(7), coordinator_id: CoordinatorId::new([3; 32]), slot: HostProcessOwnershipSlot::SandboxHostCoordinator, }, &target, |_| Ok(()), |staged| Ok(ProcessOwnershipCommitAck::v1(staged)), )?); for _ in 0..400 { if ready.is_file() { break; } if driver.process()?.has_exited()? { driver.reap()?; return Err(failure("Host Coordinator driver exited before readiness")); } thread::sleep(Duration::from_millis(5)); } if !ready.is_file() { driver.kill_and_reap()?; return Err(failure("Host Coordinator driver readiness timed out")); } wait_for_activity(&activity)?; driver.kill_and_reap()?; thread::sleep(Duration::from_millis(100)); assert_activity_stable(&activity) } struct ExactChild(Option); impl ExactChild { const fn new(process: RetainedProcess) -> Self { Self(Some(process)) } fn process(&self) -> std::io::Result<&RetainedProcess> { self.0 .as_ref() .ok_or_else(|| std::io::Error::other("exact test child handle is absent")) } fn reap(&mut self) -> std::io::Result<()> { let process = self .0 .take() .ok_or_else(|| std::io::Error::other("exact test child handle is absent"))?; child_broker().reap(process) } fn kill_and_reap(&mut self) -> std::io::Result<()> { self.process()?.send_signal(Signal::KILL)?; self.reap() } } impl Drop for ExactChild { fn drop(&mut self) { if let Some(process) = self.0.take() { let _ = process.send_signal(Signal::KILL); let _ = child_broker().reap(process); } } } fn run_host_coordinator_driver(arguments: &[std::ffi::OsString]) -> std::io::Result<()> { let tree = arguments .first() .map(PathBuf::from) .ok_or_else(|| std::io::Error::other("Host Coordinator fixture path is missing"))?; let ready = arguments .get(1) .map(PathBuf::from) .ok_or_else(|| std::io::Error::other("Host Coordinator ready path is missing"))?; if arguments.len() != 2 { return Err(std::io::Error::other( "Host Coordinator driver arguments are malformed", )); } let fixture = Fixture::in_directory(&tree).map_err(matrix_io)?; let _runtime = fixture.launch_committed().map_err(matrix_io)?; wait_for_activity(&fixture.activity).map_err(matrix_io)?; fs::write(ready, b"ready")?; loop { thread::park(); } } fn matrix_io(error: MatrixError) -> std::io::Error { match error { MatrixError::Skip(detail) | MatrixError::Failure(detail) => std::io::Error::other(detail), } } fn failure(error: impl std::fmt::Display) -> MatrixError { MatrixError::Failure(error.to_string()) } fn wait_for_activity(path: &Path) -> Result<(), MatrixError> { for _ in 0..400 { if path.metadata().is_ok_and(|metadata| metadata.len() >= 2) { return Ok(()); } thread::sleep(Duration::from_millis(5)); } Err(failure("guest activity did not start")) } fn assert_activity_stable(path: &Path) -> Result<(), MatrixError> { let before = fs::metadata(path)?.len(); thread::sleep(Duration::from_millis(100)); let after = fs::metadata(path)?.len(); if before == after { Ok(()) } else { Err(failure( "guest activity continued after causal authority loss", )) } } struct Fixture { _tree: Option, configuration: ResolvedDomainConfiguration, store: HostCoordinatorDomainStore, bundle: SandboxBundle, activity: PathBuf, benchmark_results: Option, } impl Fixture { fn new() -> Result { let tree = tempfile::tempdir()?; let root = tree.path().to_path_buf(); Self::build(&root, Some(tree)) } fn in_directory(root: &Path) -> Result { Self::build(root, None) } #[allow(clippy::too_many_arguments)] fn build_benchmark( root: &Path, workspace: &Path, host_root: &Path, output: &Path, repository: &Path, cargo_home: &Path, rustup_home: &Path, uv_bin: &Path, sample_arguments: &[std::ffi::OsString], ) -> Result { let configuration_directory = root.join("configuration"); let runtime = root.join("runtime"); for directory in [&configuration_directory, &runtime] { fs::create_dir_all(directory)?; } let configuration_path = configuration_directory.join("lazaret.toml"); let paths = [ root, workspace, host_root, output, repository, cargo_home, rustup_home, uv_bin, ]; if paths.iter().any(|path| { path.to_str() .is_none_or(|value| value.contains(['\n', '\r', '"'])) }) { return Err(failure("benchmark composition path is not TOML-safe")); } fs::write( &configuration_path, format!( "schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = [\"/tooling\", \"/fixtures\"]\nenv = {{ PATH = \"env:BENCHMARK_PATH\", CARGO_HOME = \"env:BENCHMARK_CARGO_HOME\", RUSTUP_HOME = \"env:BENCHMARK_RUSTUP_HOME\", CARGO_BUILD_JOBS = \"env:BENCHMARK_CARGO_BUILD_JOBS\" }}\nvolumes = [\n {{ host = \"{}\", container = \"/tooling/usr\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/host\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/output\", kind = \"directory\", access = \"read-write\" }},\n {{ host = \"{}\", container = \"/fixtures/repository\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/cargo\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/rustup\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/uv-bin\", kind = \"directory\", access = \"read-only\" }},\n]\n", root.join("data").display(), Path::new("/usr").display(), host_root.display(), output.display(), repository.display(), cargo_home.display(), rustup_home.display(), uv_bin.display(), ), )?; let environment = BTreeMap::from([ ( "BENCHMARK_PATH".into(), "/tooling/cargo/bin:/tooling/uv-bin:/usr/bin:/bin".into(), ), ("BENCHMARK_CARGO_HOME".into(), "/tooling/cargo".into()), ("BENCHMARK_RUSTUP_HOME".into(), "/tooling/rustup".into()), ("BENCHMARK_CARGO_BUILD_JOBS".into(), "2".into()), ]); let configuration = load_trusted_configuration(ConfigurationLoadRequest { explicit_configuration_path: Some(configuration_path), selected_profile: "default".into(), workspace: workspace.to_path_buf(), home: root.join("home"), xdg_config_home: None, xdg_data_home: None, xdg_runtime_dir: runtime, environment, sandbox_layout: production_sandbox_layout().map_err(failure)?, }) .map_err(failure)?; let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let results = output.to_path_buf(); stage_benchmark_rootfs_and_control(&runtime_domain)?; let mut guest_arguments = vec![ BENCHMARK_GUEST.to_owned(), "/fixtures/output".to_owned(), "/usr/bin/python3".to_owned(), "/fixtures/repository/tools/ignored_workspace_benchmark.py".to_owned(), "sample".to_owned(), ]; guest_arguments.extend( sample_arguments .iter() .map(|argument| argument.to_string_lossy().into_owned()), ); let root_process = RootProcess::new("/bin/runtime-composition", guest_arguments).map_err(failure)?; let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; let bundle = build_sandbox_bundle(SandboxBuildRequest { configuration: &configuration, container_id: ContainerId::new([6; 32]), expected_network_namespace: namespace, root_process, terminal: None, }) .map_err(failure)?; Self::finish_build( None, configuration, bundle, workspace.join("activity"), Some(results), ) } fn build(root: &Path, tree: Option) -> Result { let configuration_directory = root.join("configuration"); let workspace = root.join("workspace"); let runtime = root.join("runtime"); fs::create_dir(&configuration_directory)?; fs::create_dir(&workspace)?; fs::create_dir(&runtime)?; let configuration_path = configuration_directory.join("lazaret.toml"); fs::write( &configuration_path, format!( "schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = []\n", root.join("data").display() ), )?; let configuration = load_trusted_configuration(ConfigurationLoadRequest { explicit_configuration_path: Some(configuration_path), selected_profile: "default".into(), workspace: workspace.clone(), home: root.join("home"), xdg_config_home: None, xdg_data_home: None, xdg_runtime_dir: runtime, environment: BTreeMap::new(), sandbox_layout: production_sandbox_layout().map_err(failure)?, }) .map_err(failure)?; let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let container_id = ContainerId::new([6; 32]); stage_rootfs_and_control(&runtime_domain, container_id)?; let root_process = RootProcess::new( "/bin/sh", [ "-c".to_owned(), "while :; do echo x >> /workspace/activity; /bin/sleep 0.02; done".to_owned(), ], ) .map_err(failure)?; let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; let bundle = build_sandbox_bundle(SandboxBuildRequest { configuration: &configuration, container_id, expected_network_namespace: namespace, root_process, terminal: None, }) .map_err(failure)?; Self::finish_build( tree, configuration, bundle, workspace.join("activity"), None, ) } fn finish_build( tree: Option, configuration: ResolvedDomainConfiguration, bundle: SandboxBundle, activity: PathBuf, benchmark_results: Option, ) -> Result { let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let container_id = ContainerId::new([6; 32]); let bundle_path = runtime_domain .join("bundles") .join(hex(container_id.as_bytes())); fs::create_dir_all(&bundle_path)?; fs::write(bundle_path.join("config.json"), bundle.specification_json())?; fs::create_dir_all(runtime_domain.join("runsc"))?; let domain = Domain { domain_id: configuration.domain_id(), profile_name: configuration.selected_profile().to_owned(), canonical_workspace: configuration.workspace().canonical_label().clone(), }; let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label()); let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?; let store = HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain) .map_err(failure)?; store .apply(&[DomainStoreMutation::PutGeneration(Generation { domain_id: configuration.domain_id(), generation_id: GenerationId::new(7), configuration_fingerprint: configuration.configuration_fingerprint(), coordinator_id: CoordinatorId::new([3; 32]), network_sidecar_id: NetworkSidecarId::new([4; 32]), sandbox_id: SandboxId::new([5; 32]), durable_decision: DurableDecision::None, })]) .map_err(failure)?; SandboxCoordinator::new(&store) .begin_intent( GenerationId::new(7), container_id, &runtime_domain.join("runsc"), BundleDigest::new([7; 32]), current_process_identity()?, ) .map_err(failure)?; Ok(Self { _tree: tree, configuration, store, bundle, activity, benchmark_results, }) } fn launch_pending(&self) -> Result { self.launch_pending_with_command(self.bundle.command()) } fn launch_pending_with_command( &self, command: &RunscCommand, ) -> Result { let launch = RuntimeSupervisorLaunch::from_runsc_command( ProcessOwnershipContext { domain_id: self.configuration.domain_id(), generation_id: GenerationId::new(7), coordinator_id: CoordinatorId::new([3; 32]), slot: lazaret_core::records::HostProcessOwnershipSlot::SandboxRuntimeSupervisor, }, command, )?; AwaitingAttachedRunsc::launch(&SandboxCoordinator::new(&self.store), &launch) .map_err(failure) } fn alternate_bundle(&self) -> Result { let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; build_sandbox_bundle(SandboxBuildRequest { configuration: &self.configuration, container_id: ContainerId::new([6; 32]), expected_network_namespace: namespace, root_process: RootProcess::new( "/bin/sh", [ "-c".to_owned(), "echo cross-bundle >> /workspace/activity".to_owned(), ], ) .map_err(failure)?, terminal: None, }) .map_err(failure) } fn launch_committed(&self) -> Result { let handoff = self .bundle .capture_handoff_authority(&self.configuration) .map_err(failure)?; let mut pending = self.launch_pending()?; let staged = pending .receive_staged(&SandboxCoordinator::new(&self.store)) .map_err(failure)?; pending .acknowledge(staged, &handoff) .map_err(failure)? .commit(&SandboxCoordinator::new(&self.store)) .map_err(failure) } } struct NetworkFixture { _tree: TempDir, _host_loopback: TcpListener, _host_abstract: UnixListener, configuration: ResolvedDomainConfiguration, store: HostCoordinatorDomainStore, bundle: SandboxBundle, policy: lazaret_core::network::CanonicalDerivedNetworkPolicy, probe_result: PathBuf, } impl NetworkFixture { fn new() -> Result { let tree = tempfile::tempdir()?; let root = tree.path(); let workspace = root.join("workspace"); let runtime = root.join("runtime"); let configuration = network_configuration(root, &workspace, &runtime)?; let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let container_id = ContainerId::new([16; 32]); stage_rootfs_and_control(&runtime_domain, container_id)?; let rootfs = runtime_domain.join("state-anchor/merged"); fs::copy( std::env::current_exe()?, rootfs.join("bin/lazaret-network-probe"), )?; fs::create_dir_all(rootfs.join("etc"))?; fs::write(rootfs.join("etc/resolv.conf"), b"nameserver 10.0.2.3\n")?; let host_loopback = TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?; let host_loopback_port = host_loopback.local_addr()?.port(); let abstract_name = format!("lazaret-network-v3-{}", std::process::id()).into_bytes(); let host_abstract = UnixListener::bind_addr(&SocketAddr::from_abstract_name(&abstract_name)?)?; let root_process = RootProcess::new( "/bin/lazaret-network-probe", vec![ SANDBOX_NETWORK_PROBE.into(), host_loopback_port.to_string(), String::from_utf8(abstract_name.clone()).map_err(failure)?, ], ) .map_err(failure)?; let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; let bundle = build_sandbox_bundle(SandboxBuildRequest { configuration: &configuration, container_id, expected_network_namespace: namespace, root_process, terminal: None, }) .map_err(failure)?; let bundle_path = runtime_domain .join("bundles") .join(hex(container_id.as_bytes())); fs::create_dir_all(&bundle_path)?; fs::write(bundle_path.join("config.json"), bundle.specification_json())?; fs::create_dir_all(runtime_domain.join("runsc"))?; let store = network_store(&configuration, &runtime_domain, container_id)?; let policy = derive_ipv4_policy(&configuration, [IpAddress::V4([10, 0, 2, 3])]).map_err(failure)?; Ok(Self { _tree: tree, _host_loopback: host_loopback, _host_abstract: host_abstract, configuration, store, bundle, policy, probe_result: workspace.join("network-probe"), }) } fn launch(&self) -> Result { self.launch_with_permitted([1, 1, 1, 1], 443) } fn launch_with_permitted( &self, permitted_address: [u8; 4], permitted_port: u16, ) -> Result { let handoff = self .bundle .capture_handoff_authority(&self.configuration) .map_err(failure)?; let launch = RuntimeSupervisorLaunch::from_networked_runsc_command( ProcessOwnershipContext { domain_id: self.configuration.domain_id(), generation_id: GenerationId::new(7), coordinator_id: CoordinatorId::new([13; 32]), slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor, }, self.bundle.command(), RuntimeNetworkLaunch { policy: &self.policy, configuration_fingerprint: self.configuration.configuration_fingerprint(), network_sidecar_id: NetworkSidecarId::new([14; 32]), }, )?; let sandbox = SandboxCoordinator::new(&self.store); let network = NetworkCoordinator::new(&self.store); let mut pending = AwaitingAttachedRunsc::launch_networked(&sandbox, &network, &launch) .map_err(failure)?; pending.receive_helper_staged(&network).map_err(failure)?; pending .receive_network_readiness( &network, &self.policy, |namespace| NetworkProbeRequest { domain_id: self.configuration.domain_id(), generation_id: GenerationId::new(7), enforcement_namespace: namespace, denied: vec![TcpProbeTarget { address: IpAddress::V4([10, 1, 2, 3]), port: 443, }], permitted: vec![TcpProbeTarget { address: IpAddress::V4(permitted_address), port: permitted_port, }], dns: vec![DnsProbeTarget { resolver: IpAddress::V4([10, 0, 2, 3]), query_name: "example.com".into(), }], }, &ReachabilityExpectation { denied_destinations: vec![IpAddress::V4([10, 1, 2, 3])], permitted_destinations: vec![IpAddress::V4(permitted_address)], dns_queries: vec![DnsQueryExpectation { resolver: IpAddress::V4([10, 0, 2, 3]), query_name: "example.com".into(), }], }, ) .map_err(failure)?; let staged = pending.receive_staged(&sandbox).map_err(failure)?; let (runtime, _) = pending .acknowledge(staged, &handoff) .map_err(failure)? .commit_networked(&sandbox) .map_err(failure)?; Ok(runtime) } } fn network_configuration( root: &Path, workspace: &Path, runtime: &Path, ) -> Result { let configuration_directory = root.join("configuration"); fs::create_dir(&configuration_directory)?; fs::create_dir(workspace)?; fs::create_dir(runtime)?; let configuration_path = configuration_directory.join("lazaret.toml"); fs::write( &configuration_path, format!( "schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[profiles.default]\nimage = \"example.invalid/network-fixture:v1\"\npersistent_dirs = []\n", root.join("data").display() ), )?; load_trusted_configuration(ConfigurationLoadRequest { explicit_configuration_path: Some(configuration_path), selected_profile: "default".into(), workspace: workspace.to_path_buf(), home: root.join("home"), xdg_config_home: None, xdg_data_home: None, xdg_runtime_dir: runtime.to_path_buf(), environment: BTreeMap::new(), sandbox_layout: production_sandbox_layout().map_err(failure)?, }) .map_err(failure) } fn network_store( configuration: &ResolvedDomainConfiguration, runtime_domain: &Path, container_id: ContainerId, ) -> Result { let domain = Domain { domain_id: configuration.domain_id(), profile_name: configuration.selected_profile().to_owned(), canonical_workspace: configuration.workspace().canonical_label().clone(), }; let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label()); let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?; let store = HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain) .map_err(failure)?; store .apply(&[DomainStoreMutation::PutGeneration(Generation { domain_id: configuration.domain_id(), generation_id: GenerationId::new(7), configuration_fingerprint: configuration.configuration_fingerprint(), coordinator_id: CoordinatorId::new([13; 32]), network_sidecar_id: NetworkSidecarId::new([14; 32]), sandbox_id: SandboxId::new([15; 32]), durable_decision: DurableDecision::None, })]) .map_err(failure)?; SandboxCoordinator::new(&store) .begin_intent( GenerationId::new(7), container_id, &runtime_domain.join("runsc"), BundleDigest::new([17; 32]), current_process_identity()?, ) .map_err(failure)?; Ok(store) } fn failed_probe_blocks_sandbox_start() -> Result<(), MatrixError> { let fixture = NetworkFixture::new()?; if let Ok(runtime) = fixture.launch_with_permitted([203, 0, 113, 1], 9) { runtime .stop(&SandboxCoordinator::new(&fixture.store)) .map_err(failure)?; return Err(failure("failed physical probe started the sandbox")); } let sandbox = sandbox_participant(&fixture.store); if sandbox.state() != ParticipantState::Intent || fixture.probe_result.exists() { Err(failure( "failed physical probe advanced Sandbox state or guest work", )) } else { Ok(()) } } fn networked_sandbox_view_isolated() -> Result<(), MatrixError> { let fixture = NetworkFixture::new()?; let runtime = fixture.launch()?; for _ in 0..6000 { if fixture.probe_result.is_file() { break; } thread::sleep(Duration::from_millis(5)); } let result = fs::read_to_string(&fixture.probe_result)?; for required in [ "denied=true", "public=true", "dns=true", "host_loopback=false", "abstract_socket=false", ] { if !result.lines().any(|line| line == required) { return Err(failure(format!( "guest network assertion missing: {required}" ))); } } runtime .stop(&SandboxCoordinator::new(&fixture.store)) .map_err(failure) } fn run_sandbox_network_probe(arguments: &[std::ffi::OsString]) -> std::io::Result<()> { let host_port = arguments .first() .and_then(|value| value.to_str()) .and_then(|value| value.parse::().ok()) .filter(|port| *port != 0) .ok_or_else(|| std::io::Error::other("sandbox probe host port is invalid"))?; let abstract_name = arguments .get(1) .and_then(|value| value.to_str()) .ok_or_else(|| std::io::Error::other("sandbox probe abstract name is invalid"))?; let timeout = Duration::from_secs(3); let denied = TcpStream::connect_timeout( &SocketAddrV4::new(Ipv4Addr::new(10, 1, 2, 3), 443).into(), timeout, ) .is_err(); let public = TcpStream::connect_timeout( &SocketAddrV4::new(Ipv4Addr::new(1, 1, 1, 1), 443).into(), timeout, ) .is_ok(); let dns = ("example.com", 443) .to_socket_addrs() .is_ok_and(|mut addresses| addresses.next().is_some()); let host_loopback = TcpStream::connect_timeout( &SocketAddrV4::new(Ipv4Addr::new(10, 0, 2, 2), host_port).into(), timeout, ) .is_ok(); let abstract_socket = UnixStream::connect_addr(&SocketAddr::from_abstract_name(abstract_name.as_bytes())?) .is_ok(); fs::write( "/workspace/.network-probe", format!( "denied={denied}\npublic={public}\ndns={dns}\nhost_loopback={host_loopback}\nabstract_socket={abstract_socket}\n" ), )?; fs::rename("/workspace/.network-probe", "/workspace/network-probe")?; loop { thread::sleep(Duration::from_secs(1)); } } fn stage_rootfs_and_control( runtime_domain: &Path, container_id: ContainerId, ) -> Result<(), MatrixError> { let rootfs = runtime_domain.join("state-anchor/merged"); for directory in ["bin", "lib64"] { fs::create_dir_all(rootfs.join(directory))?; } for (source, destination) in [ ("/bin/sh", "bin/sh"), ("/bin/sleep", "bin/sleep"), ("/lib64/libc.so.6", "lib64/libc.so.6"), ("/lib64/libgcc_s.so.1", "lib64/libgcc_s.so.1"), ("/lib64/libm.so.6", "lib64/libm.so.6"), ("/lib64/libtinfo.so.6", "lib64/libtinfo.so.6"), ("/lib64/ld-linux-x86-64.so.2", "lib64/ld-linux-x86-64.so.2"), ] { fs::copy(source, rootfs.join(destination))?; } let control = runtime_domain .join("sandbox-control") .join(hex(container_id.as_bytes())); for directory in ["credentials", "mailbox/requests", "mailbox/results", "bin"] { fs::create_dir_all(control.join(directory))?; } fs::copy("/bin/true", control.join("bin/lazaret-escalate"))?; Ok(()) } fn stage_benchmark_rootfs_and_control(runtime_domain: &Path) -> Result<(), MatrixError> { let rootfs = runtime_domain.join("state-anchor/merged"); for directory in [ "bin", "tooling/usr", "tooling/cargo", "tooling/rustup", "tooling/uv-bin", "fixtures/host", "fixtures/output", "fixtures/repository", ] { fs::create_dir_all(rootfs.join(directory))?; } fs::copy( std::env::current_exe()?, rootfs.join("bin/runtime-composition"), )?; symlink("tooling/usr", rootfs.join("usr"))?; symlink("usr/lib", rootfs.join("lib"))?; symlink("usr/lib64", rootfs.join("lib64"))?; symlink("usr/sbin", rootfs.join("sbin"))?; let control = runtime_domain.join("sandbox-control").join(hex(&[6; 32])); for directory in ["credentials", "mailbox/requests", "mailbox/results", "bin"] { fs::create_dir_all(control.join(directory))?; } fs::copy("/bin/true", control.join("bin/lazaret-escalate"))?; Ok(()) } fn sandbox_participant(store: &HostCoordinatorDomainStore) -> lazaret_core::records::Participant { store .snapshot() .unwrap() .participants .into_iter() .find(|participant| participant.role() == ParticipantRole::Sandbox) .unwrap() } fn lock_file(path: &Path) -> File { OpenOptions::new() .read(true) .write(true) .create(true) .truncate(false) .open(path) .unwrap() } fn host_path(path: &HostPath) -> PathBuf { PathBuf::from(std::ffi::OsString::from_vec(path.as_bytes().to_vec())) } fn hex(bytes: &[u8]) -> String { use std::fmt::Write as _; bytes.iter().fold(String::new(), |mut output, byte| { write!(output, "{byte:02x}").unwrap(); output }) }