use super::{ Arc, AwaitingAttachedRunsc, BENCHMARK_GUEST, BTreeMap, BundleDigest, CommittedSandboxRuntime, ConfigurationLoadRequest, ContainerId, CoordinatorId, DnsProbeTarget, DnsQueryExpectation, Domain, DomainStoreMutation, DurableDecision, Generation, GenerationId, HostCoordinatorDomainStore, HostProcessOwnershipSlot, IpAddress, Ipv4Addr, MatrixError, NETWORK_NAMESPACE_PATH, NetworkCoordinator, NetworkProbeRequest, NetworkSidecarId, Path, PathBuf, ProcessOwnershipContext, ReachabilityExpectation, ResolvedDomainConfiguration, RootProcess, RunscCommand, RuntimeNetworkLaunch, RuntimeSupervisorLaunch, SANDBOX_NETWORK_PROBE, SandboxBuildRequest, SandboxBundle, SandboxCoordinator, SandboxId, SocketAddr, SocketAddrExt, TcpListener, TcpProbeTarget, TempDir, UnixListener, build_sandbox_bundle, current_mount_identity, current_process_identity, default_sandbox_layout, derive_ipv4_policy, failure, fs, hex, host_path, load_trusted_configuration, lock_file, stage_benchmark_rootfs_and_control, stage_rootfs_and_control, try_exclusive_flock, }; pub(super) struct Fixture { _tree: Option, pub(super) configuration: ResolvedDomainConfiguration, pub(super) store: Arc, pub(super) bundle: SandboxBundle, pub(super) activity: PathBuf, pub(super) benchmark_results: Option, } impl Fixture { pub(super) fn new() -> Result { let tree = tempfile::tempdir()?; let root = tree.path().to_path_buf(); Self::build(&root, Some(tree)) } pub(super) fn in_directory(root: &Path) -> Result { Self::build(root, None) } #[allow(clippy::too_many_arguments)] pub(super) fn build_benchmark( root: &Path, workspace: &Path, host_root: &Path, output: &Path, repository: &Path, cargo_home: &Path, rustup_home: &Path, uv_bin: &Path, sample_arguments: &[std::ffi::OsString], ) -> Result { let configuration_directory = root.join("configuration"); let runtime = root.join("runtime"); for directory in [&configuration_directory, &runtime] { fs::create_dir_all(directory)?; } let configuration_path = configuration_directory.join("lazaret.toml"); let paths = [ root, workspace, host_root, output, repository, cargo_home, rustup_home, uv_bin, ]; if paths.iter().any(|path| { path.to_str() .is_none_or(|value| value.contains(['\n', '\r', '"'])) }) { return Err(failure("benchmark composition path is not TOML-safe")); } fs::write( &configuration_path, format!( "schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = [\"/tooling\", \"/fixtures\"]\nenv = {{ PATH = \"env:BENCHMARK_PATH\", CARGO_HOME = \"env:BENCHMARK_CARGO_HOME\", RUSTUP_HOME = \"env:BENCHMARK_RUSTUP_HOME\", CARGO_BUILD_JOBS = \"env:BENCHMARK_CARGO_BUILD_JOBS\" }}\nvolumes = [\n {{ host = \"{}\", container = \"/tooling/usr\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/host\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/output\", kind = \"directory\", access = \"read-write\" }},\n {{ host = \"{}\", container = \"/fixtures/repository\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/cargo\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/rustup\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/uv-bin\", kind = \"directory\", access = \"read-only\" }},\n]\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n", root.join("data").display(), Path::new("/usr").display(), host_root.display(), output.display(), repository.display(), cargo_home.display(), rustup_home.display(), uv_bin.display(), ), )?; let environment = BTreeMap::from([ ( "BENCHMARK_PATH".into(), "/tooling/cargo/bin:/tooling/uv-bin:/usr/bin:/bin".into(), ), ("BENCHMARK_CARGO_HOME".into(), "/tooling/cargo".into()), ("BENCHMARK_RUSTUP_HOME".into(), "/tooling/rustup".into()), ("BENCHMARK_CARGO_BUILD_JOBS".into(), "2".into()), ]); let configuration = load_trusted_configuration(ConfigurationLoadRequest { explicit_configuration_path: Some(configuration_path), selected_profile: "default".into(), workspace: workspace.to_path_buf(), home: root.join("home"), xdg_config_home: None, xdg_data_home: None, xdg_runtime_dir: runtime, environment, sandbox_layout: default_sandbox_layout().map_err(failure)?, }) .map_err(failure)?; let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let results = output.to_path_buf(); stage_benchmark_rootfs_and_control(&runtime_domain)?; let mut guest_arguments = vec![ BENCHMARK_GUEST.to_owned(), "/fixtures/output".to_owned(), "/usr/bin/python3".to_owned(), "/fixtures/repository/tools/ignored_workspace_benchmark.py".to_owned(), "sample".to_owned(), ]; guest_arguments.extend( sample_arguments .iter() .map(|argument| argument.to_string_lossy().into_owned()), ); let root_process = RootProcess::new("/bin/runtime-composition", guest_arguments).map_err(failure)?; let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; let bundle = build_sandbox_bundle(SandboxBuildRequest { configuration: &configuration, container_id: ContainerId::new([6; 32]), expected_network_namespace: namespace, root_process, terminal: None, prepared_root: runtime_domain.join("prepared-root"), }) .map_err(failure)?; Self::finish_build( None, configuration, bundle, workspace.join("activity"), Some(results), ) } fn build(root: &Path, tree: Option) -> Result { let configuration_directory = root.join("configuration"); let workspace = root.join("workspace"); let runtime = root.join("runtime"); fs::create_dir(&configuration_directory)?; fs::create_dir(&workspace)?; fs::create_dir(&runtime)?; let configuration_path = configuration_directory.join("lazaret.toml"); fs::write( &configuration_path, format!( "schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n", root.join("data").display() ), )?; let configuration = load_trusted_configuration(ConfigurationLoadRequest { explicit_configuration_path: Some(configuration_path), selected_profile: "default".into(), workspace: workspace.clone(), home: root.join("home"), xdg_config_home: None, xdg_data_home: None, xdg_runtime_dir: runtime, environment: BTreeMap::new(), sandbox_layout: default_sandbox_layout().map_err(failure)?, }) .map_err(failure)?; let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let container_id = ContainerId::new([6; 32]); stage_rootfs_and_control(&runtime_domain, container_id)?; let root_process = RootProcess::new( "/bin/sh", [ "-c".to_owned(), "while :; do echo x >> /workspace/activity; /bin/sleep 0.02; done".to_owned(), ], ) .map_err(failure)?; let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; let bundle = build_sandbox_bundle(SandboxBuildRequest { configuration: &configuration, container_id, expected_network_namespace: namespace, root_process, terminal: None, prepared_root: runtime_domain.join("prepared-root"), }) .map_err(failure)?; Self::finish_build( tree, configuration, bundle, workspace.join("activity"), None, ) } fn finish_build( tree: Option, configuration: ResolvedDomainConfiguration, bundle: SandboxBundle, activity: PathBuf, benchmark_results: Option, ) -> Result { let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let container_id = ContainerId::new([6; 32]); let bundle_path = runtime_domain .join("bundles") .join(hex(container_id.as_bytes())); fs::create_dir_all(&bundle_path)?; fs::write(bundle_path.join("config.json"), bundle.specification_json())?; fs::create_dir_all(runtime_domain.join("runsc"))?; let domain = Domain { domain_id: configuration.domain_id(), profile_name: configuration.selected_profile().to_owned(), canonical_workspace: configuration.workspace().canonical_label().clone(), }; let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label()); let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?; let store = Arc::new( HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain) .map_err(failure)?, ); store .apply(&[DomainStoreMutation::PutGeneration(Generation { domain_id: configuration.domain_id(), generation_id: GenerationId::new(7), configuration_fingerprint: configuration.configuration_fingerprint(), coordinator_id: CoordinatorId::new([3; 32]), network_sidecar_id: NetworkSidecarId::new([4; 32]), sandbox_id: SandboxId::new([5; 32]), durable_decision: DurableDecision::None, })]) .map_err(failure)?; SandboxCoordinator::new(&store) .begin_intent( GenerationId::new(7), container_id, &runtime_domain.join("runsc"), current_mount_identity(&runtime_domain.join("runsc"))?, BundleDigest::new([7; 32]), current_process_identity()?, ) .map_err(failure)?; Ok(Self { _tree: tree, configuration, store, bundle, activity, benchmark_results, }) } pub(super) fn launch_pending(&self) -> Result { self.launch_pending_with_command(self.bundle.command()) } pub(super) fn launch_pending_with_command( &self, command: &RunscCommand, ) -> Result { let launch = RuntimeSupervisorLaunch::from_runsc_command( ProcessOwnershipContext { domain_id: self.configuration.domain_id(), generation_id: GenerationId::new(7), coordinator_id: CoordinatorId::new([3; 32]), slot: lazaret_core::records::HostProcessOwnershipSlot::SandboxRuntimeSupervisor, }, command, )?; AwaitingAttachedRunsc::launch(&SandboxCoordinator::new(&self.store), &launch) .map_err(failure) } pub(super) fn alternate_bundle(&self) -> Result { let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; build_sandbox_bundle(SandboxBuildRequest { configuration: &self.configuration, container_id: ContainerId::new([6; 32]), expected_network_namespace: namespace, root_process: RootProcess::new( "/bin/sh", [ "-c".to_owned(), "echo cross-bundle >> /workspace/activity".to_owned(), ], ) .map_err(failure)?, terminal: None, prepared_root: host_path(self.configuration.runtime_domain_root().canonical_label()) .join("prepared-root"), }) .map_err(failure) } pub(super) fn launch_committed(&self) -> Result { let handoff = self .bundle .capture_handoff_authority(&self.configuration) .map_err(failure)?; let mut pending = self.launch_pending()?; let staged = pending .receive_staged(&SandboxCoordinator::new(&self.store)) .map_err(failure)?; pending .acknowledge(staged, &handoff) .map_err(failure)? .commit(&SandboxCoordinator::new(&self.store)) .map_err(failure) } } pub(super) struct NetworkFixture { _tree: TempDir, _host_loopback: TcpListener, _host_abstract: UnixListener, pub(super) configuration: ResolvedDomainConfiguration, pub(super) store: HostCoordinatorDomainStore, pub(super) bundle: SandboxBundle, pub(super) policy: lazaret_core::network::CanonicalDerivedNetworkPolicy, pub(super) probe_result: PathBuf, } impl NetworkFixture { pub(super) fn new() -> Result { let tree = tempfile::tempdir()?; let root = tree.path(); let workspace = root.join("workspace"); let runtime = root.join("runtime"); let configuration = network_configuration(root, &workspace, &runtime)?; let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label()); let container_id = ContainerId::new([16; 32]); stage_rootfs_and_control(&runtime_domain, container_id)?; let rootfs = runtime_domain.join("prepared-root"); fs::copy( std::env::current_exe()?, rootfs.join("bin/lazaret-network-probe"), )?; fs::write( runtime_domain.join("state-anchor/runtime-files/resolv.conf"), b"nameserver 10.0.2.3\n", )?; let host_loopback = TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?; let host_loopback_port = host_loopback.local_addr()?.port(); let abstract_name = format!("lazaret-network-v3-{}", std::process::id()).into_bytes(); let host_abstract = UnixListener::bind_addr(&SocketAddr::from_abstract_name(&abstract_name)?)?; let root_process = RootProcess::new( "/bin/lazaret-network-probe", vec![ SANDBOX_NETWORK_PROBE.into(), host_loopback_port.to_string(), String::from_utf8(abstract_name.clone()).map_err(failure)?, ], ) .map_err(failure)?; let namespace = lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?; let bundle = build_sandbox_bundle(SandboxBuildRequest { configuration: &configuration, container_id, expected_network_namespace: namespace, root_process, terminal: None, prepared_root: runtime_domain.join("prepared-root"), }) .map_err(failure)?; let bundle_path = runtime_domain .join("bundles") .join(hex(container_id.as_bytes())); fs::create_dir_all(&bundle_path)?; fs::write(bundle_path.join("config.json"), bundle.specification_json())?; fs::create_dir_all(runtime_domain.join("runsc"))?; let store = network_store(&configuration, &runtime_domain, container_id)?; let policy = derive_ipv4_policy(&configuration, [IpAddress::V4([10, 0, 2, 3])]).map_err(failure)?; Ok(Self { _tree: tree, _host_loopback: host_loopback, _host_abstract: host_abstract, configuration, store, bundle, policy, probe_result: workspace.join("network-probe"), }) } pub(super) fn launch(&self) -> Result { self.launch_with_permitted([1, 1, 1, 1], 443) } pub(super) fn launch_with_permitted( &self, permitted_address: [u8; 4], permitted_port: u16, ) -> Result { let handoff = self .bundle .capture_handoff_authority(&self.configuration) .map_err(failure)?; let launch = RuntimeSupervisorLaunch::from_networked_runsc_command( ProcessOwnershipContext { domain_id: self.configuration.domain_id(), generation_id: GenerationId::new(7), coordinator_id: CoordinatorId::new([13; 32]), slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor, }, self.bundle.command(), RuntimeNetworkLaunch { policy: &self.policy, configuration_fingerprint: self.configuration.configuration_fingerprint(), network_sidecar_id: NetworkSidecarId::new([14; 32]), }, )?; let sandbox = SandboxCoordinator::new(&self.store); let network = NetworkCoordinator::new(&self.store); let mut pending = AwaitingAttachedRunsc::launch_networked(&sandbox, &network, &launch) .map_err(failure)?; pending.receive_helper_staged(&network).map_err(failure)?; pending .receive_network_readiness( &network, &self.policy, |namespace| NetworkProbeRequest { domain_id: self.configuration.domain_id(), generation_id: GenerationId::new(7), enforcement_namespace: namespace, denied: vec![TcpProbeTarget { address: IpAddress::V4([10, 1, 2, 3]), port: 443, }], permitted: vec![TcpProbeTarget { address: IpAddress::V4(permitted_address), port: permitted_port, }], dns: vec![DnsProbeTarget { resolver: IpAddress::V4([10, 0, 2, 3]), query_name: "example.com".into(), }], }, &ReachabilityExpectation { denied_destinations: vec![IpAddress::V4([10, 1, 2, 3])], permitted_destinations: vec![IpAddress::V4(permitted_address)], dns_queries: vec![DnsQueryExpectation { resolver: IpAddress::V4([10, 0, 2, 3]), query_name: "example.com".into(), }], }, ) .map_err(failure)?; let staged = pending.receive_staged(&sandbox).map_err(failure)?; let (runtime, _) = pending .acknowledge(staged, &handoff) .map_err(failure)? .commit_networked(&sandbox) .map_err(failure)?; Ok(runtime) } } fn network_configuration( root: &Path, workspace: &Path, runtime: &Path, ) -> Result { let configuration_directory = root.join("configuration"); fs::create_dir(&configuration_directory)?; fs::create_dir(workspace)?; fs::create_dir(runtime)?; let configuration_path = configuration_directory.join("lazaret.toml"); fs::write( &configuration_path, format!( "schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/network-fixture:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n", root.join("data").display() ), )?; load_trusted_configuration(ConfigurationLoadRequest { explicit_configuration_path: Some(configuration_path), selected_profile: "default".into(), workspace: workspace.to_path_buf(), home: root.join("home"), xdg_config_home: None, xdg_data_home: None, xdg_runtime_dir: runtime.to_path_buf(), environment: BTreeMap::new(), sandbox_layout: default_sandbox_layout().map_err(failure)?, }) .map_err(failure) } fn network_store( configuration: &ResolvedDomainConfiguration, runtime_domain: &Path, container_id: ContainerId, ) -> Result { let domain = Domain { domain_id: configuration.domain_id(), profile_name: configuration.selected_profile().to_owned(), canonical_workspace: configuration.workspace().canonical_label().clone(), }; let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label()); let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?; let store = HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain) .map_err(failure)?; store .apply(&[DomainStoreMutation::PutGeneration(Generation { domain_id: configuration.domain_id(), generation_id: GenerationId::new(7), configuration_fingerprint: configuration.configuration_fingerprint(), coordinator_id: CoordinatorId::new([13; 32]), network_sidecar_id: NetworkSidecarId::new([14; 32]), sandbox_id: SandboxId::new([15; 32]), durable_decision: DurableDecision::None, })]) .map_err(failure)?; SandboxCoordinator::new(&store) .begin_intent( GenerationId::new(7), container_id, &runtime_domain.join("runsc"), current_mount_identity(&runtime_domain.join("runsc"))?, BundleDigest::new([17; 32]), current_process_identity()?, ) .map_err(failure)?; Ok(store) }