lazaret/crates/lazaret-host-coordinator/tests/runtime-composition/fixtures.rs

534 lines
23 KiB
Rust

use super::{
Arc, AwaitingAttachedRunsc, BENCHMARK_GUEST, BTreeMap, BundleDigest, CommittedSandboxRuntime,
ConfigurationLoadRequest, ContainerId, CoordinatorId, DnsProbeTarget, DnsQueryExpectation,
Domain, DomainStoreMutation, DurableDecision, Generation, GenerationId,
HostCoordinatorDomainStore, HostProcessOwnershipSlot, IpAddress, Ipv4Addr, MatrixError,
NETWORK_NAMESPACE_PATH, NetworkCoordinator, NetworkProbeRequest, NetworkSidecarId, Path,
PathBuf, ProcessOwnershipContext, ReachabilityExpectation, ResolvedDomainConfiguration,
RootProcess, RunscCommand, RuntimeNetworkLaunch, RuntimeSupervisorLaunch,
SANDBOX_NETWORK_PROBE, SandboxBuildRequest, SandboxBundle, SandboxCoordinator, SandboxId,
SocketAddr, SocketAddrExt, TcpListener, TcpProbeTarget, TempDir, UnixListener,
build_sandbox_bundle, current_mount_identity, current_process_identity, default_sandbox_layout,
derive_ipv4_policy, failure, fs, hex, host_path, load_trusted_configuration, lock_file,
stage_benchmark_rootfs_and_control, stage_rootfs_and_control, try_exclusive_flock,
};
pub(super) struct Fixture {
_tree: Option<TempDir>,
pub(super) configuration: ResolvedDomainConfiguration,
pub(super) store: Arc<HostCoordinatorDomainStore>,
pub(super) bundle: SandboxBundle,
pub(super) activity: PathBuf,
pub(super) benchmark_results: Option<PathBuf>,
}
impl Fixture {
pub(super) fn new() -> Result<Self, MatrixError> {
let tree = tempfile::tempdir()?;
let root = tree.path().to_path_buf();
Self::build(&root, Some(tree))
}
pub(super) fn in_directory(root: &Path) -> Result<Self, MatrixError> {
Self::build(root, None)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn build_benchmark(
root: &Path,
workspace: &Path,
host_root: &Path,
output: &Path,
repository: &Path,
cargo_home: &Path,
rustup_home: &Path,
uv_bin: &Path,
sample_arguments: &[std::ffi::OsString],
) -> Result<Self, MatrixError> {
let configuration_directory = root.join("configuration");
let runtime = root.join("runtime");
for directory in [&configuration_directory, &runtime] {
fs::create_dir_all(directory)?;
}
let configuration_path = configuration_directory.join("lazaret.toml");
let paths = [
root,
workspace,
host_root,
output,
repository,
cargo_home,
rustup_home,
uv_bin,
];
if paths.iter().any(|path| {
path.to_str()
.is_none_or(|value| value.contains(['\n', '\r', '"']))
}) {
return Err(failure("benchmark composition path is not TOML-safe"));
}
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = [\"/tooling\", \"/fixtures\"]\nenv = {{ PATH = \"env:BENCHMARK_PATH\", CARGO_HOME = \"env:BENCHMARK_CARGO_HOME\", RUSTUP_HOME = \"env:BENCHMARK_RUSTUP_HOME\", CARGO_BUILD_JOBS = \"env:BENCHMARK_CARGO_BUILD_JOBS\" }}\nvolumes = [\n {{ host = \"{}\", container = \"/tooling/usr\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/host\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/output\", kind = \"directory\", access = \"read-write\" }},\n {{ host = \"{}\", container = \"/fixtures/repository\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/cargo\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/rustup\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/uv-bin\", kind = \"directory\", access = \"read-only\" }},\n]\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
root.join("data").display(),
Path::new("/usr").display(),
host_root.display(),
output.display(),
repository.display(),
cargo_home.display(),
rustup_home.display(),
uv_bin.display(),
),
)?;
let environment = BTreeMap::from([
(
"BENCHMARK_PATH".into(),
"/tooling/cargo/bin:/tooling/uv-bin:/usr/bin:/bin".into(),
),
("BENCHMARK_CARGO_HOME".into(), "/tooling/cargo".into()),
("BENCHMARK_RUSTUP_HOME".into(), "/tooling/rustup".into()),
("BENCHMARK_CARGO_BUILD_JOBS".into(), "2".into()),
]);
let configuration = load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace: workspace.to_path_buf(),
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment,
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let results = output.to_path_buf();
stage_benchmark_rootfs_and_control(&runtime_domain)?;
let mut guest_arguments = vec![
BENCHMARK_GUEST.to_owned(),
"/fixtures/output".to_owned(),
"/usr/bin/python3".to_owned(),
"/fixtures/repository/tools/ignored_workspace_benchmark.py".to_owned(),
"sample".to_owned(),
];
guest_arguments.extend(
sample_arguments
.iter()
.map(|argument| argument.to_string_lossy().into_owned()),
);
let root_process =
RootProcess::new("/bin/runtime-composition", guest_arguments).map_err(failure)?;
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
let bundle = build_sandbox_bundle(SandboxBuildRequest {
configuration: &configuration,
container_id: ContainerId::new([6; 32]),
expected_network_namespace: namespace,
root_process,
terminal: None,
prepared_root: runtime_domain.join("prepared-root"),
})
.map_err(failure)?;
Self::finish_build(
None,
configuration,
bundle,
workspace.join("activity"),
Some(results),
)
}
fn build(root: &Path, tree: Option<TempDir>) -> Result<Self, MatrixError> {
let configuration_directory = root.join("configuration");
let workspace = root.join("workspace");
let runtime = root.join("runtime");
fs::create_dir(&configuration_directory)?;
fs::create_dir(&workspace)?;
fs::create_dir(&runtime)?;
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
root.join("data").display()
),
)?;
let configuration = load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace: workspace.clone(),
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let container_id = ContainerId::new([6; 32]);
stage_rootfs_and_control(&runtime_domain, container_id)?;
let root_process = RootProcess::new(
"/bin/sh",
[
"-c".to_owned(),
"while :; do echo x >> /workspace/activity; /bin/sleep 0.02; done".to_owned(),
],
)
.map_err(failure)?;
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
let bundle = build_sandbox_bundle(SandboxBuildRequest {
configuration: &configuration,
container_id,
expected_network_namespace: namespace,
root_process,
terminal: None,
prepared_root: runtime_domain.join("prepared-root"),
})
.map_err(failure)?;
Self::finish_build(
tree,
configuration,
bundle,
workspace.join("activity"),
None,
)
}
fn finish_build(
tree: Option<TempDir>,
configuration: ResolvedDomainConfiguration,
bundle: SandboxBundle,
activity: PathBuf,
benchmark_results: Option<PathBuf>,
) -> Result<Self, MatrixError> {
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let container_id = ContainerId::new([6; 32]);
let bundle_path = runtime_domain
.join("bundles")
.join(hex(container_id.as_bytes()));
fs::create_dir_all(&bundle_path)?;
fs::write(bundle_path.join("config.json"), bundle.specification_json())?;
fs::create_dir_all(runtime_domain.join("runsc"))?;
let domain = Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
};
let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label());
let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?;
let store = Arc::new(
HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain)
.map_err(failure)?,
);
store
.apply(&[DomainStoreMutation::PutGeneration(Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(7),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([3; 32]),
network_sidecar_id: NetworkSidecarId::new([4; 32]),
sandbox_id: SandboxId::new([5; 32]),
durable_decision: DurableDecision::None,
})])
.map_err(failure)?;
SandboxCoordinator::new(&store)
.begin_intent(
GenerationId::new(7),
container_id,
&runtime_domain.join("runsc"),
current_mount_identity(&runtime_domain.join("runsc"))?,
BundleDigest::new([7; 32]),
current_process_identity()?,
)
.map_err(failure)?;
Ok(Self {
_tree: tree,
configuration,
store,
bundle,
activity,
benchmark_results,
})
}
pub(super) fn launch_pending(&self) -> Result<AwaitingAttachedRunsc, MatrixError> {
self.launch_pending_with_command(self.bundle.command())
}
pub(super) fn launch_pending_with_command(
&self,
command: &RunscCommand,
) -> Result<AwaitingAttachedRunsc, MatrixError> {
let launch = RuntimeSupervisorLaunch::from_runsc_command(
ProcessOwnershipContext {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(7),
coordinator_id: CoordinatorId::new([3; 32]),
slot: lazaret_core::records::HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
},
command,
)?;
AwaitingAttachedRunsc::launch(&SandboxCoordinator::new(&self.store), &launch)
.map_err(failure)
}
pub(super) fn alternate_bundle(&self) -> Result<SandboxBundle, MatrixError> {
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
build_sandbox_bundle(SandboxBuildRequest {
configuration: &self.configuration,
container_id: ContainerId::new([6; 32]),
expected_network_namespace: namespace,
root_process: RootProcess::new(
"/bin/sh",
[
"-c".to_owned(),
"echo cross-bundle >> /workspace/activity".to_owned(),
],
)
.map_err(failure)?,
terminal: None,
prepared_root: host_path(self.configuration.runtime_domain_root().canonical_label())
.join("prepared-root"),
})
.map_err(failure)
}
pub(super) fn launch_committed(&self) -> Result<CommittedSandboxRuntime, MatrixError> {
let handoff = self
.bundle
.capture_handoff_authority(&self.configuration)
.map_err(failure)?;
let mut pending = self.launch_pending()?;
let staged = pending
.receive_staged(&SandboxCoordinator::new(&self.store))
.map_err(failure)?;
pending
.acknowledge(staged, &handoff)
.map_err(failure)?
.commit(&SandboxCoordinator::new(&self.store))
.map_err(failure)
}
}
pub(super) struct NetworkFixture {
_tree: TempDir,
_host_loopback: TcpListener,
_host_abstract: UnixListener,
pub(super) configuration: ResolvedDomainConfiguration,
pub(super) store: HostCoordinatorDomainStore,
pub(super) bundle: SandboxBundle,
pub(super) policy: lazaret_core::network::CanonicalDerivedNetworkPolicy,
pub(super) probe_result: PathBuf,
}
impl NetworkFixture {
pub(super) fn new() -> Result<Self, MatrixError> {
let tree = tempfile::tempdir()?;
let root = tree.path();
let workspace = root.join("workspace");
let runtime = root.join("runtime");
let configuration = network_configuration(root, &workspace, &runtime)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let container_id = ContainerId::new([16; 32]);
stage_rootfs_and_control(&runtime_domain, container_id)?;
let rootfs = runtime_domain.join("prepared-root");
fs::copy(
std::env::current_exe()?,
rootfs.join("bin/lazaret-network-probe"),
)?;
fs::write(
runtime_domain.join("state-anchor/runtime-files/resolv.conf"),
b"nameserver 10.0.2.3\n",
)?;
let host_loopback = TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?;
let host_loopback_port = host_loopback.local_addr()?.port();
let abstract_name = format!("lazaret-network-v3-{}", std::process::id()).into_bytes();
let host_abstract =
UnixListener::bind_addr(&SocketAddr::from_abstract_name(&abstract_name)?)?;
let root_process = RootProcess::new(
"/bin/lazaret-network-probe",
vec![
SANDBOX_NETWORK_PROBE.into(),
host_loopback_port.to_string(),
String::from_utf8(abstract_name.clone()).map_err(failure)?,
],
)
.map_err(failure)?;
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
let bundle = build_sandbox_bundle(SandboxBuildRequest {
configuration: &configuration,
container_id,
expected_network_namespace: namespace,
root_process,
terminal: None,
prepared_root: runtime_domain.join("prepared-root"),
})
.map_err(failure)?;
let bundle_path = runtime_domain
.join("bundles")
.join(hex(container_id.as_bytes()));
fs::create_dir_all(&bundle_path)?;
fs::write(bundle_path.join("config.json"), bundle.specification_json())?;
fs::create_dir_all(runtime_domain.join("runsc"))?;
let store = network_store(&configuration, &runtime_domain, container_id)?;
let policy =
derive_ipv4_policy(&configuration, [IpAddress::V4([10, 0, 2, 3])]).map_err(failure)?;
Ok(Self {
_tree: tree,
_host_loopback: host_loopback,
_host_abstract: host_abstract,
configuration,
store,
bundle,
policy,
probe_result: workspace.join("network-probe"),
})
}
pub(super) fn launch(&self) -> Result<CommittedSandboxRuntime, MatrixError> {
self.launch_with_permitted([1, 1, 1, 1], 443)
}
pub(super) fn launch_with_permitted(
&self,
permitted_address: [u8; 4],
permitted_port: u16,
) -> Result<CommittedSandboxRuntime, MatrixError> {
let handoff = self
.bundle
.capture_handoff_authority(&self.configuration)
.map_err(failure)?;
let launch = RuntimeSupervisorLaunch::from_networked_runsc_command(
ProcessOwnershipContext {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(7),
coordinator_id: CoordinatorId::new([13; 32]),
slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
},
self.bundle.command(),
RuntimeNetworkLaunch {
policy: &self.policy,
configuration_fingerprint: self.configuration.configuration_fingerprint(),
network_sidecar_id: NetworkSidecarId::new([14; 32]),
},
)?;
let sandbox = SandboxCoordinator::new(&self.store);
let network = NetworkCoordinator::new(&self.store);
let mut pending = AwaitingAttachedRunsc::launch_networked(&sandbox, &network, &launch)
.map_err(failure)?;
pending.receive_helper_staged(&network).map_err(failure)?;
pending
.receive_network_readiness(
&network,
&self.policy,
|namespace| NetworkProbeRequest {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(7),
enforcement_namespace: namespace,
denied: vec![TcpProbeTarget {
address: IpAddress::V4([10, 1, 2, 3]),
port: 443,
}],
permitted: vec![TcpProbeTarget {
address: IpAddress::V4(permitted_address),
port: permitted_port,
}],
dns: vec![DnsProbeTarget {
resolver: IpAddress::V4([10, 0, 2, 3]),
query_name: "example.com".into(),
}],
},
&ReachabilityExpectation {
denied_destinations: vec![IpAddress::V4([10, 1, 2, 3])],
permitted_destinations: vec![IpAddress::V4(permitted_address)],
dns_queries: vec![DnsQueryExpectation {
resolver: IpAddress::V4([10, 0, 2, 3]),
query_name: "example.com".into(),
}],
},
)
.map_err(failure)?;
let staged = pending.receive_staged(&sandbox).map_err(failure)?;
let (runtime, _) = pending
.acknowledge(staged, &handoff)
.map_err(failure)?
.commit_networked(&sandbox)
.map_err(failure)?;
Ok(runtime)
}
}
fn network_configuration(
root: &Path,
workspace: &Path,
runtime: &Path,
) -> Result<ResolvedDomainConfiguration, MatrixError> {
let configuration_directory = root.join("configuration");
fs::create_dir(&configuration_directory)?;
fs::create_dir(workspace)?;
fs::create_dir(runtime)?;
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/network-fixture:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
root.join("data").display()
),
)?;
load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace: workspace.to_path_buf(),
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime.to_path_buf(),
environment: BTreeMap::new(),
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)
}
fn network_store(
configuration: &ResolvedDomainConfiguration,
runtime_domain: &Path,
container_id: ContainerId,
) -> Result<HostCoordinatorDomainStore, MatrixError> {
let domain = Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
};
let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label());
let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?;
let store =
HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain)
.map_err(failure)?;
store
.apply(&[DomainStoreMutation::PutGeneration(Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(7),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([13; 32]),
network_sidecar_id: NetworkSidecarId::new([14; 32]),
sandbox_id: SandboxId::new([15; 32]),
durable_decision: DurableDecision::None,
})])
.map_err(failure)?;
SandboxCoordinator::new(&store)
.begin_intent(
GenerationId::new(7),
container_id,
&runtime_domain.join("runsc"),
current_mount_identity(&runtime_domain.join("runsc"))?,
BundleDigest::new([17; 32]),
current_process_identity()?,
)
.map_err(failure)?;
Ok(store)
}