lazaret/crates/lazaret-host-coordinator/tests/runtime-composition/fixtures.rs

630 lines
26 KiB
Rust
Raw Normal View History

use super::{
Arc, AwaitingAttachedRunsc, BENCHMARK_GUEST, BTreeMap, BundleDigest, CommittedSandboxRuntime,
ConfigurationLoadRequest, ContainerId, CoordinatorId, Domain, DomainStoreMutation,
DurableDecision, File, Generation, GenerationId, GitRoutingCoordinator,
HostCoordinatorDomainStore, HostPath, HostProcessOwnershipSlot, Ipv4Addr, MatrixError,
NETWORK_NAMESPACE_PATH, NetworkCoordinator, NetworkSidecarId, Path, PathBuf,
ProcessOwnershipContext, ResolvedDomainConfiguration, RootProcess, RunscBuildId, RunscCommand,
RunscDigest, RuntimeGitRoutingAuthority, RuntimeGitRoutingLaunch, RuntimeNetworkLaunch,
RuntimeSupervisorLaunch, SandboxBuildRequest, SandboxBundle, SandboxCoordinator, SandboxId,
SandboxIntent, SocketAddr, SocketAddrExt, TargetCommand, TcpListener, TempDir, UnixListener,
UserNamespaceMap, build_sandbox_bundle, build_sandbox_bundle_with_workspace, child_broker,
current_mount_identity, current_process_identity, default_sandbox_layout, derive_ipv4_policy,
failure, fs, hex, host_path, load_trusted_configuration, lock_file,
stage_benchmark_rootfs_and_control, stage_rootfs_and_control, try_exclusive_flock,
};
pub(super) struct Fixture {
_tree: Option<TempDir>,
pub(super) configuration: ResolvedDomainConfiguration,
pub(super) store: Arc<HostCoordinatorDomainStore>,
pub(super) bundle: SandboxBundle,
pub(super) activity: PathBuf,
pub(super) benchmark_results: Option<PathBuf>,
git_routing: Option<GitRoutingFixture>,
}
impl Fixture {
pub(super) fn new() -> Result<Self, MatrixError> {
let tree = tempfile::tempdir()?;
let root = tree.path().to_path_buf();
Self::build(&root, Some(tree), false)
}
pub(super) fn new_git_routed() -> Result<Self, MatrixError> {
let tree = tempfile::tempdir()?;
let root = tree.path().to_path_buf();
Self::build(&root, Some(tree), true)
}
pub(super) fn in_directory(root: &Path) -> Result<Self, MatrixError> {
Self::build(root, None, false)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn build_benchmark(
root: &Path,
workspace: &Path,
host_root: &Path,
output: &Path,
repository: &Path,
cargo_home: &Path,
rustup_home: &Path,
uv_bin: &Path,
sample_arguments: &[std::ffi::OsString],
) -> Result<Self, MatrixError> {
let configuration_directory = root.join("configuration");
let runtime = root.join("runtime");
for directory in [&configuration_directory, &runtime] {
fs::create_dir_all(directory)?;
}
let configuration_path = configuration_directory.join("lazaret.toml");
let paths = [
root,
workspace,
host_root,
output,
repository,
cargo_home,
rustup_home,
uv_bin,
];
if paths.iter().any(|path| {
path.to_str()
.is_none_or(|value| value.contains(['\n', '\r', '"']))
}) {
return Err(failure("benchmark composition path is not TOML-safe"));
}
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = [\"/tooling\", \"/fixtures\"]\nenv = {{ PATH = \"env:BENCHMARK_PATH\", CARGO_HOME = \"env:BENCHMARK_CARGO_HOME\", RUSTUP_HOME = \"env:BENCHMARK_RUSTUP_HOME\", CARGO_BUILD_JOBS = \"env:BENCHMARK_CARGO_BUILD_JOBS\" }}\nvolumes = [\n {{ host = \"{}\", container = \"/tooling/usr\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/host\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/output\", kind = \"directory\", access = \"read-write\" }},\n {{ host = \"{}\", container = \"/fixtures/repository\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/cargo\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/rustup\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/uv-bin\", kind = \"directory\", access = \"read-only\" }},\n]\n
",
root.join("data").display(),
Path::new("/usr").display(),
host_root.display(),
output.display(),
repository.display(),
cargo_home.display(),
rustup_home.display(),
uv_bin.display(),
),
)?;
let environment = BTreeMap::from([
(
"BENCHMARK_PATH".into(),
"/tooling/cargo/bin:/tooling/uv-bin:/usr/bin:/bin".into(),
),
("BENCHMARK_CARGO_HOME".into(), "/tooling/cargo".into()),
("BENCHMARK_RUSTUP_HOME".into(), "/tooling/rustup".into()),
("BENCHMARK_CARGO_BUILD_JOBS".into(), "2".into()),
]);
let configuration = load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace: workspace.to_path_buf(),
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment,
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let results = output.to_path_buf();
stage_benchmark_rootfs_and_control(&runtime_domain)?;
let mut guest_arguments = vec![
BENCHMARK_GUEST.to_owned(),
"/fixtures/output".to_owned(),
"/usr/bin/python3".to_owned(),
"/fixtures/repository/tools/ignored_workspace_benchmark.py".to_owned(),
"sample".to_owned(),
];
guest_arguments.extend(
sample_arguments
.iter()
.map(|argument| argument.to_string_lossy().into_owned()),
);
let root_process =
RootProcess::new("/bin/runtime-composition", guest_arguments).map_err(failure)?;
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
let bundle = build_sandbox_bundle(SandboxBuildRequest {
configuration: &configuration,
runsc_program: Path::new("/usr/local/bin/runsc"),
container_id: ContainerId::new([6; 32]),
expected_network_namespace: namespace,
root_process,
terminal: None,
prepared_root: runtime_domain.join("prepared-root"),
})
.map_err(failure)?;
Self::finish_build(
None,
configuration,
bundle,
workspace.join("activity"),
Some(results),
None,
)
}
fn build(
root: &Path,
tree: Option<TempDir>,
route_git_workspace: bool,
) -> Result<Self, MatrixError> {
let configuration_directory = root.join("configuration");
let workspace = root.join("workspace");
let runtime = root.join("runtime");
fs::create_dir(&configuration_directory)?;
fs::create_dir(&workspace)?;
fs::create_dir(&runtime)?;
let git_routing = route_git_workspace
.then(|| GitRoutingFixture::create(root, &workspace))
.transpose()?;
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = []\nenv = {{ PROFILE_SENTINEL = \"env:HOST_PROFILE_SENTINEL\" }}\n
",
root.join("data").display()
),
)?;
let configuration = load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace: workspace.clone(),
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::from([("HOST_PROFILE_SENTINEL".into(), "profile-value".into())]),
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let container_id = ContainerId::new([6; 32]);
stage_rootfs_and_control(&runtime_domain, container_id)?;
let root_process = RootProcess::new(
"/bin/sh",
[
"-c".to_owned(),
"while :; do echo x >> /workspace/activity; /bin/sleep 0.02; done".to_owned(),
],
)
.map_err(failure)?;
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
let request = SandboxBuildRequest {
configuration: &configuration,
runsc_program: Path::new("/usr/local/bin/runsc"),
container_id,
expected_network_namespace: namespace,
root_process,
terminal: None,
prepared_root: runtime_domain.join("prepared-root"),
};
let bundle = match &git_routing {
Some(git_routing) => {
build_sandbox_bundle_with_workspace(request, git_routing.mountpoint.clone())
}
None => build_sandbox_bundle(request),
}
.map_err(failure)?;
Self::finish_build(
tree,
configuration,
bundle,
workspace.join("activity"),
None,
git_routing,
)
}
fn finish_build(
tree: Option<TempDir>,
configuration: ResolvedDomainConfiguration,
bundle: SandboxBundle,
activity: PathBuf,
benchmark_results: Option<PathBuf>,
git_routing: Option<GitRoutingFixture>,
) -> Result<Self, MatrixError> {
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let container_id = ContainerId::new([6; 32]);
let bundle_path = runtime_domain
.join("bundles")
.join(hex(container_id.as_bytes()));
fs::create_dir_all(&bundle_path)?;
fs::write(bundle_path.join("config.json"), bundle.specification_json())?;
fs::create_dir_all(runtime_domain.join("runsc"))?;
let domain = Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
};
let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label());
let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?;
let store = Arc::new(
HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain)
.map_err(failure)?,
);
store
.apply(&[DomainStoreMutation::PutGeneration(Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(7),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([3; 32]),
network_sidecar_id: NetworkSidecarId::new([4; 32]),
sandbox_id: SandboxId::new([5; 32]),
durable_decision: DurableDecision::None,
})])
.map_err(failure)?;
SandboxCoordinator::new(&store)
.begin_intent(SandboxIntent {
generation_id: GenerationId::new(7),
container_id,
runtime_root: &runtime_domain.join("runsc"),
runtime_root_identity: current_mount_identity(&runtime_domain.join("runsc"))?,
bundle_digest: BundleDigest::new([7; 32]),
runsc_build_id: RunscBuildId::new([8; 32]),
runsc_digest: RunscDigest::new([8; 32]),
host_coordinator: current_process_identity()?,
})
.map_err(failure)?;
Ok(Self {
_tree: tree,
configuration,
store,
bundle,
activity,
benchmark_results,
git_routing,
})
}
pub(super) fn launch_pending(&self) -> Result<AwaitingAttachedRunsc, MatrixError> {
self.launch_pending_with_command(self.bundle.command())
}
pub(super) fn launch_pending_with_command(
&self,
command: &RunscCommand,
) -> Result<AwaitingAttachedRunsc, MatrixError> {
let mut launch = RuntimeSupervisorLaunch::from_runsc_command(
ProcessOwnershipContext {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(7),
coordinator_id: CoordinatorId::new([3; 32]),
slot: lazaret_core::records::HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
},
command,
)?;
if let Some(git_routing) = &self.git_routing {
launch = launch.with_git_routing(git_routing.launch()?)?;
}
launch = launch
.with_user_namespace_map(UserNamespaceMap::resolve_current().map_err(failure)?)?;
let mut pending =
AwaitingAttachedRunsc::launch(&SandboxCoordinator::new(&self.store), &launch)
.map_err(failure)?;
if let Some(git_routing) = &self.git_routing {
let coordinator = GitRoutingCoordinator::new(&self.store);
coordinator
.begin_intent(
GenerationId::new(7),
HostPath::new(
git_routing
.mountpoint
.as_os_str()
.as_encoded_bytes()
.to_vec(),
),
HostPath::new(
git_routing
.workspace
.as_os_str()
.as_encoded_bytes()
.to_vec(),
),
HostPath::new(git_routing.isolated.as_os_str().as_encoded_bytes().to_vec()),
)
.map_err(failure)?;
pending.start_git_routing(&coordinator).map_err(failure)?;
}
Ok(pending)
}
pub(super) fn alternate_bundle(&self) -> Result<SandboxBundle, MatrixError> {
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
build_sandbox_bundle(SandboxBuildRequest {
configuration: &self.configuration,
runsc_program: Path::new("/usr/local/bin/runsc"),
container_id: ContainerId::new([6; 32]),
expected_network_namespace: namespace,
root_process: RootProcess::new(
"/bin/sh",
[
"-c".to_owned(),
"echo cross-bundle >> /workspace/activity".to_owned(),
],
)
.map_err(failure)?,
terminal: None,
prepared_root: host_path(self.configuration.runtime_domain_root().canonical_label())
.join("prepared-root"),
})
.map_err(failure)
}
pub(super) fn launch_committed(&self) -> Result<CommittedSandboxRuntime, MatrixError> {
let handoff = self
.bundle
.capture_handoff_authority(&self.configuration)
.map_err(failure)?;
let mut pending = self.launch_pending()?;
let staged = pending
.receive_staged(&SandboxCoordinator::new(&self.store))
.map_err(failure)?;
pending
.acknowledge(staged, &handoff)
.map_err(failure)?
.commit(&SandboxCoordinator::new(&self.store))
.map_err(failure)
}
}
struct GitRoutingFixture {
workspace: PathBuf,
isolated: PathBuf,
mountpoint: PathBuf,
git_common: PathBuf,
transient_hooks: PathBuf,
transient_config: PathBuf,
}
impl GitRoutingFixture {
fn create(root: &Path, workspace: &Path) -> Result<Self, MatrixError> {
child_broker()
.run_transient(
&TargetCommand::new("git")
.argument("init")
.argument("--quiet")
.argument("--initial-branch=main")
.argument(workspace),
)
.map_err(failure)?;
let routing = root.join("git-routing");
let isolated = routing.join("isolated");
let mountpoint = routing.join("workspace");
let transient_hooks = routing.join("hooks");
let transient_config = routing.join("config");
for directory in [&isolated, &mountpoint, &transient_hooks] {
fs::create_dir_all(directory)?;
}
fs::write(&transient_config, b"")?;
Ok(Self {
workspace: workspace.to_path_buf(),
isolated,
mountpoint,
git_common: workspace.join(".git"),
transient_hooks,
transient_config,
})
}
fn launch(&self) -> Result<RuntimeGitRoutingLaunch, MatrixError> {
RuntimeGitRoutingLaunch::new(RuntimeGitRoutingAuthority {
workspace: File::open(&self.workspace)?,
workspace_label: self.workspace.clone(),
isolated: File::open(&self.isolated)?,
mountpoint: File::open(&self.mountpoint)?,
mountpoint_label: self.mountpoint.clone(),
git_common: File::open(&self.git_common)?,
git_common_label: self.git_common.clone(),
transient_hooks: File::open(&self.transient_hooks)?,
transient_hooks_label: self.transient_hooks.clone(),
transient_config: File::options()
.read(true)
.write(true)
.open(&self.transient_config)?,
transient_config_label: self.transient_config.clone(),
global_excludes: None,
isolate_ignored: true,
})
.map_err(failure)
}
}
pub(super) struct NetworkFixture {
_tree: TempDir,
_host_loopback: TcpListener,
_host_abstract: UnixListener,
pub(super) configuration: ResolvedDomainConfiguration,
pub(super) store: HostCoordinatorDomainStore,
pub(super) bundle: SandboxBundle,
pub(super) policy: lazaret_core::network::CanonicalDerivedNetworkPolicy,
}
impl NetworkFixture {
pub(super) fn new() -> Result<Self, MatrixError> {
let tree = tempfile::tempdir()?;
let root = tree.path();
let workspace = root.join("workspace");
let runtime = root.join("runtime");
let configuration = network_configuration(root, &workspace, &runtime)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
let container_id = ContainerId::new([16; 32]);
stage_rootfs_and_control(&runtime_domain, container_id)?;
let rootfs = runtime_domain.join("prepared-root");
fs::copy(
std::env::current_exe()?,
rootfs.join("bin/lazaret-network-probe"),
)?;
fs::write(
runtime_domain.join("state-anchor/runtime-files/resolv.conf"),
b"nameserver 10.137.54.3\n",
)?;
let host_loopback = TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?;
let host_loopback_port = host_loopback.local_addr()?.port();
let abstract_name = format!("lazaret-network-v3-{}", std::process::id()).into_bytes();
let host_abstract =
UnixListener::bind_addr(&SocketAddr::from_abstract_name(&abstract_name)?)?;
let _ = (host_loopback_port, abstract_name);
let root_process = RootProcess::new(
lazaret_runtime_supervisor::sandbox::SANDBOX_INIT_DESTINATION,
vec![lazaret_host_coordinator::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE.into()],
)
.map_err(failure)?;
let namespace =
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
let bundle = build_sandbox_bundle(SandboxBuildRequest {
configuration: &configuration,
runsc_program: Path::new("/usr/local/bin/runsc"),
container_id,
expected_network_namespace: namespace,
root_process,
terminal: None,
prepared_root: runtime_domain.join("prepared-root"),
})
.map_err(failure)?;
let bundle_path = runtime_domain
.join("bundles")
.join(hex(container_id.as_bytes()));
fs::create_dir_all(&bundle_path)?;
fs::write(bundle_path.join("config.json"), bundle.specification_json())?;
fs::create_dir_all(runtime_domain.join("runsc"))?;
let store = network_store(&configuration, &runtime_domain, container_id)?;
let policy = derive_ipv4_policy(&configuration).map_err(failure)?;
Ok(Self {
_tree: tree,
_host_loopback: host_loopback,
_host_abstract: host_abstract,
configuration,
store,
bundle,
policy,
})
}
pub(super) fn launch(&self) -> Result<CommittedSandboxRuntime, MatrixError> {
self.launch_with_permitted([1, 1, 1, 1], 443)
}
pub(super) fn launch_with_permitted(
&self,
_permitted_address: [u8; 4],
_permitted_port: u16,
) -> Result<CommittedSandboxRuntime, MatrixError> {
let handoff = self
.bundle
.capture_handoff_authority(&self.configuration)
.map_err(failure)?;
let launch = RuntimeSupervisorLaunch::from_networked_runsc_command(
ProcessOwnershipContext {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(7),
coordinator_id: CoordinatorId::new([13; 32]),
slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
},
self.bundle.command(),
RuntimeNetworkLaunch {
policy: &self.policy,
configuration_fingerprint: self.configuration.configuration_fingerprint(),
network_sidecar_id: NetworkSidecarId::new([14; 32]),
},
)?
.with_user_namespace_map(UserNamespaceMap::resolve_current().map_err(failure)?)?;
let sandbox = SandboxCoordinator::new(&self.store);
let network = NetworkCoordinator::new(&self.store);
let mut pending = AwaitingAttachedRunsc::launch_networked(&sandbox, &network, &launch)
.map_err(failure)?;
pending.receive_helper_staged(&network).map_err(failure)?;
pending
.receive_network_attestation(&network, &self.policy)
.map_err(failure)?;
let staged = pending.receive_staged(&sandbox).map_err(failure)?;
let (runtime, _) = pending
.acknowledge(staged, &handoff)
.map_err(failure)?
.commit_networked(&sandbox, &network, &self.policy)
.map_err(failure)?;
Ok(runtime)
}
}
fn network_configuration(
root: &Path,
workspace: &Path,
runtime: &Path,
) -> Result<ResolvedDomainConfiguration, MatrixError> {
let configuration_directory = root.join("configuration");
fs::create_dir(&configuration_directory)?;
fs::create_dir(workspace)?;
fs::create_dir(runtime)?;
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/network-fixture:v1\"\npersistent_dirs = []\n
",
root.join("data").display()
),
)?;
load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace: workspace.to_path_buf(),
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime.to_path_buf(),
environment: BTreeMap::new(),
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)
}
fn network_store(
configuration: &ResolvedDomainConfiguration,
runtime_domain: &Path,
container_id: ContainerId,
) -> Result<HostCoordinatorDomainStore, MatrixError> {
let domain = Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
};
let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label());
let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?;
let store =
HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain)
.map_err(failure)?;
store
.apply(&[DomainStoreMutation::PutGeneration(Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(7),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([13; 32]),
network_sidecar_id: NetworkSidecarId::new([14; 32]),
sandbox_id: SandboxId::new([15; 32]),
durable_decision: DurableDecision::None,
})])
.map_err(failure)?;
SandboxCoordinator::new(&store)
.begin_intent(SandboxIntent {
generation_id: GenerationId::new(7),
container_id,
runtime_root: &runtime_domain.join("runsc"),
runtime_root_identity: current_mount_identity(&runtime_domain.join("runsc"))?,
bundle_digest: BundleDigest::new([17; 32]),
runsc_build_id: RunscBuildId::new([18; 32]),
runsc_digest: RunscDigest::new([18; 32]),
host_coordinator: current_process_identity()?,
})
.map_err(failure)?;
Ok(store)
}