Replace external reachability with an owned canary

This commit is contained in:
ms2k 2026-07-19 22:39:31 +09:00
commit 31e9ab0f2d
7 changed files with 1409 additions and 1972 deletions

View file

@ -10,16 +10,13 @@ use lazaret_core::identity::{
ContainerId, CoordinatorId, DeviceId, GenerationId, HostPath, IdMapDigest, InodeId, PolicyId,
PreparedArtifactId, SandboxId, SelectedManifestDigest, SourceIdentity,
};
use lazaret_core::records::{
ArtifactState, Domain, DurableDecision, Generation, IpAddress, PreparedArtifact,
};
use lazaret_core::records::{ArtifactState, Domain, DurableDecision, Generation, PreparedArtifact};
use lazaret_host_coordinator::cold_start::{ColdStart, ColdStartInputs};
use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore;
use lazaret_linux::filesystem::{namespace_identity, try_exclusive_flock};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_linux::user_namespace::{
REQUIRED_SUBORDINATE_IDS, UserNamespaceIdRange, UserNamespaceMap,
};
use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::{RootProcess, default_sandbox_layout};
use lazaret_storage_catalog::Catalog;
use rustix::process::{getgid, getuid};
@ -202,20 +199,6 @@ impl ColdStartFixture {
),
root_process: RootProcess::new("/bin/sh", ["-c".into(), "sleep infinity".into()])
.unwrap(),
resolver_addresses: vec![IpAddress::V4([10, 0, 2, 3])],
network_probe: NetworkProbeRequest {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(1),
enforcement_namespace: namespace_identity(Path::new("/proc/self/ns/net")).unwrap(),
denied: Vec::new(),
permitted: Vec::new(),
dns: Vec::new(),
},
reachability: lazaret_core::network::ReachabilityExpectation {
denied_destinations: Vec::new(),
permitted_destinations: Vec::new(),
dns_queries: Vec::new(),
},
container_id: self.container_id,
invoking_uid: getuid().as_raw(),
escalation_client,
@ -303,7 +286,8 @@ fn configuration(root: &Path) -> lazaret_core::configuration::ResolvedDomainConf
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\npersistent_dirs = []\n
",
root.join("data").display()
),
)

View file

@ -217,9 +217,9 @@ fn start_sandbox(
lifecycle.mount_state_anchor().unwrap();
lifecycle.start_sidecar().unwrap();
lifecycle.start_connectivity_helper().unwrap();
lifecycle.mark_network_ready().unwrap();
lifecycle.arm_runtime_causal_edge().unwrap();
lifecycle.start_sandbox().unwrap();
lifecycle.mark_network_ready().unwrap();
}
fn publish(

View file

@ -249,9 +249,9 @@ fn complete_startup(
lifecycle.mount_state_anchor().unwrap();
lifecycle.start_sidecar().unwrap();
lifecycle.start_connectivity_helper().unwrap();
lifecycle.mark_network_ready().unwrap();
lifecycle.arm_runtime_causal_edge().unwrap();
lifecycle.start_sandbox().unwrap();
lifecycle.mark_network_ready().unwrap();
lifecycle.publish_domain().unwrap();
}
@ -422,9 +422,9 @@ fn exact_start_join_disconnect_and_teardown_actions_are_ordered() {
LifecycleAction::MountStateAnchor,
LifecycleAction::StartSidecar,
LifecycleAction::StartConnectivityHelper,
LifecycleAction::MarkNetworkReady,
LifecycleAction::ArmRuntimeCausalEdge,
LifecycleAction::StartSandbox,
LifecycleAction::MarkNetworkReady,
LifecycleAction::PublishDomain,
LifecycleAction::AdvanceWaitingSession,
LifecycleAction::RegisterSession,
@ -477,9 +477,9 @@ fn cold_start_entry_reaches_publication_in_exact_modeled_order() {
LifecycleAction::MountStateAnchor,
LifecycleAction::StartSidecar,
LifecycleAction::StartConnectivityHelper,
LifecycleAction::MarkNetworkReady,
LifecycleAction::ArmRuntimeCausalEdge,
LifecycleAction::StartSandbox,
LifecycleAction::MarkNetworkReady,
LifecycleAction::PublishDomain,
]
);

View file

@ -18,9 +18,9 @@ use lazaret_core::identity::{
BundleDigest, ClientId, ContainerId, CoordinatorId, GenerationId, HostPath, NetworkSidecarId,
PidfsProcessIdentityV1, SandboxId, SessionAnchorId, SessionId,
};
use lazaret_core::network::{DnsQueryExpectation, ReachabilityExpectation, derive_ipv4_policy};
use lazaret_core::network::derive_ipv4_policy;
use lazaret_core::records::{
Domain, DurableDecision, Generation, HostProcessOwnershipSlot, IpAddress, ParticipantRole,
Domain, DurableDecision, Generation, HostProcessOwnershipSlot, ParticipantRole,
ParticipantState, ProcessOwnershipCommitAck, Session, SessionPhase, SessionTerminalEvent,
SessionTerminalMessage, TerminalResize, TerminalSignal, TerminalSignalKind,
};
@ -40,7 +40,6 @@ use lazaret_linux::process::{
current_process_identity, read_start_time,
};
use lazaret_linux::user_namespace::UserNamespaceMap;
use lazaret_runtime_supervisor::network::{DnsProbeTarget, NetworkProbeRequest, TcpProbeTarget};
use lazaret_runtime_supervisor::sandbox::{
NETWORK_NAMESPACE_PATH, RootProcess, RunscCommand, SandboxBuildRequest, SandboxBundle,
build_sandbox_bundle, default_sandbox_layout,
@ -331,7 +330,9 @@ fn run_session_terminal_matrix() -> Result<(), MatrixError> {
let second_signals = workspace.join("session-second-signals");
let line = "x".repeat(1024);
let script = format!(
"trap 'echo INT >> /workspace/session-signals' INT; \
"test \"$PROFILE_SENTINEL\" = profile-value || exit 97; \
command -v sleep >/dev/null || exit 96; \
trap 'echo INT >> /workspace/session-signals' INT; \
trap 'echo QUIT >> /workspace/session-signals' QUIT; \
trap 'echo TSTP >> /workspace/session-signals' TSTP; \
trap 'echo WINCH >> /workspace/session-signals' WINCH; \
@ -356,6 +357,7 @@ fn run_session_terminal_matrix() -> Result<(), MatrixError> {
host_coordinator: current_process_identity()?,
runsc_program: command.program().to_owned(),
runsc_global_arguments: command.arguments()[..run].to_vec(),
environment: fixture.configuration.ephemeral_environment().clone(),
internal_pid_file: runtime_domain.join("session-internal.pid"),
argv: vec![b"/bin/sh".to_vec(), b"-c".to_vec(), script.into_bytes()],
};
@ -627,7 +629,7 @@ use handoff::{
runtime_supervisor_loss_stops_guest,
};
use network::{
failed_probe_blocks_sandbox_start, hex, host_path, lock_file, networked_sandbox_view_isolated,
run_sandbox_network_probe, sandbox_participant, stage_benchmark_rootfs_and_control,
stage_rootfs_and_control,
hex, host_path, lock_file, networked_sandbox_view_isolated, run_sandbox_network_probe,
sandbox_participant, stage_benchmark_rootfs_and_control, stage_rootfs_and_control,
unreachable_permitted_destination_does_not_block_start,
};

View file

@ -1,16 +1,14 @@
use super::{
Arc, AwaitingAttachedRunsc, BENCHMARK_GUEST, BTreeMap, BundleDigest, CommittedSandboxRuntime,
ConfigurationLoadRequest, ContainerId, CoordinatorId, DnsProbeTarget, DnsQueryExpectation,
Domain, DomainStoreMutation, DurableDecision, Generation, GenerationId,
HostCoordinatorDomainStore, HostProcessOwnershipSlot, IpAddress, Ipv4Addr, MatrixError,
NETWORK_NAMESPACE_PATH, NetworkCoordinator, NetworkProbeRequest, NetworkSidecarId, Path,
PathBuf, ProcessOwnershipContext, ReachabilityExpectation, ResolvedDomainConfiguration,
RootProcess, RunscCommand, RuntimeNetworkLaunch, RuntimeSupervisorLaunch,
SANDBOX_NETWORK_PROBE, SandboxBuildRequest, SandboxBundle, SandboxCoordinator, SandboxId,
SocketAddr, SocketAddrExt, TcpListener, TcpProbeTarget, TempDir, UnixListener,
UserNamespaceMap, build_sandbox_bundle, current_mount_identity, current_process_identity,
default_sandbox_layout, derive_ipv4_policy, failure, fs, hex, host_path,
load_trusted_configuration, lock_file, stage_benchmark_rootfs_and_control,
ConfigurationLoadRequest, ContainerId, CoordinatorId, Domain, DomainStoreMutation,
DurableDecision, Generation, GenerationId, HostCoordinatorDomainStore,
HostProcessOwnershipSlot, Ipv4Addr, MatrixError, NETWORK_NAMESPACE_PATH, NetworkCoordinator,
NetworkSidecarId, Path, PathBuf, ProcessOwnershipContext, ResolvedDomainConfiguration,
RootProcess, RunscCommand, RuntimeNetworkLaunch, RuntimeSupervisorLaunch, SandboxBuildRequest,
SandboxBundle, SandboxCoordinator, SandboxId, SocketAddr, SocketAddrExt, TcpListener, TempDir,
UnixListener, UserNamespaceMap, build_sandbox_bundle, current_mount_identity,
current_process_identity, default_sandbox_layout, derive_ipv4_policy, failure, fs, hex,
host_path, load_trusted_configuration, lock_file, stage_benchmark_rootfs_and_control,
stage_rootfs_and_control, try_exclusive_flock,
};
@ -71,7 +69,8 @@ impl Fixture {
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = [\"/tooling\", \"/fixtures\"]\nenv = {{ PATH = \"env:BENCHMARK_PATH\", CARGO_HOME = \"env:BENCHMARK_CARGO_HOME\", RUSTUP_HOME = \"env:BENCHMARK_RUSTUP_HOME\", CARGO_BUILD_JOBS = \"env:BENCHMARK_CARGO_BUILD_JOBS\" }}\nvolumes = [\n {{ host = \"{}\", container = \"/tooling/usr\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/host\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/output\", kind = \"directory\", access = \"read-write\" }},\n {{ host = \"{}\", container = \"/fixtures/repository\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/cargo\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/rustup\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/uv-bin\", kind = \"directory\", access = \"read-only\" }},\n]\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = [\"/tooling\", \"/fixtures\"]\nenv = {{ PATH = \"env:BENCHMARK_PATH\", CARGO_HOME = \"env:BENCHMARK_CARGO_HOME\", RUSTUP_HOME = \"env:BENCHMARK_RUSTUP_HOME\", CARGO_BUILD_JOBS = \"env:BENCHMARK_CARGO_BUILD_JOBS\" }}\nvolumes = [\n {{ host = \"{}\", container = \"/tooling/usr\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/host\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/output\", kind = \"directory\", access = \"read-write\" }},\n {{ host = \"{}\", container = \"/fixtures/repository\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/cargo\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/rustup\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/uv-bin\", kind = \"directory\", access = \"read-only\" }},\n]\n
",
root.join("data").display(),
Path::new("/usr").display(),
host_root.display(),
@ -151,7 +150,8 @@ impl Fixture {
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = []\nenv = {{ PROFILE_SENTINEL = \"env:HOST_PROFILE_SENTINEL\" }}\n
",
root.join("data").display()
),
)?;
@ -163,7 +163,7 @@ impl Fixture {
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
environment: BTreeMap::from([("HOST_PROFILE_SENTINEL".into(), "profile-value".into())]),
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)?;
@ -324,7 +324,6 @@ pub(super) struct NetworkFixture {
pub(super) store: HostCoordinatorDomainStore,
pub(super) bundle: SandboxBundle,
pub(super) policy: lazaret_core::network::CanonicalDerivedNetworkPolicy,
pub(super) probe_result: PathBuf,
}
impl NetworkFixture {
@ -351,13 +350,10 @@ impl NetworkFixture {
let abstract_name = format!("lazaret-network-v3-{}", std::process::id()).into_bytes();
let host_abstract =
UnixListener::bind_addr(&SocketAddr::from_abstract_name(&abstract_name)?)?;
let _ = (host_loopback_port, abstract_name);
let root_process = RootProcess::new(
"/bin/lazaret-network-probe",
vec![
SANDBOX_NETWORK_PROBE.into(),
host_loopback_port.to_string(),
String::from_utf8(abstract_name.clone()).map_err(failure)?,
],
lazaret_runtime_supervisor::sandbox::SANDBOX_INIT_DESTINATION,
vec![lazaret_host_coordinator::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE.into()],
)
.map_err(failure)?;
let namespace =
@ -378,8 +374,7 @@ impl NetworkFixture {
fs::write(bundle_path.join("config.json"), bundle.specification_json())?;
fs::create_dir_all(runtime_domain.join("runsc"))?;
let store = network_store(&configuration, &runtime_domain, container_id)?;
let policy =
derive_ipv4_policy(&configuration, [IpAddress::V4([10, 0, 2, 3])]).map_err(failure)?;
let policy = derive_ipv4_policy(&configuration).map_err(failure)?;
Ok(Self {
_tree: tree,
_host_loopback: host_loopback,
@ -388,7 +383,6 @@ impl NetworkFixture {
store,
bundle,
policy,
probe_result: workspace.join("network-probe"),
})
}
@ -398,8 +392,8 @@ impl NetworkFixture {
pub(super) fn launch_with_permitted(
&self,
permitted_address: [u8; 4],
permitted_port: u16,
_permitted_address: [u8; 4],
_permitted_port: u16,
) -> Result<CommittedSandboxRuntime, MatrixError> {
let handoff = self
.bundle
@ -426,41 +420,13 @@ impl NetworkFixture {
.map_err(failure)?;
pending.receive_helper_staged(&network).map_err(failure)?;
pending
.receive_network_readiness(
&network,
&self.policy,
|namespace| NetworkProbeRequest {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(7),
enforcement_namespace: namespace,
denied: vec![TcpProbeTarget {
address: IpAddress::V4([10, 1, 2, 3]),
port: 443,
}],
permitted: vec![TcpProbeTarget {
address: IpAddress::V4(permitted_address),
port: permitted_port,
}],
dns: vec![DnsProbeTarget {
resolver: IpAddress::V4([10, 0, 2, 3]),
query_name: "example.com".into(),
}],
},
&ReachabilityExpectation {
denied_destinations: vec![IpAddress::V4([10, 1, 2, 3])],
permitted_destinations: vec![IpAddress::V4(permitted_address)],
dns_queries: vec![DnsQueryExpectation {
resolver: IpAddress::V4([10, 0, 2, 3]),
query_name: "example.com".into(),
}],
},
)
.receive_network_attestation(&network, &self.policy)
.map_err(failure)?;
let staged = pending.receive_staged(&sandbox).map_err(failure)?;
let (runtime, _) = pending
.acknowledge(staged, &handoff)
.map_err(failure)?
.commit_networked(&sandbox)
.commit_networked(&sandbox, &network, &self.policy)
.map_err(failure)?;
Ok(runtime)
}
@ -479,7 +445,8 @@ fn network_configuration(
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/network-fixture:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/network-fixture:v1\"\npersistent_dirs = []\n
",
root.join("data").display()
),
)?;

View file

@ -3,8 +3,8 @@ use super::{
HostProcessOwnershipSlot, MatrixError, ParticipantState, Path, PathBuf, PidfsProcessIdentityV1,
PlatformObservation, ProcessOwnershipCommitAck, ProcessOwnershipContext, RetainedProcess,
SandboxCoordinator, Signal, TargetCommand, UnixListener, check_platform, child_broker,
current_process_identity, failed_probe_blocks_sandbox_start, fs, host_path,
networked_sandbox_view_isolated, read_start_time, sandbox_participant, thread,
current_process_identity, fs, host_path, networked_sandbox_view_isolated, read_start_time,
sandbox_participant, thread, unreachable_permitted_destination_does_not_block_start,
};
pub(super) fn run_handoff_matrix() -> Result<(), MatrixError> {
@ -14,7 +14,7 @@ pub(super) fn run_handoff_matrix() -> Result<(), MatrixError> {
}
pub(super) fn run_network_cases() -> Result<(), MatrixError> {
failed_probe_blocks_sandbox_start()?;
unreachable_permitted_destination_does_not_block_start()?;
networked_sandbox_view_isolated()
}

View file

@ -5,46 +5,33 @@ use super::{
UnixStream, failure, fs, symlink, thread,
};
pub(super) fn failed_probe_blocks_sandbox_start() -> Result<(), MatrixError> {
pub(super) fn unreachable_permitted_destination_does_not_block_start() -> Result<(), MatrixError> {
let fixture = NetworkFixture::new()?;
if let Ok(runtime) = fixture.launch_with_permitted([203, 0, 113, 1], 9) {
runtime
.stop(&SandboxCoordinator::new(&fixture.store))
.map_err(failure)?;
return Err(failure("failed physical probe started the sandbox"));
}
let sandbox = sandbox_participant(&fixture.store);
if sandbox.state() != ParticipantState::Intent || fixture.probe_result.exists() {
Err(failure(
"failed physical probe advanced Sandbox state or guest work",
))
} else {
Ok(())
let runtime = fixture.launch_with_permitted([203, 0, 113, 1], 9)?;
let network = fixture
.store
.snapshot()
.map_err(failure)?
.participants
.into_iter()
.find(|participant| participant.role() == ParticipantRole::Network)
.ok_or_else(|| failure("network participant is absent"))?;
if network.state() != ParticipantState::Committed {
return Err(failure(
"unreachable configured destination prevented owned readiness",
));
}
runtime
.stop(&SandboxCoordinator::new(&fixture.store))
.map_err(failure)
}
pub(super) fn networked_sandbox_view_isolated() -> Result<(), MatrixError> {
let fixture = NetworkFixture::new()?;
let runtime = fixture.launch()?;
for _ in 0..6000 {
if fixture.probe_result.is_file() {
break;
}
thread::sleep(Duration::from_millis(5));
}
let result = fs::read_to_string(&fixture.probe_result)?;
for required in [
"denied=true",
"public=true",
"dns=true",
"host_loopback=false",
"abstract_socket=false",
] {
if !result.lines().any(|line| line == required) {
return Err(failure(format!(
"guest network assertion missing: {required}"
)));
}
let sandbox = sandbox_participant(&fixture.store);
if sandbox.state() != ParticipantState::Committed {
return Err(failure("owned canary did not commit sandbox readiness"));
}
runtime
.stop(&SandboxCoordinator::new(&fixture.store))
@ -152,11 +139,20 @@ pub(super) fn stage_rootfs_and_control(
let control = runtime_domain
.join("sandbox-control")
.join(hex(container_id.as_bytes()));
for directory in ["credentials", "mailbox/requests", "mailbox/results", "bin"] {
for directory in [
"credentials",
"readiness",
"mailbox/requests",
"mailbox/results",
"bin",
] {
fs::create_dir_all(control.join(directory))?;
}
fs::copy("/bin/true", control.join("bin/lazaret-escalate"))?;
fs::copy("/bin/true", control.join("bin/lazaret-sandbox-init"))?;
fs::copy(
std::env::current_exe()?,
control.join("bin/lazaret-sandbox-init"),
)?;
Ok(())
}
@ -213,7 +209,13 @@ pub(super) fn stage_benchmark_rootfs_and_control(runtime_domain: &Path) -> Resul
fs::create_dir_all(runtime_domain.join("runsc-overlay").join(hex(&[6; 32])))?;
let control = runtime_domain.join("sandbox-control").join(hex(&[6; 32]));
for directory in ["credentials", "mailbox/requests", "mailbox/results", "bin"] {
for directory in [
"credentials",
"readiness",
"mailbox/requests",
"mailbox/results",
"bin",
] {
fs::create_dir_all(control.join(directory))?;
}
fs::copy("/bin/true", control.join("bin/lazaret-escalate"))?;