558 lines
19 KiB
Rust
558 lines
19 KiB
Rust
use std::collections::BTreeMap;
|
|
use std::fs::{self, File, OpenOptions};
|
|
use std::os::unix::ffi::OsStringExt;
|
|
use std::os::unix::net::UnixListener;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::ExitCode;
|
|
use std::thread;
|
|
use std::time::Duration;
|
|
|
|
use lazaret_core::configuration::{
|
|
ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
|
|
};
|
|
use lazaret_core::identity::{
|
|
BundleDigest, ContainerId, CoordinatorId, GenerationId, HostPath, NetworkSidecarId, SandboxId,
|
|
};
|
|
use lazaret_core::records::{
|
|
Domain, DurableDecision, Generation, HostProcessOwnershipSlot, ParticipantRole,
|
|
ParticipantState, ProcessOwnershipCommitAck,
|
|
};
|
|
use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
|
use lazaret_host_coordinator::runtime::{
|
|
AwaitingAttachedRunsc, CommittedSandboxRuntime, SandboxCoordinator,
|
|
};
|
|
use lazaret_linux::filesystem::try_exclusive_flock;
|
|
use lazaret_linux::platform::{PlatformObservation, check_platform};
|
|
use lazaret_linux::process::{
|
|
ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, current_process_identity,
|
|
};
|
|
use lazaret_runtime_supervisor::sandbox::{
|
|
NETWORK_NAMESPACE_PATH, RootProcess, SandboxBuildRequest, SandboxBundle, build_sandbox_bundle,
|
|
production_sandbox_layout,
|
|
};
|
|
use lazaret_runtime_supervisor::supervision::{
|
|
AttachedRunscSignal, RuntimeSupervisorLaunch, dispatch_private_runtime_role,
|
|
};
|
|
use rustix::process::Signal;
|
|
use tempfile::TempDir;
|
|
|
|
const SKIP: u8 = 77;
|
|
const HOST_COORDINATOR_DRIVER: &str = "--runtime-composition-host-coordinator";
|
|
|
|
fn main() -> ExitCode {
|
|
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
|
|
if let Some(result) = lazaret_linux::process::dispatch_private_process_role(&arguments) {
|
|
return exit(result);
|
|
}
|
|
if let Some(result) = dispatch_private_runtime_role(&arguments) {
|
|
return exit(result);
|
|
}
|
|
if arguments.first().and_then(|argument| argument.to_str()) == Some(HOST_COORDINATOR_DRIVER) {
|
|
return exit(run_host_coordinator_driver(&arguments[1..]));
|
|
}
|
|
match run_matrix() {
|
|
Ok(()) => {
|
|
println!("PASS foreground runtime composition and causal-loss matrix");
|
|
ExitCode::SUCCESS
|
|
}
|
|
Err(MatrixError::Skip(reason)) => {
|
|
println!("SKIP foreground runtime composition: {reason}");
|
|
ExitCode::from(SKIP)
|
|
}
|
|
Err(MatrixError::Failure(error)) => {
|
|
eprintln!("runtime composition failed: {error}");
|
|
ExitCode::FAILURE
|
|
}
|
|
}
|
|
}
|
|
|
|
fn exit(result: std::io::Result<()>) -> ExitCode {
|
|
match result {
|
|
Ok(()) => ExitCode::SUCCESS,
|
|
Err(error) => {
|
|
eprintln!("private runtime role failed: {error}");
|
|
ExitCode::FAILURE
|
|
}
|
|
}
|
|
}
|
|
|
|
enum MatrixError {
|
|
Skip(String),
|
|
Failure(String),
|
|
}
|
|
|
|
impl From<std::io::Error> for MatrixError {
|
|
fn from(error: std::io::Error) -> Self {
|
|
Self::Failure(error.to_string())
|
|
}
|
|
}
|
|
|
|
fn run_matrix() -> Result<(), MatrixError> {
|
|
let release = fs::read_to_string("/proc/sys/kernel/osrelease")?;
|
|
check_platform(&PlatformObservation {
|
|
operating_system: std::env::consts::OS,
|
|
architecture: std::env::consts::ARCH,
|
|
kernel_release: release.trim(),
|
|
})
|
|
.map_err(|error| MatrixError::Skip(error.to_string()))?;
|
|
require_runtime_prerequisites()?;
|
|
lost_acknowledgment_refuses_target_work()?;
|
|
normal_stop_and_stop_resume()?;
|
|
attached_runsc_loss_stops_guest()?;
|
|
runtime_supervisor_loss_stops_guest()?;
|
|
host_coordinator_loss_stops_guest()
|
|
}
|
|
|
|
fn require_runtime_prerequisites() -> Result<(), MatrixError> {
|
|
let runsc = std::env::var_os("PATH")
|
|
.into_iter()
|
|
.flat_map(|path| std::env::split_paths(&path).collect::<Vec<_>>())
|
|
.map(|directory| directory.join("runsc"))
|
|
.find(|candidate| candidate.is_file())
|
|
.ok_or_else(|| MatrixError::Skip("runsc is not installed on PATH".into()))?;
|
|
for path in [
|
|
runsc,
|
|
PathBuf::from("/bin/sh"),
|
|
PathBuf::from("/bin/sleep"),
|
|
PathBuf::from("/lib64/libc.so.6"),
|
|
PathBuf::from("/lib64/libtinfo.so.6"),
|
|
PathBuf::from("/lib64/ld-linux-x86-64.so.2"),
|
|
] {
|
|
if !path.is_file() {
|
|
return Err(MatrixError::Skip(format!(
|
|
"required runtime fixture is absent: {}",
|
|
path.display()
|
|
)));
|
|
}
|
|
}
|
|
let socket_directory = tempfile::tempdir()?;
|
|
let socket_path = socket_directory.path().join("runsc-control.sock");
|
|
match UnixListener::bind(&socket_path) {
|
|
Ok(listener) => drop(listener),
|
|
Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => {
|
|
return Err(MatrixError::Skip(format!(
|
|
"filesystem Unix sockets required by runsc are denied: {error}"
|
|
)));
|
|
}
|
|
Err(error) => return Err(error.into()),
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn lost_acknowledgment_refuses_target_work() -> Result<(), MatrixError> {
|
|
let fixture = Fixture::new()?;
|
|
let mut pending = fixture.launch_pending()?;
|
|
let staged = pending
|
|
.receive_staged(&SandboxCoordinator::new(&fixture.store))
|
|
.map_err(failure)?;
|
|
let participant = sandbox_participant(&fixture.store);
|
|
assert_eq!(participant.state(), ParticipantState::Intent);
|
|
let lazaret_core::records::ParticipantOwnership::Sandbox(ownership) =
|
|
participant.ownership().unwrap()
|
|
else {
|
|
unreachable!();
|
|
};
|
|
assert_eq!(
|
|
ownership.attached_runsc.authoritative_pidfs().unwrap(),
|
|
&staged.identity()
|
|
);
|
|
pending.abort().map_err(failure)?;
|
|
if fixture.activity.exists() {
|
|
return Err(failure("lost acknowledgment released sandbox work"));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn normal_stop_and_stop_resume() -> Result<(), MatrixError> {
|
|
let fixture = Fixture::new()?;
|
|
let runtime = fixture.launch_committed()?;
|
|
wait_for_activity(&fixture.activity)?;
|
|
runtime
|
|
.signal_attached_runsc(AttachedRunscSignal::Stop)
|
|
.map_err(failure)?;
|
|
thread::sleep(Duration::from_millis(50));
|
|
runtime
|
|
.signal_attached_runsc(AttachedRunscSignal::Continue)
|
|
.map_err(failure)?;
|
|
runtime
|
|
.signal_runtime_supervisor(Signal::STOP)
|
|
.map_err(failure)?;
|
|
let stopped_size = fs::metadata(&fixture.activity)?.len();
|
|
thread::sleep(Duration::from_millis(50));
|
|
if fs::metadata(&fixture.activity)?.len() <= stopped_size {
|
|
return Err(failure(
|
|
"stopped Runtime Supervisor incorrectly stopped a present sandbox",
|
|
));
|
|
}
|
|
runtime
|
|
.signal_runtime_supervisor(Signal::CONT)
|
|
.map_err(failure)?;
|
|
runtime
|
|
.stop(&SandboxCoordinator::new(&fixture.store))
|
|
.map_err(failure)?;
|
|
assert_eq!(
|
|
sandbox_participant(&fixture.store).state(),
|
|
ParticipantState::Clean
|
|
);
|
|
assert_activity_stable(&fixture.activity)
|
|
}
|
|
|
|
fn attached_runsc_loss_stops_guest() -> Result<(), MatrixError> {
|
|
let fixture = Fixture::new()?;
|
|
let runtime = fixture.launch_committed()?;
|
|
wait_for_activity(&fixture.activity)?;
|
|
runtime
|
|
.signal_attached_runsc(AttachedRunscSignal::Kill)
|
|
.map_err(failure)?;
|
|
runtime.wait_for_attached_runsc_exit().map_err(failure)?;
|
|
assert_eq!(
|
|
sandbox_participant(&fixture.store).state(),
|
|
ParticipantState::Committed
|
|
);
|
|
assert_activity_stable(&fixture.activity)
|
|
}
|
|
|
|
fn runtime_supervisor_loss_stops_guest() -> Result<(), MatrixError> {
|
|
let fixture = Fixture::new()?;
|
|
let runtime = fixture.launch_committed()?;
|
|
wait_for_activity(&fixture.activity)?;
|
|
runtime
|
|
.signal_runtime_supervisor(Signal::KILL)
|
|
.map_err(failure)?;
|
|
runtime
|
|
.wait_for_runtime_supervisor_exit()
|
|
.map_err(failure)?;
|
|
assert_eq!(
|
|
sandbox_participant(&fixture.store).state(),
|
|
ParticipantState::Committed
|
|
);
|
|
assert_activity_stable(&fixture.activity)
|
|
}
|
|
|
|
fn host_coordinator_loss_stops_guest() -> Result<(), MatrixError> {
|
|
let tree = tempfile::tempdir()?;
|
|
let ready = tree.path().join("host-coordinator-ready");
|
|
let activity = tree.path().join("workspace/activity");
|
|
let parent = current_process_identity()?;
|
|
let target = TargetCommand::new(std::env::current_exe()?)
|
|
.argument(HOST_COORDINATOR_DRIVER)
|
|
.argument(tree.path())
|
|
.argument(&ready);
|
|
let mut driver = ExactChild::new(child_broker().acquire(
|
|
parent,
|
|
ProcessOwnershipContext {
|
|
domain_id: lazaret_core::identity::DomainId::new([1; 32]),
|
|
generation_id: GenerationId::new(7),
|
|
coordinator_id: CoordinatorId::new([3; 32]),
|
|
slot: HostProcessOwnershipSlot::SandboxHostCoordinator,
|
|
},
|
|
&target,
|
|
|_| Ok(()),
|
|
|staged| Ok(ProcessOwnershipCommitAck::v1(staged)),
|
|
)?);
|
|
for _ in 0..400 {
|
|
if ready.is_file() {
|
|
break;
|
|
}
|
|
if driver.process()?.has_exited()? {
|
|
driver.reap()?;
|
|
return Err(failure("Host Coordinator driver exited before readiness"));
|
|
}
|
|
thread::sleep(Duration::from_millis(5));
|
|
}
|
|
if !ready.is_file() {
|
|
driver.kill_and_reap()?;
|
|
return Err(failure("Host Coordinator driver readiness timed out"));
|
|
}
|
|
wait_for_activity(&activity)?;
|
|
driver.kill_and_reap()?;
|
|
thread::sleep(Duration::from_millis(100));
|
|
assert_activity_stable(&activity)
|
|
}
|
|
|
|
struct ExactChild(Option<RetainedProcess>);
|
|
|
|
impl ExactChild {
|
|
const fn new(process: RetainedProcess) -> Self {
|
|
Self(Some(process))
|
|
}
|
|
|
|
fn process(&self) -> std::io::Result<&RetainedProcess> {
|
|
self.0
|
|
.as_ref()
|
|
.ok_or_else(|| std::io::Error::other("exact test child handle is absent"))
|
|
}
|
|
|
|
fn reap(&mut self) -> std::io::Result<()> {
|
|
let process = self
|
|
.0
|
|
.take()
|
|
.ok_or_else(|| std::io::Error::other("exact test child handle is absent"))?;
|
|
child_broker().reap(process)
|
|
}
|
|
|
|
fn kill_and_reap(&mut self) -> std::io::Result<()> {
|
|
self.process()?.send_signal(Signal::KILL)?;
|
|
self.reap()
|
|
}
|
|
}
|
|
|
|
impl Drop for ExactChild {
|
|
fn drop(&mut self) {
|
|
if let Some(process) = self.0.take() {
|
|
let _ = process.send_signal(Signal::KILL);
|
|
let _ = child_broker().reap(process);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn run_host_coordinator_driver(arguments: &[std::ffi::OsString]) -> std::io::Result<()> {
|
|
let tree = arguments
|
|
.first()
|
|
.map(PathBuf::from)
|
|
.ok_or_else(|| std::io::Error::other("Host Coordinator fixture path is missing"))?;
|
|
let ready = arguments
|
|
.get(1)
|
|
.map(PathBuf::from)
|
|
.ok_or_else(|| std::io::Error::other("Host Coordinator ready path is missing"))?;
|
|
if arguments.len() != 2 {
|
|
return Err(std::io::Error::other(
|
|
"Host Coordinator driver arguments are malformed",
|
|
));
|
|
}
|
|
let fixture = Fixture::in_directory(&tree).map_err(matrix_io)?;
|
|
let _runtime = fixture.launch_committed().map_err(matrix_io)?;
|
|
wait_for_activity(&fixture.activity).map_err(matrix_io)?;
|
|
fs::write(ready, b"ready")?;
|
|
loop {
|
|
thread::park();
|
|
}
|
|
}
|
|
|
|
fn matrix_io(error: MatrixError) -> std::io::Error {
|
|
match error {
|
|
MatrixError::Skip(detail) | MatrixError::Failure(detail) => std::io::Error::other(detail),
|
|
}
|
|
}
|
|
|
|
fn failure(error: impl std::fmt::Display) -> MatrixError {
|
|
MatrixError::Failure(error.to_string())
|
|
}
|
|
|
|
fn wait_for_activity(path: &Path) -> Result<(), MatrixError> {
|
|
for _ in 0..400 {
|
|
if path.metadata().is_ok_and(|metadata| metadata.len() >= 2) {
|
|
return Ok(());
|
|
}
|
|
thread::sleep(Duration::from_millis(5));
|
|
}
|
|
Err(failure("guest activity did not start"))
|
|
}
|
|
|
|
fn assert_activity_stable(path: &Path) -> Result<(), MatrixError> {
|
|
let before = fs::metadata(path)?.len();
|
|
thread::sleep(Duration::from_millis(100));
|
|
let after = fs::metadata(path)?.len();
|
|
if before == after {
|
|
Ok(())
|
|
} else {
|
|
Err(failure(
|
|
"guest activity continued after causal authority loss",
|
|
))
|
|
}
|
|
}
|
|
|
|
struct Fixture {
|
|
_tree: Option<TempDir>,
|
|
configuration: ResolvedDomainConfiguration,
|
|
store: HostCoordinatorDomainStore,
|
|
bundle: SandboxBundle,
|
|
activity: PathBuf,
|
|
}
|
|
|
|
impl Fixture {
|
|
fn new() -> Result<Self, MatrixError> {
|
|
let tree = tempfile::tempdir()?;
|
|
let root = tree.path().to_path_buf();
|
|
Self::build(&root, Some(tree))
|
|
}
|
|
|
|
fn in_directory(root: &Path) -> Result<Self, MatrixError> {
|
|
Self::build(root, None)
|
|
}
|
|
|
|
fn build(root: &Path, tree: Option<TempDir>) -> Result<Self, MatrixError> {
|
|
let configuration_directory = root.join("configuration");
|
|
let workspace = root.join("workspace");
|
|
let runtime = root.join("runtime");
|
|
fs::create_dir(&configuration_directory)?;
|
|
fs::create_dir(&workspace)?;
|
|
fs::create_dir(&runtime)?;
|
|
let configuration_path = configuration_directory.join("lazaret.toml");
|
|
fs::write(
|
|
&configuration_path,
|
|
format!(
|
|
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = []\n",
|
|
root.join("data").display()
|
|
),
|
|
)?;
|
|
let configuration = load_trusted_configuration(ConfigurationLoadRequest {
|
|
explicit_configuration_path: Some(configuration_path),
|
|
selected_profile: "default".into(),
|
|
workspace: workspace.clone(),
|
|
home: root.join("home"),
|
|
xdg_config_home: None,
|
|
xdg_data_home: None,
|
|
xdg_runtime_dir: runtime,
|
|
environment: BTreeMap::new(),
|
|
sandbox_layout: production_sandbox_layout().map_err(failure)?,
|
|
})
|
|
.map_err(failure)?;
|
|
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
|
|
stage_rootfs_and_control(&runtime_domain)?;
|
|
let container_id = ContainerId::new([6; 32]);
|
|
let root_process = RootProcess::new(
|
|
"/bin/sh",
|
|
[
|
|
"-c".to_owned(),
|
|
"while :; do echo x >> /workspace/activity; /bin/sleep 0.02; done".to_owned(),
|
|
],
|
|
)
|
|
.map_err(failure)?;
|
|
let namespace =
|
|
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
|
|
let bundle = build_sandbox_bundle(SandboxBuildRequest {
|
|
configuration: &configuration,
|
|
container_id,
|
|
expected_network_namespace: namespace,
|
|
root_process,
|
|
terminal: None,
|
|
})
|
|
.map_err(failure)?;
|
|
let bundle_path = runtime_domain
|
|
.join("bundles")
|
|
.join(hex(container_id.as_bytes()));
|
|
fs::create_dir_all(&bundle_path)?;
|
|
fs::write(bundle_path.join("config.json"), bundle.specification_json())?;
|
|
fs::create_dir_all(runtime_domain.join("runsc"))?;
|
|
let domain = Domain {
|
|
domain_id: configuration.domain_id(),
|
|
profile_name: configuration.selected_profile().to_owned(),
|
|
canonical_workspace: configuration.workspace().canonical_label().clone(),
|
|
};
|
|
let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label());
|
|
let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?;
|
|
let store =
|
|
HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain)
|
|
.map_err(failure)?;
|
|
store
|
|
.apply(&[DomainStoreMutation::PutGeneration(Generation {
|
|
domain_id: configuration.domain_id(),
|
|
generation_id: GenerationId::new(7),
|
|
configuration_fingerprint: configuration.configuration_fingerprint(),
|
|
coordinator_id: CoordinatorId::new([3; 32]),
|
|
network_sidecar_id: NetworkSidecarId::new([4; 32]),
|
|
sandbox_id: SandboxId::new([5; 32]),
|
|
durable_decision: DurableDecision::None,
|
|
})])
|
|
.map_err(failure)?;
|
|
let coordinator = SandboxCoordinator::new(&store);
|
|
coordinator
|
|
.begin_intent(
|
|
GenerationId::new(7),
|
|
container_id,
|
|
&runtime_domain.join("runsc"),
|
|
BundleDigest::new([7; 32]),
|
|
current_process_identity()?,
|
|
)
|
|
.map_err(failure)?;
|
|
Ok(Self {
|
|
_tree: tree,
|
|
configuration,
|
|
store,
|
|
bundle,
|
|
activity: workspace.join("activity"),
|
|
})
|
|
}
|
|
|
|
fn launch_pending(&self) -> Result<AwaitingAttachedRunsc, MatrixError> {
|
|
let launch = RuntimeSupervisorLaunch::from_runsc_command(
|
|
ProcessOwnershipContext {
|
|
domain_id: self.configuration.domain_id(),
|
|
generation_id: GenerationId::new(7),
|
|
coordinator_id: CoordinatorId::new([3; 32]),
|
|
slot: lazaret_core::records::HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
|
|
},
|
|
self.bundle.command(),
|
|
)?;
|
|
AwaitingAttachedRunsc::launch(&SandboxCoordinator::new(&self.store), &launch)
|
|
.map_err(failure)
|
|
}
|
|
|
|
fn launch_committed(&self) -> Result<CommittedSandboxRuntime, MatrixError> {
|
|
let mut pending = self.launch_pending()?;
|
|
let staged = pending
|
|
.receive_staged(&SandboxCoordinator::new(&self.store))
|
|
.map_err(failure)?;
|
|
pending
|
|
.acknowledge(staged)
|
|
.map_err(failure)?
|
|
.commit(&SandboxCoordinator::new(&self.store))
|
|
.map_err(failure)
|
|
}
|
|
}
|
|
|
|
fn stage_rootfs_and_control(runtime_domain: &Path) -> Result<(), MatrixError> {
|
|
let rootfs = runtime_domain.join("state-anchor/merged");
|
|
for directory in ["bin", "lib64"] {
|
|
fs::create_dir_all(rootfs.join(directory))?;
|
|
}
|
|
for (source, destination) in [
|
|
("/bin/sh", "bin/sh"),
|
|
("/bin/sleep", "bin/sleep"),
|
|
("/lib64/libc.so.6", "lib64/libc.so.6"),
|
|
("/lib64/libtinfo.so.6", "lib64/libtinfo.so.6"),
|
|
("/lib64/ld-linux-x86-64.so.2", "lib64/ld-linux-x86-64.so.2"),
|
|
] {
|
|
fs::copy(source, rootfs.join(destination))?;
|
|
}
|
|
let control = runtime_domain.join("sandbox-control").join(hex(&[6; 32]));
|
|
for directory in ["credentials", "mailbox/requests", "mailbox/results", "bin"] {
|
|
fs::create_dir_all(control.join(directory))?;
|
|
}
|
|
fs::copy("/bin/true", control.join("bin/lazaret-escalate"))?;
|
|
Ok(())
|
|
}
|
|
|
|
fn sandbox_participant(store: &HostCoordinatorDomainStore) -> lazaret_core::records::Participant {
|
|
store
|
|
.snapshot()
|
|
.unwrap()
|
|
.participants
|
|
.into_iter()
|
|
.find(|participant| participant.role() == ParticipantRole::Sandbox)
|
|
.unwrap()
|
|
}
|
|
|
|
fn lock_file(path: &Path) -> File {
|
|
OpenOptions::new()
|
|
.read(true)
|
|
.write(true)
|
|
.create(true)
|
|
.truncate(false)
|
|
.open(path)
|
|
.unwrap()
|
|
}
|
|
|
|
fn host_path(path: &HostPath) -> PathBuf {
|
|
PathBuf::from(std::ffi::OsString::from_vec(path.as_bytes().to_vec()))
|
|
}
|
|
|
|
fn hex(bytes: &[u8]) -> String {
|
|
use std::fmt::Write as _;
|
|
|
|
bytes.iter().fold(String::new(), |mut output, byte| {
|
|
write!(output, "{byte:02x}").unwrap();
|
|
output
|
|
})
|
|
}
|