ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Decompose oversized Rust files by responsibility

Browse source
This commit is contained in:
ms2k 2026-07-19 03:02:44 +09:00
commit ee9201b057
26 changed files with 18648 additions and 18513 deletions
Download patch file Download diff file Expand all files Collapse all files

675
crates/lazaret-host-coordinator/src/cold_start.rs
View file

@ -417,675 +417,6 @@ fn require_inputs(inputs: &ColdStartInputs<'_>) -> Result<(), ColdStartError> {
Ok(())
}
struct GitRoutingLayout {
_runtime_directory: RuntimeGitRoutingDirectory,
workspace: File,
workspace_label: PathBuf,
isolated: File,
isolated_label: PathBuf,
mountpoint: File,
mountpoint_label: PathBuf,
git_common: File,
git_common_label: PathBuf,
transient_hooks: File,
transient_hooks_label: PathBuf,
transient_config: File,
transient_config_label: PathBuf,
global_excludes: Option<PathBuf>,
isolate_ignored: bool,
}
impl GitRoutingLayout {
fn create(
configuration: &ResolvedDomainConfiguration,
container_id: ContainerId,
invoking_uid: u32,
) -> Result<Option<Self>, ColdStartError> {
let workspace_label = host_path(configuration.workspace().canonical_label().as_bytes());
let workspace = File::from(
configuration
.workspace()
.open_authority()
.map_err(failure)?,
);
match statat(&workspace, ".git", AtFlags::SYMLINK_NOFOLLOW) {
Ok(_) => {}
Err(rustix::io::Errno::NOENT) => return Ok(None),
Err(error) => return Err(failure(error)),
}
let persistent_root = configuration
.persistent_domain_root()
.open_authority()
.map_err(failure)?;
let isolated = open_or_create_private_directory(
&persistent_root,
"ignored-workspace",
invoking_uid,
false,
)?;
let runtime_root = configuration
.runtime_domain_root()
.open_authority()
.map_err(failure)?;
let runtime_parent =
open_or_create_private_directory(&runtime_root, "git-routing", invoking_uid, false)?;
let runtime_name = OsString::from(hex(container_id.as_bytes()));
let directory =
open_or_create_private_directory(&runtime_parent, &runtime_name, invoking_uid, true)?;
let identity = FileIdentity::capture(&directory)?;
let runtime_directory = RuntimeGitRoutingDirectory {
parent: runtime_parent,
directory,
name: runtime_name.clone(),
identity,
};
let mountpoint = open_or_create_private_directory(
&runtime_directory.directory,
"workspace",
invoking_uid,
true,
)?;
let transient_hooks = open_or_create_private_directory(
&runtime_directory.directory,
"hooks",
invoking_uid,
true,
)?;
let transient_config = openat2(
&runtime_directory.directory,
"config",
OFlags::RDWR | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RUSR | Mode::WUSR,
confined_resolve_flags(),
)
.map_err(failure)?;
fchmod(&transient_config, Mode::RUSR | Mode::WUSR).map_err(failure)?;
require_private_file(&transient_config, invoking_uid, 0o600)?;
let isolated_label = host_path(
configuration
.persistent_domain_root()
.canonical_label()
.as_bytes(),
)
.join("ignored-workspace");
let runtime_label = host_path(
configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
)
.join("git-routing")
.join(&runtime_name);
let mountpoint_label = runtime_label.join("workspace");
let transient_hooks_label = runtime_label.join("hooks");
let transient_config_label = runtime_label.join("config");
let git_common_path = git_common_directory(&workspace_label).map_err(failure)?;
let git_common = open_absolute_directory(&git_common_path)?;
let global_excludes = global_excludes_path(&workspace_label)?;
Ok(Some(Self {
_runtime_directory: runtime_directory,
workspace,
workspace_label,
isolated: File::from(isolated),
isolated_label,
mountpoint: File::from(mountpoint),
mountpoint_label,
git_common,
git_common_label: git_common_path,
transient_hooks: File::from(transient_hooks),
transient_hooks_label,
transient_config: File::from(transient_config),
transient_config_label,
global_excludes,
isolate_ignored: configuration.fingerprint_record().isolate_gitignored,
}))
}
fn launch_authority(
&self,
configuration: &ResolvedDomainConfiguration,
) -> Result<RuntimeGitRoutingLaunch, ColdStartError> {
configuration
.workspace()
.open_authority()
.map_err(failure)?;
RuntimeGitRoutingLaunch::new(RuntimeGitRoutingAuthority {
workspace: self.workspace.try_clone().map_err(failure)?,
workspace_label: self.workspace_label.clone(),
isolated: self.isolated.try_clone().map_err(failure)?,
mountpoint: self.mountpoint.try_clone().map_err(failure)?,
mountpoint_label: self.mountpoint_label.clone(),
git_common: self.git_common.try_clone().map_err(failure)?,
git_common_label: self.git_common_label.clone(),
transient_hooks: self.transient_hooks.try_clone().map_err(failure)?,
transient_hooks_label: self.transient_hooks_label.clone(),
transient_config: self.transient_config.try_clone().map_err(failure)?,
transient_config_label: self.transient_config_label.clone(),
global_excludes: self.global_excludes.clone(),
isolate_ignored: self.isolate_ignored,
})
.map_err(failure)
}
fn mountpoint(&self) -> HostPath {
HostPath::new(
self.mountpoint_label
.as_os_str()
.as_encoded_bytes()
.to_vec(),
)
}
fn host_root(&self) -> HostPath {
HostPath::new(self.workspace_label.as_os_str().as_encoded_bytes().to_vec())
}
fn isolated_root(&self) -> HostPath {
HostPath::new(self.isolated_label.as_os_str().as_encoded_bytes().to_vec())
}
}
struct RuntimeGitRoutingDirectory {
parent: OwnedFd,
directory: OwnedFd,
name: OsString,
identity: FileIdentity,
}
impl Drop for RuntimeGitRoutingDirectory {
fn drop(&mut self) {
remove_staging_directory(&self.parent, &self.name, &self.directory, self.identity);
}
}
fn open_absolute_directory(path: &Path) -> Result<File, ColdStartError> {
let descriptor = openat2(
rustix::fs::CWD,
path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
)
.map_err(failure)?;
Ok(File::from(descriptor))
}
fn global_excludes_path(workspace: &Path) -> Result<Option<PathBuf>, ColdStartError> {
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).map_err(failure)?;
let target = TargetCommand::new("git")
.argument("-C")
.argument(workspace)
.argument("config")
.argument("--path")
.argument("--get")
.argument("--default")
.argument("")
.argument("core.excludesFile");
child_broker()
.run_transient_with_private_stdout(&target, writer)
.map_err(|error| failure(format!("Git global-excludes discovery: {error}")))?;
let mut output = Vec::new();
File::from(reader)
.read_to_end(&mut output)
.map_err(failure)?;
parse_global_excludes_path(workspace, &output)
}
fn parse_global_excludes_path(
workspace: &Path,
output: &[u8],
) -> Result<Option<PathBuf>, ColdStartError> {
let value = output.strip_suffix(b"\n").unwrap_or(output);
if value.contains(&0) {
return Err(ColdStartError("Git global-excludes path is invalid".into()));
}
if value.is_empty() {
return Ok(default_global_excludes_path());
}
let path = PathBuf::from(OsString::from_vec(value.to_vec()));
Ok(Some(if path.is_absolute() {
path
} else {
workspace.join(path)
}))
}
fn default_global_excludes_path() -> Option<PathBuf> {
std::env::var_os("XDG_CONFIG_HOME")
.filter(|path| !path.is_empty())
.map(PathBuf::from)
.or_else(|| {
std::env::var_os("HOME")
.filter(|path| !path.is_empty())
.map(|home| PathBuf::from(home).join(".config"))
})
.map(|home| home.join("git/ignore"))
}
pub(crate) struct RuntimeStaging {
runsc_root: OwnedFd,
bundles: OwnedFd,
bundle: OwnedFd,
controls: OwnedFd,
control: OwnedFd,
overlays: OwnedFd,
overlay: OwnedFd,
name: OsString,
bundle_identity: FileIdentity,
control_identity: FileIdentity,
overlay_identity: FileIdentity,
}
impl RuntimeStaging {
pub(crate) fn runsc_root_identity(&self) -> Result<MountIdentity, ColdStartError> {
mount_identity(&self.runsc_root).map_err(failure)
}
pub(crate) fn cleanup_overlay(&self) -> Result<(), ColdStartError> {
let retained = fstat(&self.overlay).map_err(failure)?;
if !self.overlay_identity.matches(&retained) {
return Err(ColdStartError(
"runsc overlay retained authority changed".into(),
));
}
remove_children(&self.overlay).map_err(failure)?;
let current =
statat(&self.overlays, &self.name, AtFlags::SYMLINK_NOFOLLOW).map_err(failure)?;
if !self.overlay_identity.matches(&current)
|| !FileType::from_raw_mode(current.st_mode).is_dir()
{
return Err(ColdStartError(
"runsc overlay path changed before cleanup".into(),
));
}
unlinkat(&self.overlays, &self.name, AtFlags::REMOVEDIR).map_err(failure)?;
Ok(())
}
pub(crate) fn escalation_mailboxes(
&self,
invoking_uid: u32,
) -> Result<(RegularFileMailbox, RegularFileMailbox, RegularFileMailbox), ColdStartError> {
let credentials = RegularFileMailbox::openat_private(
&self.control,
OsStr::new("credentials"),
invoking_uid,
)
.map_err(failure)?;
let mailbox =
RegularFileMailbox::openat_private(&self.control, OsStr::new("mailbox"), invoking_uid)
.map_err(failure)?;
let requests =
RegularFileMailbox::openat_private(&mailbox, OsStr::new("requests"), invoking_uid)
.map_err(failure)?;
let results =
RegularFileMailbox::openat_private(&mailbox, OsStr::new("results"), invoking_uid)
.map_err(failure)?;
Ok((credentials, requests, results))
}
fn create(
runtime_domain: &ConfinedPathPlan,
container_id: ContainerId,
escalation_client: &File,
sandbox_init: &File,
invoking_uid: u32,
) -> Result<Self, ColdStartError> {
let root = runtime_domain.open_authority().map_err(failure)?;
let bundles = open_or_create_private_directory(&root, "bundles", invoking_uid, false)?;
let controls =
open_or_create_private_directory(&root, "sandbox-control", invoking_uid, false)?;
let overlays =
open_or_create_private_directory(&root, "runsc-overlay", invoking_uid, false)?;
let runsc_root = open_or_create_private_directory(&root, "runsc", invoking_uid, false)?;
let name = OsString::from(hex(container_id.as_bytes()));
let bundle = open_or_create_private_directory(&bundles, &name, invoking_uid, true)?;
let bundle_identity = FileIdentity::capture(&bundle)?;
let control = match open_or_create_private_directory(&controls, &name, invoking_uid, true) {
Ok(control) => control,
Err(error) => {
remove_staging_directory(&bundles, &name, &bundle, bundle_identity);
return Err(error);
}
};
let control_identity = FileIdentity::capture(&control)?;
let overlay = match open_or_create_private_directory(&overlays, &name, invoking_uid, true) {
Ok(overlay) => overlay,
Err(error) => {
remove_staging_directory(&controls, &name, &control, control_identity);
remove_staging_directory(&bundles, &name, &bundle, bundle_identity);
return Err(error);
}
};
let overlay_identity = FileIdentity::capture(&overlay)?;
let staging = Self {
runsc_root,
bundles,
bundle,
controls,
control,
overlays,
overlay,
name,
bundle_identity,
control_identity,
overlay_identity,
};
staging.populate_control(escalation_client, sandbox_init)?;
Ok(staging)
}
fn populate_control(
&self,
escalation_client: &File,
sandbox_init: &File,
) -> Result<(), ColdStartError> {
let invoking_uid = fstat(&self.control).map_err(failure)?.st_uid;
drop(open_or_create_private_directory(
&self.control,
"credentials",
invoking_uid,
true,
)?);
let mailbox =
open_or_create_private_directory(&self.control, "mailbox", invoking_uid, true)?;
drop(open_or_create_private_directory(
&mailbox,
"requests",
invoking_uid,
true,
)?);
drop(open_or_create_private_directory(
&mailbox,
"results",
invoking_uid,
true,
)?);
let bin = open_or_create_private_directory(&self.control, "bin", invoking_uid, true)?;
require_executable_regular_file(escalation_client, "escalation client")?;
copy_descriptor(escalation_client, &bin, "lazaret-escalate", invoking_uid)?;
require_executable_regular_file(sandbox_init, "sandbox init")?;
copy_descriptor(sandbox_init, &bin, "lazaret-sandbox-init", invoking_uid)?;
Ok(())
}
fn write_bundle(&self, bytes: &[u8]) -> Result<(), ColdStartError> {
let descriptor = openat2(
&self.bundle,
"config.json",
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RUSR | Mode::WUSR,
confined_resolve_flags(),
)
.map_err(failure)?;
let mut file = File::from(descriptor);
file.write_all(bytes).map_err(failure)?;
fchmod(file.as_fd(), Mode::RUSR | Mode::WUSR).map_err(failure)?;
file.sync_all().map_err(failure)?;
require_private_file(&file, fstat(&self.bundle).map_err(failure)?.st_uid, 0o600)?;
Ok(())
}
}
fn copy_descriptor(
source: &File,
parent: &OwnedFd,
name: &str,
invoking_uid: u32,
) -> Result<(), ColdStartError> {
let descriptor = openat2(
parent,
name,
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RWXU,
confined_resolve_flags(),
)
.map_err(failure)?;
let mut destination = File::from(descriptor);
fchmod(destination.as_fd(), Mode::RWXU).map_err(failure)?;
let mut offset = 0_u64;
let mut buffer = [0_u8; 16 * 1024];
loop {
let read = source.read_at(&mut buffer, offset).map_err(failure)?;
if read == 0 {
break;
}
destination.write_all(&buffer[..read]).map_err(failure)?;
offset = offset
.checked_add(read as u64)
.ok_or_else(|| ColdStartError("escalation client is too large".into()))?;
}
destination.sync_all().map_err(failure)?;
require_private_file(&destination, invoking_uid, 0o700)
}
fn require_executable_regular_file(source: &File, label: &str) -> Result<(), ColdStartError> {
let metadata = source
.metadata()
.map_err(|error| failure(format!("inspecting {label}: {error}")))?;
if !metadata.file_type().is_file() || metadata.permissions().mode() & 0o111 == 0 {
return Err(ColdStartError(format!(
"{label} is not an executable regular file"
)));
}
Ok(())
}
#[derive(Clone, Copy)]
struct FileIdentity {
device: u64,
inode: u64,
}
impl FileIdentity {
fn capture(descriptor: impl AsFd) -> Result<Self, ColdStartError> {
let metadata = fstat(descriptor).map_err(failure)?;
Ok(Self {
device: metadata.st_dev,
inode: metadata.st_ino,
})
}
fn matches(self, metadata: &rustix::fs::Stat) -> bool {
metadata.st_dev == self.device && metadata.st_ino == self.inode
}
}
impl Drop for RuntimeStaging {
fn drop(&mut self) {
remove_staging_directory(
&self.overlays,
&self.name,
&self.overlay,
self.overlay_identity,
);
remove_staging_directory(
&self.controls,
&self.name,
&self.control,
self.control_identity,
);
remove_staging_directory(
&self.bundles,
&self.name,
&self.bundle,
self.bundle_identity,
);
}
}
fn open_or_create_private_directory(
parent: impl AsFd,
name: impl AsRef<std::ffi::OsStr>,
invoking_uid: u32,
require_new: bool,
) -> Result<OwnedFd, ColdStartError> {
let name = name.as_ref();
let created = match mkdirat(parent.as_fd(), name, Mode::RWXU) {
Ok(()) => true,
Err(rustix::io::Errno::EXIST) if !require_new => false,
Err(error) => return Err(failure(error)),
};
let directory = openat2(
parent.as_fd(),
name,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
confined_resolve_flags(),
)
.map_err(failure)?;
if created {
fchmod(&directory, Mode::RWXU).map_err(failure)?;
}
require_private_directory(&directory, invoking_uid)?;
Ok(directory)
}
fn require_private_directory(
descriptor: impl AsFd,
invoking_uid: u32,
) -> Result<(), ColdStartError> {
let metadata = fstat(descriptor).map_err(failure)?;
if !FileType::from_raw_mode(metadata.st_mode).is_dir()
|| metadata.st_uid != invoking_uid
|| metadata.st_mode & 0o7777 != 0o700
{
return Err(ColdStartError(
"runtime staging directory is not invoking-user-owned mode 0700".into(),
));
}
Ok(())
}
fn require_private_file(
descriptor: impl AsFd,
invoking_uid: u32,
mode: u32,
) -> Result<(), ColdStartError> {
let metadata = fstat(descriptor).map_err(failure)?;
if !FileType::from_raw_mode(metadata.st_mode).is_file()
|| metadata.st_uid != invoking_uid
|| metadata.st_mode & 0o7777 != mode
|| metadata.st_nlink != 1
{
return Err(ColdStartError(
"runtime staging file has unsafe type, ownership, mode, or link count".into(),
));
}
Ok(())
}
fn remove_staging_directory(
parent: impl AsFd,
name: &std::ffi::OsStr,
directory: impl AsFd,
expected: FileIdentity,
) {
let retained = fstat(directory.as_fd());
if !retained
.as_ref()
.is_ok_and(|metadata| expected.matches(metadata))
{
return;
}
if remove_children(directory.as_fd()).is_err() {
return;
}
let Ok(current) = statat(parent.as_fd(), name, AtFlags::SYMLINK_NOFOLLOW) else {
return;
};
if expected.matches(&current) && FileType::from_raw_mode(current.st_mode).is_dir() {
let _ = unlinkat(parent.as_fd(), name, AtFlags::REMOVEDIR);
}
}
fn remove_children(directory: impl AsFd) -> std::io::Result<()> {
let duplicate = rustix::io::dup(directory.as_fd())?;
let mut buffer = [std::mem::MaybeUninit::uninit(); 8192];
let names = {
let mut names = Vec::new();
let mut entries = RawDir::new(duplicate, &mut buffer);
while let Some(entry) = entries.next() {
let entry = entry?;
let bytes = entry.file_name().to_bytes();
if bytes != b"." && bytes != b".." {
names.push(OsString::from_vec(bytes.to_vec()));
}
}
names
};
for name in names {
let metadata = statat(directory.as_fd(), &name, AtFlags::SYMLINK_NOFOLLOW)?;
if FileType::from_raw_mode(metadata.st_mode).is_dir() {
let child = openat2(
directory.as_fd(),
&name,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
confined_resolve_flags(),
)?;
remove_children(&child)?;
unlinkat(directory.as_fd(), &name, AtFlags::REMOVEDIR)?;
} else {
unlinkat(directory.as_fd(), &name, AtFlags::empty())?;
}
}
Ok(())
}
fn confined_resolve_flags() -> ResolveFlags {
ResolveFlags::BENEATH
| ResolveFlags::NO_MAGICLINKS
| ResolveFlags::NO_SYMLINKS
| ResolveFlags::NO_XDEV
}
fn host_path(bytes: &[u8]) -> PathBuf {
PathBuf::from(OsString::from_vec(bytes.to_vec()))
}
fn hex(bytes: &[u8]) -> String {
use std::fmt::Write as _;
bytes.iter().fold(String::new(), |mut output, byte| {
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
output
})
}
#[cfg(test)]
mod tests {
use std::fs::File;
use std::os::fd::OwnedFd;
use rustix::pipe::{PipeFlags, pipe_with};
use super::BootstrapOwnerGuard;
#[test]
fn bootstrap_owner_guard_tracks_the_exact_inherited_pipe() {
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).unwrap();
let guard = BootstrapOwnerGuard::from_inherited(reader).unwrap();
assert!(guard.is_live().unwrap());
drop(writer);
assert!(!guard.is_live().unwrap());
}
#[test]
fn bootstrap_owner_guard_refuses_non_pipe_authority() {
let descriptor: OwnedFd = File::open("/dev/null").unwrap().into();
assert!(BootstrapOwnerGuard::from_inherited(descriptor).is_err());
}
#[test]
fn repository_relative_global_excludes_are_workspace_relative() {
let root = tempfile::tempdir().unwrap();
let workspace = root.path().join("workspace");
assert_eq!(
super::parse_global_excludes_path(&workspace, b"policy/ignore\n").unwrap(),
Some(workspace.join("policy/ignore"))
);
assert!(super::parse_global_excludes_path(&workspace, b"bad\0path\n").is_err());
}
}
include!("cold_start/git_routing.rs");
include!("cold_start/runtime_staging.rs");
include!("cold_start/tests.rs");

245
crates/lazaret-host-coordinator/src/cold_start/git_routing.rs Normal file
View file

@ -0,0 +1,245 @@
struct GitRoutingLayout {
_runtime_directory: RuntimeGitRoutingDirectory,
workspace: File,
workspace_label: PathBuf,
isolated: File,
isolated_label: PathBuf,
mountpoint: File,
mountpoint_label: PathBuf,
git_common: File,
git_common_label: PathBuf,
transient_hooks: File,
transient_hooks_label: PathBuf,
transient_config: File,
transient_config_label: PathBuf,
global_excludes: Option<PathBuf>,
isolate_ignored: bool,
}
impl GitRoutingLayout {
fn create(
configuration: &ResolvedDomainConfiguration,
container_id: ContainerId,
invoking_uid: u32,
) -> Result<Option<Self>, ColdStartError> {
let workspace_label = host_path(configuration.workspace().canonical_label().as_bytes());
let workspace = File::from(
configuration
.workspace()
.open_authority()
.map_err(failure)?,
);
match statat(&workspace, ".git", AtFlags::SYMLINK_NOFOLLOW) {
Ok(_) => {}
Err(rustix::io::Errno::NOENT) => return Ok(None),
Err(error) => return Err(failure(error)),
}
let persistent_root = configuration
.persistent_domain_root()
.open_authority()
.map_err(failure)?;
let isolated = open_or_create_private_directory(
&persistent_root,
"ignored-workspace",
invoking_uid,
false,
)?;
let runtime_root = configuration
.runtime_domain_root()
.open_authority()
.map_err(failure)?;
let runtime_parent =
open_or_create_private_directory(&runtime_root, "git-routing", invoking_uid, false)?;
let runtime_name = OsString::from(hex(container_id.as_bytes()));
let directory =
open_or_create_private_directory(&runtime_parent, &runtime_name, invoking_uid, true)?;
let identity = FileIdentity::capture(&directory)?;
let runtime_directory = RuntimeGitRoutingDirectory {
parent: runtime_parent,
directory,
name: runtime_name.clone(),
identity,
};
let mountpoint = open_or_create_private_directory(
&runtime_directory.directory,
"workspace",
invoking_uid,
true,
)?;
let transient_hooks = open_or_create_private_directory(
&runtime_directory.directory,
"hooks",
invoking_uid,
true,
)?;
let transient_config = openat2(
&runtime_directory.directory,
"config",
OFlags::RDWR | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RUSR | Mode::WUSR,
confined_resolve_flags(),
)
.map_err(failure)?;
fchmod(&transient_config, Mode::RUSR | Mode::WUSR).map_err(failure)?;
require_private_file(&transient_config, invoking_uid, 0o600)?;
let isolated_label = host_path(
configuration
.persistent_domain_root()
.canonical_label()
.as_bytes(),
)
.join("ignored-workspace");
let runtime_label = host_path(
configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
)
.join("git-routing")
.join(&runtime_name);
let mountpoint_label = runtime_label.join("workspace");
let transient_hooks_label = runtime_label.join("hooks");
let transient_config_label = runtime_label.join("config");
let git_common_path = git_common_directory(&workspace_label).map_err(failure)?;
let git_common = open_absolute_directory(&git_common_path)?;
let global_excludes = global_excludes_path(&workspace_label)?;
Ok(Some(Self {
_runtime_directory: runtime_directory,
workspace,
workspace_label,
isolated: File::from(isolated),
isolated_label,
mountpoint: File::from(mountpoint),
mountpoint_label,
git_common,
git_common_label: git_common_path,
transient_hooks: File::from(transient_hooks),
transient_hooks_label,
transient_config: File::from(transient_config),
transient_config_label,
global_excludes,
isolate_ignored: configuration.fingerprint_record().isolate_gitignored,
}))
}
fn launch_authority(
&self,
configuration: &ResolvedDomainConfiguration,
) -> Result<RuntimeGitRoutingLaunch, ColdStartError> {
configuration
.workspace()
.open_authority()
.map_err(failure)?;
RuntimeGitRoutingLaunch::new(RuntimeGitRoutingAuthority {
workspace: self.workspace.try_clone().map_err(failure)?,
workspace_label: self.workspace_label.clone(),
isolated: self.isolated.try_clone().map_err(failure)?,
mountpoint: self.mountpoint.try_clone().map_err(failure)?,
mountpoint_label: self.mountpoint_label.clone(),
git_common: self.git_common.try_clone().map_err(failure)?,
git_common_label: self.git_common_label.clone(),
transient_hooks: self.transient_hooks.try_clone().map_err(failure)?,
transient_hooks_label: self.transient_hooks_label.clone(),
transient_config: self.transient_config.try_clone().map_err(failure)?,
transient_config_label: self.transient_config_label.clone(),
global_excludes: self.global_excludes.clone(),
isolate_ignored: self.isolate_ignored,
})
.map_err(failure)
}
fn mountpoint(&self) -> HostPath {
HostPath::new(
self.mountpoint_label
.as_os_str()
.as_encoded_bytes()
.to_vec(),
)
}
fn host_root(&self) -> HostPath {
HostPath::new(self.workspace_label.as_os_str().as_encoded_bytes().to_vec())
}
fn isolated_root(&self) -> HostPath {
HostPath::new(self.isolated_label.as_os_str().as_encoded_bytes().to_vec())
}
}
struct RuntimeGitRoutingDirectory {
parent: OwnedFd,
directory: OwnedFd,
name: OsString,
identity: FileIdentity,
}
impl Drop for RuntimeGitRoutingDirectory {
fn drop(&mut self) {
remove_staging_directory(&self.parent, &self.name, &self.directory, self.identity);
}
}
fn open_absolute_directory(path: &Path) -> Result<File, ColdStartError> {
let descriptor = openat2(
rustix::fs::CWD,
path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
)
.map_err(failure)?;
Ok(File::from(descriptor))
}
fn global_excludes_path(workspace: &Path) -> Result<Option<PathBuf>, ColdStartError> {
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).map_err(failure)?;
let target = TargetCommand::new("git")
.argument("-C")
.argument(workspace)
.argument("config")
.argument("--path")
.argument("--get")
.argument("--default")
.argument("")
.argument("core.excludesFile");
child_broker()
.run_transient_with_private_stdout(&target, writer)
.map_err(|error| failure(format!("Git global-excludes discovery: {error}")))?;
let mut output = Vec::new();
File::from(reader)
.read_to_end(&mut output)
.map_err(failure)?;
parse_global_excludes_path(workspace, &output)
}
fn parse_global_excludes_path(
workspace: &Path,
output: &[u8],
) -> Result<Option<PathBuf>, ColdStartError> {
let value = output.strip_suffix(b"\n").unwrap_or(output);
if value.contains(&0) {
return Err(ColdStartError("Git global-excludes path is invalid".into()));
}
if value.is_empty() {
return Ok(default_global_excludes_path());
}
let path = PathBuf::from(OsString::from_vec(value.to_vec()));
Ok(Some(if path.is_absolute() {
path
} else {
workspace.join(path)
}))
}
fn default_global_excludes_path() -> Option<PathBuf> {
std::env::var_os("XDG_CONFIG_HOME")
.filter(|path| !path.is_empty())
.map(PathBuf::from)
.or_else(|| {
std::env::var_os("HOME")
.filter(|path| !path.is_empty())
.map(|home| PathBuf::from(home).join(".config"))
})
.map(|home| home.join("git/ignore"))
}

391
crates/lazaret-host-coordinator/src/cold_start/runtime_staging.rs Normal file
View file

@ -0,0 +1,391 @@
pub(crate) struct RuntimeStaging {
runsc_root: OwnedFd,
bundles: OwnedFd,
bundle: OwnedFd,
controls: OwnedFd,
control: OwnedFd,
overlays: OwnedFd,
overlay: OwnedFd,
name: OsString,
bundle_identity: FileIdentity,
control_identity: FileIdentity,
overlay_identity: FileIdentity,
}
impl RuntimeStaging {
pub(crate) fn runsc_root_identity(&self) -> Result<MountIdentity, ColdStartError> {
mount_identity(&self.runsc_root).map_err(failure)
}
pub(crate) fn cleanup_overlay(&self) -> Result<(), ColdStartError> {
let retained = fstat(&self.overlay).map_err(failure)?;
if !self.overlay_identity.matches(&retained) {
return Err(ColdStartError(
"runsc overlay retained authority changed".into(),
));
}
remove_children(&self.overlay).map_err(failure)?;
let current =
statat(&self.overlays, &self.name, AtFlags::SYMLINK_NOFOLLOW).map_err(failure)?;
if !self.overlay_identity.matches(&current)
|| !FileType::from_raw_mode(current.st_mode).is_dir()
{
return Err(ColdStartError(
"runsc overlay path changed before cleanup".into(),
));
}
unlinkat(&self.overlays, &self.name, AtFlags::REMOVEDIR).map_err(failure)?;
Ok(())
}
pub(crate) fn escalation_mailboxes(
&self,
invoking_uid: u32,
) -> Result<(RegularFileMailbox, RegularFileMailbox, RegularFileMailbox), ColdStartError> {
let credentials = RegularFileMailbox::openat_private(
&self.control,
OsStr::new("credentials"),
invoking_uid,
)
.map_err(failure)?;
let mailbox =
RegularFileMailbox::openat_private(&self.control, OsStr::new("mailbox"), invoking_uid)
.map_err(failure)?;
let requests =
RegularFileMailbox::openat_private(&mailbox, OsStr::new("requests"), invoking_uid)
.map_err(failure)?;
let results =
RegularFileMailbox::openat_private(&mailbox, OsStr::new("results"), invoking_uid)
.map_err(failure)?;
Ok((credentials, requests, results))
}
fn create(
runtime_domain: &ConfinedPathPlan,
container_id: ContainerId,
escalation_client: &File,
sandbox_init: &File,
invoking_uid: u32,
) -> Result<Self, ColdStartError> {
let root = runtime_domain.open_authority().map_err(failure)?;
let bundles = open_or_create_private_directory(&root, "bundles", invoking_uid, false)?;
let controls =
open_or_create_private_directory(&root, "sandbox-control", invoking_uid, false)?;
let overlays =
open_or_create_private_directory(&root, "runsc-overlay", invoking_uid, false)?;
let runsc_root = open_or_create_private_directory(&root, "runsc", invoking_uid, false)?;
let name = OsString::from(hex(container_id.as_bytes()));
let bundle = open_or_create_private_directory(&bundles, &name, invoking_uid, true)?;
let bundle_identity = FileIdentity::capture(&bundle)?;
let control = match open_or_create_private_directory(&controls, &name, invoking_uid, true) {
Ok(control) => control,
Err(error) => {
remove_staging_directory(&bundles, &name, &bundle, bundle_identity);
return Err(error);
}
};
let control_identity = FileIdentity::capture(&control)?;
let overlay = match open_or_create_private_directory(&overlays, &name, invoking_uid, true) {
Ok(overlay) => overlay,
Err(error) => {
remove_staging_directory(&controls, &name, &control, control_identity);
remove_staging_directory(&bundles, &name, &bundle, bundle_identity);
return Err(error);
}
};
let overlay_identity = FileIdentity::capture(&overlay)?;
let staging = Self {
runsc_root,
bundles,
bundle,
controls,
control,
overlays,
overlay,
name,
bundle_identity,
control_identity,
overlay_identity,
};
staging.populate_control(escalation_client, sandbox_init)?;
Ok(staging)
}
fn populate_control(
&self,
escalation_client: &File,
sandbox_init: &File,
) -> Result<(), ColdStartError> {
let invoking_uid = fstat(&self.control).map_err(failure)?.st_uid;
drop(open_or_create_private_directory(
&self.control,
"credentials",
invoking_uid,
true,
)?);
let mailbox =
open_or_create_private_directory(&self.control, "mailbox", invoking_uid, true)?;
drop(open_or_create_private_directory(
&mailbox,
"requests",
invoking_uid,
true,
)?);
drop(open_or_create_private_directory(
&mailbox,
"results",
invoking_uid,
true,
)?);
let bin = open_or_create_private_directory(&self.control, "bin", invoking_uid, true)?;
require_executable_regular_file(escalation_client, "escalation client")?;
copy_descriptor(escalation_client, &bin, "lazaret-escalate", invoking_uid)?;
require_executable_regular_file(sandbox_init, "sandbox init")?;
copy_descriptor(sandbox_init, &bin, "lazaret-sandbox-init", invoking_uid)?;
Ok(())
}
fn write_bundle(&self, bytes: &[u8]) -> Result<(), ColdStartError> {
let descriptor = openat2(
&self.bundle,
"config.json",
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RUSR | Mode::WUSR,
confined_resolve_flags(),
)
.map_err(failure)?;
let mut file = File::from(descriptor);
file.write_all(bytes).map_err(failure)?;
fchmod(file.as_fd(), Mode::RUSR | Mode::WUSR).map_err(failure)?;
file.sync_all().map_err(failure)?;
require_private_file(&file, fstat(&self.bundle).map_err(failure)?.st_uid, 0o600)?;
Ok(())
}
}
fn copy_descriptor(
source: &File,
parent: &OwnedFd,
name: &str,
invoking_uid: u32,
) -> Result<(), ColdStartError> {
let descriptor = openat2(
parent,
name,
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RWXU,
confined_resolve_flags(),
)
.map_err(failure)?;
let mut destination = File::from(descriptor);
fchmod(destination.as_fd(), Mode::RWXU).map_err(failure)?;
let mut offset = 0_u64;
let mut buffer = [0_u8; 16 * 1024];
loop {
let read = source.read_at(&mut buffer, offset).map_err(failure)?;
if read == 0 {
break;
}
destination.write_all(&buffer[..read]).map_err(failure)?;
offset = offset
.checked_add(read as u64)
.ok_or_else(|| ColdStartError("escalation client is too large".into()))?;
}
destination.sync_all().map_err(failure)?;
require_private_file(&destination, invoking_uid, 0o700)
}
fn require_executable_regular_file(source: &File, label: &str) -> Result<(), ColdStartError> {
let metadata = source
.metadata()
.map_err(|error| failure(format!("inspecting {label}: {error}")))?;
if !metadata.file_type().is_file() || metadata.permissions().mode() & 0o111 == 0 {
return Err(ColdStartError(format!(
"{label} is not an executable regular file"
)));
}
Ok(())
}
#[derive(Clone, Copy)]
struct FileIdentity {
device: u64,
inode: u64,
}
impl FileIdentity {
fn capture(descriptor: impl AsFd) -> Result<Self, ColdStartError> {
let metadata = fstat(descriptor).map_err(failure)?;
Ok(Self {
device: metadata.st_dev,
inode: metadata.st_ino,
})
}
fn matches(self, metadata: &rustix::fs::Stat) -> bool {
metadata.st_dev == self.device && metadata.st_ino == self.inode
}
}
impl Drop for RuntimeStaging {
fn drop(&mut self) {
remove_staging_directory(
&self.overlays,
&self.name,
&self.overlay,
self.overlay_identity,
);
remove_staging_directory(
&self.controls,
&self.name,
&self.control,
self.control_identity,
);
remove_staging_directory(
&self.bundles,
&self.name,
&self.bundle,
self.bundle_identity,
);
}
}
fn open_or_create_private_directory(
parent: impl AsFd,
name: impl AsRef<std::ffi::OsStr>,
invoking_uid: u32,
require_new: bool,
) -> Result<OwnedFd, ColdStartError> {
let name = name.as_ref();
let created = match mkdirat(parent.as_fd(), name, Mode::RWXU) {
Ok(()) => true,
Err(rustix::io::Errno::EXIST) if !require_new => false,
Err(error) => return Err(failure(error)),
};
let directory = openat2(
parent.as_fd(),
name,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
confined_resolve_flags(),
)
.map_err(failure)?;
if created {
fchmod(&directory, Mode::RWXU).map_err(failure)?;
}
require_private_directory(&directory, invoking_uid)?;
Ok(directory)
}
fn require_private_directory(
descriptor: impl AsFd,
invoking_uid: u32,
) -> Result<(), ColdStartError> {
let metadata = fstat(descriptor).map_err(failure)?;
if !FileType::from_raw_mode(metadata.st_mode).is_dir()
|| metadata.st_uid != invoking_uid
|| metadata.st_mode & 0o7777 != 0o700
{
return Err(ColdStartError(
"runtime staging directory is not invoking-user-owned mode 0700".into(),
));
}
Ok(())
}
fn require_private_file(
descriptor: impl AsFd,
invoking_uid: u32,
mode: u32,
) -> Result<(), ColdStartError> {
let metadata = fstat(descriptor).map_err(failure)?;
if !FileType::from_raw_mode(metadata.st_mode).is_file()
|| metadata.st_uid != invoking_uid
|| metadata.st_mode & 0o7777 != mode
|| metadata.st_nlink != 1
{
return Err(ColdStartError(
"runtime staging file has unsafe type, ownership, mode, or link count".into(),
));
}
Ok(())
}
fn remove_staging_directory(
parent: impl AsFd,
name: &std::ffi::OsStr,
directory: impl AsFd,
expected: FileIdentity,
) {
let retained = fstat(directory.as_fd());
if !retained
.as_ref()
.is_ok_and(|metadata| expected.matches(metadata))
{
return;
}
if remove_children(directory.as_fd()).is_err() {
return;
}
let Ok(current) = statat(parent.as_fd(), name, AtFlags::SYMLINK_NOFOLLOW) else {
return;
};
if expected.matches(&current) && FileType::from_raw_mode(current.st_mode).is_dir() {
let _ = unlinkat(parent.as_fd(), name, AtFlags::REMOVEDIR);
}
}
fn remove_children(directory: impl AsFd) -> std::io::Result<()> {
let duplicate = rustix::io::dup(directory.as_fd())?;
let mut buffer = [std::mem::MaybeUninit::uninit(); 8192];
let names = {
let mut names = Vec::new();
let mut entries = RawDir::new(duplicate, &mut buffer);
while let Some(entry) = entries.next() {
let entry = entry?;
let bytes = entry.file_name().to_bytes();
if bytes != b"." && bytes != b".." {
names.push(OsString::from_vec(bytes.to_vec()));
}
}
names
};
for name in names {
let metadata = statat(directory.as_fd(), &name, AtFlags::SYMLINK_NOFOLLOW)?;
if FileType::from_raw_mode(metadata.st_mode).is_dir() {
let child = openat2(
directory.as_fd(),
&name,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
confined_resolve_flags(),
)?;
remove_children(&child)?;
unlinkat(directory.as_fd(), &name, AtFlags::REMOVEDIR)?;
} else {
unlinkat(directory.as_fd(), &name, AtFlags::empty())?;
}
}
Ok(())
}
fn confined_resolve_flags() -> ResolveFlags {
ResolveFlags::BENEATH
| ResolveFlags::NO_MAGICLINKS
| ResolveFlags::NO_SYMLINKS
| ResolveFlags::NO_XDEV
}
fn host_path(bytes: &[u8]) -> PathBuf {
PathBuf::from(OsString::from_vec(bytes.to_vec()))
}
fn hex(bytes: &[u8]) -> String {
use std::fmt::Write as _;
bytes.iter().fold(String::new(), |mut output, byte| {
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
output
})
}

36
crates/lazaret-host-coordinator/src/cold_start/tests.rs Normal file
View file

@ -0,0 +1,36 @@
#[cfg(test)]
mod tests {
use std::fs::File;
use std::os::fd::OwnedFd;
use rustix::pipe::{PipeFlags, pipe_with};
use super::BootstrapOwnerGuard;
#[test]
fn bootstrap_owner_guard_tracks_the_exact_inherited_pipe() {
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).unwrap();
let guard = BootstrapOwnerGuard::from_inherited(reader).unwrap();
assert!(guard.is_live().unwrap());
drop(writer);
assert!(!guard.is_live().unwrap());
}
#[test]
fn bootstrap_owner_guard_refuses_non_pipe_authority() {
let descriptor: OwnedFd = File::open("/dev/null").unwrap().into();
assert!(BootstrapOwnerGuard::from_inherited(descriptor).is_err());
}
#[test]
fn repository_relative_global_excludes_are_workspace_relative() {
let root = tempfile::tempdir().unwrap();
let workspace = root.path().join("workspace");
assert_eq!(
super::parse_global_excludes_path(&workspace, b"policy/ignore\n").unwrap(),
Some(workspace.join("policy/ignore"))
);
assert!(super::parse_global_excludes_path(&workspace, b"bad\0path\n").is_err());
}
}

1560
crates/lazaret-host-coordinator/src/control.rs
View file

File diff suppressed because it is too large Load diff

354
crates/lazaret-host-coordinator/src/control/execution.rs Normal file
View file

@ -0,0 +1,354 @@
struct ControlPaths {
root: PathBuf,
root_file: File,
root_identity: FileIdentity,
socket: PathBuf,
identity: PathBuf,
staged_socket: PathBuf,
staged_identity: PathBuf,
}
impl ControlPaths {
fn open(root: &Path, uid: u32) -> Result<Self, ControlError> {
if !root.is_absolute() {
return Err(ControlError::InvalidPath(
"runtime Domain root is not absolute",
));
}
let canonical = std::fs::canonicalize(root)?;
if canonical != root {
return Err(ControlError::InvalidPath(
"runtime Domain root contains a symlink",
));
}
let metadata = std::fs::symlink_metadata(root)?;
if !metadata.file_type().is_dir()
|| metadata.uid() != uid
|| metadata.mode() & 0o7777 != 0o700
{
return Err(ControlError::InvalidPath(
"runtime Domain root is not invoking-user-owned mode 0700",
));
}
let root_file = OpenOptions::new().read(true).open(root)?;
let root_identity = FileIdentity::from_metadata(&root_file.metadata()?);
Ok(Self {
root: root.to_path_buf(),
root_file,
root_identity,
socket: root.join(CONTROL_SOCKET_NAME),
identity: root.join(PUBLICATION_IDENTITY_NAME),
staged_socket: root.join(STAGED_CONTROL_SOCKET_NAME),
staged_identity: root.join(STAGED_PUBLICATION_IDENTITY_NAME),
})
}
fn validate_root(&self, uid: u32) -> Result<(), ControlError> {
let descriptor = self.root_file.metadata()?;
let current = std::fs::symlink_metadata(&self.root)?;
if FileIdentity::from_metadata(&descriptor) != self.root_identity
|| FileIdentity::from_metadata(&current) != self.root_identity
|| !current.file_type().is_dir()
|| current.uid() != uid
|| current.mode() & 0o7777 != 0o700
{
return Err(ControlError::InvalidPath("runtime Domain root changed"));
}
Ok(())
}
fn create_named_identity(
&self,
name: &str,
bytes: &[u8],
uid: u32,
) -> Result<(File, FileIdentity), ControlError> {
let fd = openat(
self.root_file.as_fd(),
name,
OFlags::CREATE | OFlags::EXCL | OFlags::RDWR | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RUSR | Mode::WUSR,
)
.map_err(io::Error::from)?;
let mut file = File::from(fd);
let identity = FileIdentity::from_metadata(&file.metadata()?);
let result = (|| {
fchmod(file.as_fd(), Mode::RUSR | Mode::WUSR).map_err(io::Error::from)?;
file.write_all(bytes)
.map_err(|_| ControlError::InvalidPublication("identity write failed"))?;
file.sync_all()
.map_err(|_| ControlError::InvalidPublication("identity sync failed"))?;
self.root_file
.sync_all()
.map_err(|_| ControlError::InvalidPublication("directory sync failed"))?;
self.validate_identity_at(&self.root.join(name), &file, identity, uid, bytes.len())
})();
if let Err(error) = result {
remove_if_identity_matches(&self.root.join(name), identity);
return Err(error);
}
Ok((file, identity))
}
fn validate_identity(
&self,
file: &File,
expected: FileIdentity,
uid: u32,
expected_length: usize,
) -> Result<(), ControlError> {
self.validate_identity_at(&self.identity, file, expected, uid, expected_length)
}
fn validate_identity_at(
&self,
path: &Path,
file: &File,
expected: FileIdentity,
uid: u32,
expected_length: usize,
) -> Result<(), ControlError> {
self.validate_root(uid)?;
let descriptor = file.metadata()?;
let current = std::fs::symlink_metadata(path)?;
if FileIdentity::from_metadata(&descriptor) != expected
|| FileIdentity::from_metadata(&current) != expected
|| !descriptor.file_type().is_file()
|| !current.file_type().is_file()
|| descriptor.uid() != uid
|| current.uid() != uid
|| descriptor.mode() & 0o7777 != 0o600
|| current.mode() & 0o7777 != 0o600
|| usize::try_from(descriptor.len()).ok() != Some(expected_length)
|| current.len() != descriptor.len()
{
return Err(ControlError::InvalidPublication(
"publication identity changed type, ownership, mode, or content length",
));
}
Ok(())
}
}
pub struct PublishedControlFace {
listener: std::os::unix::net::UnixListener,
paths: ControlPaths,
socket_identity: FileIdentity,
identity_file: File,
identity_file_identity: FileIdentity,
publication_identity: ProtocolPublicationIdentity,
publication_bytes: Vec<u8>,
uid: u32,
anchor: TrustAnchor,
server_config: Arc<ServerConfig>,
registry: Arc<SessionRegistry>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
interactive_executor: Arc<dyn InteractiveSessionExecutor>,
}
pub(crate) struct AcceptedControlConnection {
socket: std::os::unix::net::UnixStream,
}
impl AcceptedControlConnection {
pub(crate) fn cancellation_handle(&self) -> std::io::Result<std::os::unix::net::UnixStream> {
self.socket.try_clone()
}
}
pub trait NonInteractiveSessionExecutor: Send + Sync {
/// # Errors
/// Returns a runtime, stream, or exact-session execution failure.
fn execute(
&self,
request: &NonInteractiveSessionRequest,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<SessionExit, ControlError>;
}
struct ExecutionUnavailable;
impl NonInteractiveSessionExecutor for ExecutionUnavailable {
fn execute(
&self,
_request: &NonInteractiveSessionRequest,
_emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<SessionExit, ControlError> {
Err(ControlError::InvalidRequest)
}
}
pub trait InteractiveControlSession: Send {
/// # Errors
/// Returns an exact-session terminal, prompt-routing, or transport failure.
fn exchange(
&mut self,
event: InteractiveSessionEvent,
) -> Result<InteractiveSessionBatch, ControlError>;
}
pub trait InteractiveSessionExecutor: Send + Sync {
/// # Errors
/// Returns an acquisition or exact-session identity failure.
fn start(
&self,
request: &InteractiveSessionRequest,
) -> Result<Box<dyn InteractiveControlSession>, ControlError>;
}
struct InteractiveExecutionUnavailable;
impl InteractiveSessionExecutor for InteractiveExecutionUnavailable {
fn start(
&self,
_request: &InteractiveSessionRequest,
) -> Result<Box<dyn InteractiveControlSession>, ControlError> {
Err(ControlError::InvalidRequest)
}
}
pub struct RunscNonInteractiveExecutor {
program: OsString,
global_arguments: Vec<OsString>,
container: OsString,
runner: Arc<dyn NonInteractiveCommandRunner>,
}
trait NonInteractiveCommandRunner: Send + Sync {
fn run(
&self,
target: &TargetCommand,
session_id: SessionId,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<ProcessExit, ControlError>;
}
struct ChildBrokerCommandRunner;
impl NonInteractiveCommandRunner for ChildBrokerCommandRunner {
fn run(
&self,
target: &TargetCommand,
session_id: SessionId,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<ProcessExit, ControlError> {
let mut child = child_broker().run_transient_captured(target)?;
let mut stdout_done = false;
let mut stderr_done = false;
while !(stdout_done && stderr_done && child.process().has_exited()?) {
let (stdout, stderr) = child.streams();
stdout_done |= emit_available(stdout, session_id, true, emit)?;
stderr_done |= emit_available(stderr, session_id, false, emit)?;
if !(stdout_done && stderr_done) {
std::thread::yield_now();
}
}
child_broker()
.reap_status(child.into_process())
.map_err(Into::into)
}
}
impl RunscNonInteractiveExecutor {
/// Derives the exact `runsc exec` prefix from the generation's attached `runsc run` command.
///
/// # Errors
/// Refuses a non-runsc command or a command without one exact run/container suffix.
pub fn from_run_command(
command: &RunscCommand,
container_id: ContainerId,
) -> Result<Self, ControlError> {
if command.program() != OsStr::new("runsc") {
return Err(ControlError::InvalidRequest);
}
let run = command
.arguments()
.iter()
.position(|argument| argument == OsStr::new("run"))
.ok_or(ControlError::InvalidRequest)?;
if run + 3 != command.arguments().len()
|| !command.arguments()[run + 1]
.as_encoded_bytes()
.starts_with(b"-bundle=")
{
return Err(ControlError::InvalidRequest);
}
let container = OsString::from(hex_lower(container_id.as_bytes()));
if command.arguments()[run + 2] != container {
return Err(ControlError::InvalidRequest);
}
Ok(Self {
program: command.program().to_owned(),
global_arguments: command.arguments()[..run].to_vec(),
container,
runner: Arc::new(ChildBrokerCommandRunner),
})
}
}
impl NonInteractiveSessionExecutor for RunscNonInteractiveExecutor {
fn execute(
&self,
request: &NonInteractiveSessionRequest,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<SessionExit, ControlError> {
let mut target = TargetCommand::new(self.program.clone());
for argument in &self.global_arguments {
target = target.argument(argument.clone());
}
target = target
.argument("exec")
.argument("-env")
.argument(format!(
"LAZARET_SESSION_ID={}",
hex_lower(request.session_id.as_bytes())
))
.argument(self.container.clone());
for argument in &request.argv {
target = target.argument(OsString::from_vec(argument.clone()));
}
let process_status = self.runner.run(&target, request.session_id, emit)?;
match (process_status.exit_code, process_status.terminating_signal) {
(Some(code), None) => Ok(SessionExit::exited(request.session_id, code)),
(None, Some(_)) => Ok(SessionExit::terminated(request.session_id)),
_ => Err(ControlError::InvalidRequest),
}
}
}
fn emit_available(
stream: &mut File,
session_id: SessionId,
stdout: bool,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<bool, ControlError> {
let mut bytes = [0_u8; 16 * 1024];
loop {
match stream.read(&mut bytes) {
Ok(0) => return Ok(true),
Ok(length) => {
let output = lazaret_core::records::NonInteractiveOutput {
session_id,
bytes: bytes[..length].to_vec(),
};
emit(if stdout {
NonInteractiveSessionMessage::Stdout(output)
} else {
NonInteractiveSessionMessage::Stderr(output)
})?;
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => return Ok(false),
Err(error) => return Err(error.into()),
}
}
}
fn hex_lower(bytes: &[u8]) -> String {
use fmt::Write as _;
bytes.iter().fold(String::new(), |mut output, byte| {
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
output
})
}

772
crates/lazaret-host-coordinator/src/control/publication.rs Normal file
View file

@ -0,0 +1,772 @@
impl PublishedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Selects blocking or polling admission for the publication listener.
///
/// # Errors
/// Returns an operating-system failure changing the listener mode.
pub fn set_nonblocking(&self, nonblocking: bool) -> Result<(), ControlError> {
self.listener.set_nonblocking(nonblocking)?;
Ok(())
}
/// Discovers one publication through the retained runtime Domain authority.
///
/// # Errors
/// Refuses changed root authority or absent, replaced, non-private, oversized, or malformed
/// publication objects.
pub fn discover_confined(
runtime_domain_root: &ConfinedPathPlan,
uid: u32,
) -> Result<(File, PathBuf, ProtocolPublicationIdentity), ControlError> {
let root = File::from(
runtime_domain_root
.open_authority()
.map_err(|_| ControlError::InvalidPath("runtime Domain authority changed"))?,
);
let root_metadata = root.metadata()?;
if !root_metadata.is_dir()
|| root_metadata.uid() != uid
|| root_metadata.mode() & 0o7777 != 0o700
{
return Err(ControlError::InvalidPath(
"runtime Domain root is not invoking-user-owned mode 0700",
));
}
let descriptor = openat(
root.as_fd(),
PUBLICATION_IDENTITY_NAME,
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(io::Error::from)?;
let mut identity = File::from(descriptor);
let identity_metadata = identity.metadata()?;
if !identity_metadata.file_type().is_file()
|| identity_metadata.uid() != uid
|| identity_metadata.mode() & 0o7777 != 0o600
|| identity_metadata.nlink() != 1
|| identity_metadata.len() > MAX_CONTROL_BYTES as u64
{
return Err(ControlError::InvalidPublication(
"publication identity is not an exact private regular file",
));
}
let socket = File::from(
openat(
root.as_fd(),
CONTROL_SOCKET_NAME,
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(io::Error::from)?,
);
let socket_metadata = socket.metadata()?;
if !socket_metadata.file_type().is_socket()
|| socket_metadata.uid() != uid
|| socket_metadata.mode() & 0o7777 != 0o600
{
return Err(ControlError::InvalidPublication(
"control socket is not an invoking-user-private socket",
));
}
let mut bytes = Vec::new();
identity.read_to_end(&mut bytes)?;
let socket_path = PathBuf::from(format!(
"/proc/self/fd/{}/{}",
root.as_raw_fd(),
CONTROL_SOCKET_NAME
));
Ok((root, socket_path, decode(&bytes)?))
}
/// Discovers one exact current control publication beneath a validated runtime Domain root.
///
/// # Errors
/// Refuses absent, replaced, non-private, oversized, or malformed publication objects.
pub fn discover(
runtime_domain_root: &Path,
uid: u32,
) -> Result<(PathBuf, ProtocolPublicationIdentity), ControlError> {
let paths = ControlPaths::open(runtime_domain_root, uid)?;
let descriptor = openat(
paths.root_file.as_fd(),
PUBLICATION_IDENTITY_NAME,
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(io::Error::from)?;
let mut identity = File::from(descriptor);
let descriptor_metadata = identity.metadata()?;
let path_metadata = std::fs::symlink_metadata(&paths.identity)?;
let expected = FileIdentity::from_metadata(&descriptor_metadata);
if FileIdentity::from_metadata(&path_metadata) != expected
|| !descriptor_metadata.file_type().is_file()
|| descriptor_metadata.uid() != uid
|| descriptor_metadata.mode() & 0o7777 != 0o600
|| descriptor_metadata.nlink() != 1
|| descriptor_metadata.len() > MAX_CONTROL_BYTES as u64
{
return Err(ControlError::InvalidPublication(
"publication identity is not an exact private regular file",
));
}
let socket = std::fs::symlink_metadata(&paths.socket)?;
if !socket.file_type().is_socket() || socket.uid() != uid || socket.mode() & 0o7777 != 0o600
{
return Err(ControlError::InvalidPublication(
"control socket is not an invoking-user-private socket",
));
}
let mut bytes = Vec::new();
identity.read_to_end(&mut bytes)?;
Ok((paths.socket, decode(&bytes)?))
}
/// Publishes one generation control face in the fixed host-private runtime path.
///
/// # Errors
/// Refuses stale generations, existing publication objects, path/ownership/mode errors, and
/// TLS configuration failures.
pub fn publish(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
store: Arc<HostCoordinatorDomainStore>,
) -> Result<Self, ControlError> {
Self::publish_with_executor(
runtime_domain_root,
uid,
anchor,
store,
Arc::new(ExecutionUnavailable),
)
}
/// Publishes one generation control face with non-interactive session execution.
///
/// # Errors
/// Refuses the same invalid publication conditions as [`Self::publish`].
pub fn publish_with_executor(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
store: Arc<HostCoordinatorDomainStore>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
) -> Result<Self, ControlError> {
let registry = Arc::new(SessionRegistry::new(
store,
anchor.generation_identity.clone(),
));
StagedControlFace::stage_with_executor(
runtime_domain_root,
uid,
anchor,
registry,
executor,
)?
.expose()
}
/// Publishes one generation control face with command and terminal execution.
///
/// # Errors
/// Refuses the same invalid publication conditions as [`Self::publish`].
pub fn publish_with_executors(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
store: Arc<HostCoordinatorDomainStore>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
interactive_executor: Arc<dyn InteractiveSessionExecutor>,
) -> Result<Self, ControlError> {
let registry = Arc::new(SessionRegistry::new(
store,
anchor.generation_identity.clone(),
));
StagedControlFace::stage_with_executors(
runtime_domain_root,
uid,
anchor,
registry,
executor,
interactive_executor,
)?
.expose()
}
#[must_use]
pub fn publication_identity(&self) -> &ProtocolPublicationIdentity {
&self.publication_identity
}
#[must_use]
pub fn registry(&self) -> Arc<SessionRegistry> {
Arc::clone(&self.registry)
}
#[must_use]
pub fn socket_path(&self) -> &Path {
&self.paths.socket
}
#[must_use]
pub fn identity_path(&self) -> &Path {
&self.paths.identity
}
/// Withdraws exact discovery and connection paths while retaining live accepted streams.
///
/// # Errors
/// Refuses changed artifacts, removal failures, or artifacts that remain after removal.
pub fn withdraw(&self) -> Result<(), ControlError> {
remove_exact_if_identity_matches(&self.paths.socket, self.socket_identity)?;
remove_exact_if_identity_matches(&self.paths.identity, self.identity_file_identity)
}
/// Accepts and serves one TLS connection. Registered connections remain live until EOF.
///
/// # Errors
/// Refuses peer UID, publication, TLS, ALPN, certificate, framing, tuple, duplicate, and
/// persistence failures.
pub fn serve_next(&self) -> Result<(), ControlError> {
self.serve_next_coordinated(
|admission| {
self.registry
.register(admission.request, &admission.credential)
.map(|_| ())
},
|session_id| self.registry.disconnect(session_id),
)
}
/// Serves one connection while routing authenticated session state through its lifecycle.
///
/// # Errors
/// Refuses the same conditions as [`Self::serve_next`] plus lifecycle admission or disconnect
/// failure.
pub fn serve_next_coordinated<Admit, Disconnect>(
&self,
admit: Admit,
disconnect: Disconnect,
) -> Result<(), ControlError>
where
Admit: FnMut(ControlSessionAdmission) -> Result<(), ControlError>,
Disconnect: FnMut(SessionId) -> Result<(), ControlError>,
{
self.serve_next_coordinated_with_registration(admit, |_, _| Ok(()), disconnect)
}
pub(crate) fn serve_next_coordinated_with_registration<Admit, Registered, Disconnect>(
&self,
admit: Admit,
registered: Registered,
disconnect: Disconnect,
) -> Result<(), ControlError>
where
Admit: FnMut(ControlSessionAdmission) -> Result<(), ControlError>,
Registered: FnMut(SessionBinding, SessionRegistrationResponse) -> Result<(), ControlError>,
Disconnect: FnMut(SessionId) -> Result<(), ControlError>,
{
let connection = self.accept_connection()?;
self.serve_connection_coordinated(connection, admit, registered, disconnect)
}
pub(crate) fn accept_connection(&self) -> Result<AcceptedControlConnection, ControlError> {
let (socket, _) = self.listener.accept()?;
Ok(AcceptedControlConnection { socket })
}
pub(crate) fn serve_connection_coordinated<Admit, Registered, Disconnect>(
&self,
connection: AcceptedControlConnection,
mut admit: Admit,
mut registered: Registered,
mut disconnect: Disconnect,
) -> Result<(), ControlError>
where
Admit: FnMut(ControlSessionAdmission) -> Result<(), ControlError>,
Registered: FnMut(SessionBinding, SessionRegistrationResponse) -> Result<(), ControlError>,
Disconnect: FnMut(SessionId) -> Result<(), ControlError>,
{
let socket = connection.socket;
let peer = socket_peercred(&socket).map_err(io::Error::from)?;
require_peer_uid(peer.uid.as_raw(), self.uid)?;
self.validate_publication()?;
let mut connection = ServerConnection::new(Arc::clone(&self.server_config))?;
let mut socket = socket;
while connection.is_handshaking() {
connection.complete_io(&mut socket)?;
}
if connection.alpn_protocol() != Some(CONTROL_ALPN) {
return Err(ControlError::InvalidRequest);
}
let peer_certificate = connection.peer_certificates().map(|certificates| {
certificates
.iter()
.map(|certificate| certificate.to_vec())
.collect::<Vec<_>>()
});
let mut stream = rustls::StreamOwned::new(connection, socket);
match peer_certificate {
None => self.serve_enrollment(&mut stream),
Some(certificates) => self.serve_registration(
&mut stream,
&certificates,
&mut admit,
&mut registered,
&mut disconnect,
),
}
}
fn serve_enrollment(
&self,
stream: &mut rustls::StreamOwned<ServerConnection, std::os::unix::net::UnixStream>,
) -> Result<(), ControlError> {
let request: ControlRequest = read_frame(stream, MAX_ENROLLMENT_BYTES)?;
let ControlRequest::Enroll(request) = request else {
return Err(ControlError::InvalidRequest);
};
let credential = self.anchor.issue_client_credential(&request)?;
self.registry.enroll(credential.clone())?;
write_frame(
stream,
&ControlResponse::Enrolled(ClientEnrollmentResponse {
protocol_version: ProtocolVersion::V1,
credential,
}),
)
}
fn serve_registration<Admit, Registered, Disconnect>(
&self,
stream: &mut rustls::StreamOwned<ServerConnection, std::os::unix::net::UnixStream>,
certificates: &[Vec<u8>],
admit: &mut Admit,
registered: &mut Registered,
disconnect: &mut Disconnect,
) -> Result<(), ControlError>
where
Admit: FnMut(ControlSessionAdmission) -> Result<(), ControlError>,
Registered: FnMut(SessionBinding, SessionRegistrationResponse) -> Result<(), ControlError>,
Disconnect: FnMut(SessionId) -> Result<(), ControlError>,
{
let [leaf] = certificates else {
return Err(ControlError::InvalidCertificateProfile);
};
let credential = self.registry.credential_for_leaf(leaf)?;
let request: ControlRequest = read_frame(stream, MAX_CONTROL_BYTES)?;
let ControlRequest::Register(request) = request else {
return Err(ControlError::InvalidRequest);
};
let capability = issue_session_capability(
request.session_binding.clone(),
self.anchor.coordinator_protocol_credential.credential_id,
self.anchor.sandbox_protocol_credential.credential_id,
&self.anchor.coordinator_protocol_key,
)?;
let session_id = request.session_binding.session_id;
let binding = request.session_binding.clone();
admit(ControlSessionAdmission {
request,
credential,
})?;
let result = (|| {
let registration = SessionRegistrationResponse {
protocol_version: ProtocolVersion::V1,
session_capability: capability,
};
registered(binding, registration.clone())?;
write_frame(stream, &ControlResponse::Registered(registration))?;
let request: ControlRequest = match read_frame(stream, MAX_CONTROL_BYTES) {
Ok(request) => request,
Err(ControlError::Io(error))
if matches!(
error.kind(),
io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset
) =>
{
return Ok(());
}
Err(error) => return Err(error),
};
match request {
ControlRequest::Execute(request) => {
validate_command(request.session_id, session_id, &request.argv)?;
let mut send_message = |message: NonInteractiveSessionMessage| {
if matches!(message, NonInteractiveSessionMessage::Exit(_)) {
return Err(ControlError::InvalidRequest);
}
require_non_interactive_session(&message, request.session_id)?;
write_frame(stream, &message)
};
let outcome = self.executor.execute(&request, &mut send_message)?;
require_session_id(outcome.session_id(), request.session_id)?;
write_frame(stream, &NonInteractiveSessionMessage::Exit(outcome))
}
ControlRequest::ExecuteInteractive(request) => {
validate_command(request.session_id, session_id, &request.argv)?;
if request.initial_size.session_id != session_id {
return Err(ControlError::InvalidRequest);
}
let mut terminal = self.interactive_executor.start(&request)?;
let mut event = InteractiveSessionEvent::Poll { session_id };
loop {
let batch = terminal.exchange(event)?;
require_interactive_batch(&batch, session_id)?;
let complete = batch.messages.iter().any(|message| {
matches!(
message,
InteractiveSessionMessage::Terminal(
lazaret_core::records::SessionTerminalMessage::Exit(_)
)
)
});
write_frame(stream, &batch)?;
if complete {
break Ok(());
}
event = read_frame(stream, MAX_CONTROL_BYTES)?;
require_interactive_event(&event, session_id)?;
}
}
ControlRequest::Enroll(_) | ControlRequest::Register(_) => {
Err(ControlError::InvalidRequest)
}
}
})();
let disconnected = disconnect(session_id);
result.and(disconnected)
}
fn validate_publication(&self) -> Result<(), ControlError> {
self.paths.validate_root(self.uid)?;
let socket = std::fs::symlink_metadata(&self.paths.socket)?;
let identity = std::fs::symlink_metadata(&self.paths.identity)?;
if FileIdentity::from_metadata(&socket) != self.socket_identity
|| !socket.file_type().is_socket()
|| socket.uid() != self.uid
|| socket.mode() & 0o7777 != 0o600
|| FileIdentity::from_metadata(&identity) != self.identity_file_identity
|| !identity.file_type().is_file()
|| identity.uid() != self.uid
|| identity.mode() & 0o7777 != 0o600
|| usize::try_from(self.identity_file.metadata()?.len()).ok()
!= Some(self.publication_bytes.len())
{
return Err(ControlError::InvalidPublication(
"publication objects changed identity, ownership, or mode",
));
}
let mut bytes = vec![0; self.publication_bytes.len()];
self.identity_file.read_exact_at(&mut bytes, 0)?;
if bytes != self.publication_bytes {
return Err(ControlError::InvalidPublication(
"publication identity bytes changed",
));
}
self.registry.require_serviceable_generation()
}
/// Creates a TLS client configuration pinned to this publication identity.
///
/// # Errors
/// Refuses malformed CA or client certificate/key material.
pub fn client_config(
publication: &ProtocolPublicationIdentity,
client: Option<(&TlsCredential, PrivateKeyDer<'static>)>,
) -> Result<ClientConfig, ControlError> {
let mut roots = RootCertStore::empty();
roots.add(CertificateDer::from(
publication.tls_ca_certificate_der.clone(),
))?;
let builder = ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13])
.with_root_certificates(roots);
let mut config = if let Some((credential, private_key)) = client {
verify_tls_credential(
credential,
&publication.generation_identity,
TlsRole::ClientCli,
credential.subject_id(),
)?;
builder.with_client_auth_cert(
vec![CertificateDer::from(credential.certificate_der.clone())],
private_key,
)?
} else {
builder.with_no_client_auth()
};
config.alpn_protocols = vec![CONTROL_ALPN.to_vec()];
config.enable_early_data = false;
config.resumption = rustls::client::Resumption::disabled();
Ok(config)
}
/// Connects a rustls stream to the fixed control socket using an existing configuration.
///
/// # Errors
/// Returns path, connection, or TLS initialization failures.
pub fn connect(
socket_path: &Path,
config: ClientConfig,
publication: &ProtocolPublicationIdentity,
) -> Result<rustls::StreamOwned<ClientConnection, std::os::unix::net::UnixStream>, ControlError>
{
let socket = std::os::unix::net::UnixStream::connect(socket_path)?;
let server_name = ServerName::try_from(CONTROL_SERVER_NAME)
.map_err(|_| ControlError::InvalidCertificateProfile)?;
let mut connection = ClientConnection::new(Arc::new(config), server_name)?;
let mut socket = socket;
while connection.is_handshaking() {
connection.complete_io(&mut socket)?;
}
let [certificate] = connection.peer_certificates().unwrap_or_default() else {
return Err(ControlError::InvalidCertificateProfile);
};
if connection.protocol_version() != Some(rustls::ProtocolVersion::TLSv1_3)
|| connection.alpn_protocol() != Some(CONTROL_ALPN)
|| certificate_digest(certificate) != publication.coordinator_tls_certificate_sha256
{
return Err(ControlError::InvalidCertificateProfile);
}
Ok(rustls::StreamOwned::new(connection, socket))
}
/// Writes one length-delimited control record.
///
/// # Errors
/// Returns encoding or TLS I/O failures.
pub fn send<Request: serde::Serialize>(
stream: &mut impl Write,
request: &Request,
) -> Result<(), ControlError> {
write_frame(stream, request)
}
/// Reads one length-delimited control record.
///
/// # Errors
/// Returns framing, size, decoding, or TLS I/O failures.
pub fn receive<Response: serde::de::DeserializeOwned + serde::Serialize>(
stream: &mut impl Read,
) -> Result<Response, ControlError> {
read_frame(stream, MAX_CONTROL_BYTES)
}
}
impl Drop for PublishedControlFace {
fn drop(&mut self) {
remove_if_identity_matches(&self.paths.socket, self.socket_identity);
remove_if_identity_matches(&self.paths.identity, self.identity_file_identity);
}
}
fn remove_exact_if_identity_matches(
path: &Path,
expected: FileIdentity,
) -> Result<(), ControlError> {
let metadata = match std::fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
};
if FileIdentity::from_metadata(&metadata) != expected {
return Err(ControlError::InvalidPublication(
"publication artifact identity changed before withdrawal",
));
}
std::fs::remove_file(path)?;
match std::fs::symlink_metadata(path) {
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
Ok(_) => Err(ControlError::InvalidPublication(
"publication artifact remained after withdrawal",
)),
}
}
fn remove_if_identity_matches(path: &Path, expected: FileIdentity) {
if std::fs::symlink_metadata(path)
.ok()
.is_some_and(|metadata| FileIdentity::from_metadata(&metadata) == expected)
{
let _ = std::fs::remove_file(path);
}
}
fn require_non_interactive_session(
message: &NonInteractiveSessionMessage,
expected: SessionId,
) -> Result<(), ControlError> {
let actual = match message {
NonInteractiveSessionMessage::Stdout(output)
| NonInteractiveSessionMessage::Stderr(output) => output.session_id,
NonInteractiveSessionMessage::Exit(exit) => exit.session_id(),
};
if actual == expected {
Ok(())
} else {
Err(ControlError::InvalidRequest)
}
}
fn validate_command(
actual: SessionId,
expected: SessionId,
argv: &[Vec<u8>],
) -> Result<(), ControlError> {
require_session_id(actual, expected)?;
if argv.is_empty() || argv[0].is_empty() || argv.iter().any(|argument| argument.contains(&0)) {
return Err(ControlError::InvalidRequest);
}
Ok(())
}
fn require_session_id(actual: SessionId, expected: SessionId) -> Result<(), ControlError> {
if actual == expected {
Ok(())
} else {
Err(ControlError::InvalidRequest)
}
}
fn require_interactive_event(
event: &InteractiveSessionEvent,
expected: SessionId,
) -> Result<(), ControlError> {
let actual = match event {
InteractiveSessionEvent::Terminal(event) => match event {
lazaret_core::records::SessionTerminalEvent::Input(event) => event.session_id,
lazaret_core::records::SessionTerminalEvent::Resize(event) => event.session_id,
lazaret_core::records::SessionTerminalEvent::Signal(event) => event.session_id,
lazaret_core::records::SessionTerminalEvent::Close(event) => event.session_id,
},
InteractiveSessionEvent::PromptDecision { session_id, .. }
| InteractiveSessionEvent::Poll { session_id } => *session_id,
};
require_session_id(actual, expected)
}
fn require_interactive_batch(
batch: &InteractiveSessionBatch,
expected: SessionId,
) -> Result<(), ControlError> {
require_session_id(batch.session_id, expected)?;
for message in &batch.messages {
let actual = match message {
InteractiveSessionMessage::Terminal(message) => match message {
lazaret_core::records::SessionTerminalMessage::Output(output) => output.session_id,
lazaret_core::records::SessionTerminalMessage::Exit(exit) => exit.session_id(),
},
InteractiveSessionMessage::Prompt(prompt) => prompt.session_id,
};
require_session_id(actual, expected)?;
}
Ok(())
}
fn leaf_parameters(
common_name: &str,
usage: ExtendedKeyUsagePurpose,
now: OffsetDateTime,
) -> Result<CertificateParams, ControlError> {
let mut params = CertificateParams::new(vec![common_name.to_owned()])?;
params.distinguished_name = DistinguishedName::new();
params
.distinguished_name
.push(DnType::CommonName, common_name);
params.not_before = now - Duration::minutes(1);
params.not_after = now + Duration::hours(1);
params.is_ca = IsCa::NoCa;
params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
params.extended_key_usages = vec![usage];
Ok(params)
}
fn random_signing_key() -> Result<SigningKey, ControlError> {
let mut bytes = [0_u8; 32];
getrandom::fill(&mut bytes).map_err(|_| ControlError::InvalidCertificateProfile)?;
Ok(SigningKey::from_bytes(&bytes))
}
fn enrollment_binding(
generation_identity: &GenerationIdentity,
cli_id: ClientId,
) -> Result<String, ControlError> {
let encoded = encode(&(generation_identity, cli_id))?;
Ok(format!("lazaret-client-{}", hex(&sha256(&encoded))))
}
fn certificate_digest(certificate: impl AsRef<[u8]>) -> CertificateDigest {
CertificateDigest::new(sha256(certificate.as_ref()))
}
fn sha256(bytes: &[u8]) -> [u8; 32] {
Sha256::digest(bytes).into()
}
fn short_identity(bytes: &[u8; 32]) -> String {
hex(&bytes[..8])
}
fn hex(bytes: &[u8]) -> String {
let mut encoded = String::with_capacity(bytes.len() * 2);
for byte in bytes {
use fmt::Write as _;
let _ = write!(encoded, "{byte:02x}");
}
encoded
}
fn require_peer_uid(actual: u32, expected: u32) -> Result<(), ControlError> {
if actual == expected {
Ok(())
} else {
Err(ControlError::UnauthorizedPeer)
}
}
fn is_live_session(phase: SessionPhase) -> bool {
matches!(
phase,
SessionPhase::Registering
| SessionPhase::Attached
| SessionPhase::Prompting
| SessionPhase::HostExecuting
)
}
fn read_frame<Record: serde::de::DeserializeOwned + serde::Serialize>(
reader: &mut impl Read,
maximum: usize,
) -> Result<Record, ControlError> {
let mut length = [0_u8; 4];
reader.read_exact(&mut length)?;
let length = u32::from_be_bytes(length) as usize;
if length > maximum {
return Err(ControlError::OversizeRequest { length, maximum });
}
let mut bytes = vec![0_u8; length];
reader.read_exact(&mut bytes)?;
decode(&bytes).map_err(Into::into)
}
fn write_frame<Record: serde::Serialize>(
writer: &mut impl Write,
record: &Record,
) -> Result<(), ControlError> {
let bytes = encode(record)?;
let length = u32::try_from(bytes.len())
.map_err(|_| ControlError::OversizeRequest {
length: bytes.len(),
maximum: u32::MAX as usize,
})?
.to_be_bytes();
writer.write_all(&length)?;
writer.write_all(&bytes)?;
writer.flush()?;
Ok(())
}

236
crates/lazaret-host-coordinator/src/control/staging.rs Normal file
View file

@ -0,0 +1,236 @@
pub struct StagedControlFace {
listener: Option<std::os::unix::net::UnixListener>,
paths: ControlPaths,
staged_socket_identity: FileIdentity,
identity_file: File,
identity_file_identity: FileIdentity,
publication_identity: ProtocolPublicationIdentity,
publication_bytes: Vec<u8>,
uid: u32,
anchor: Option<TrustAnchor>,
server_config: Arc<ServerConfig>,
registry: Arc<SessionRegistry>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
interactive_executor: Arc<dyn InteractiveSessionExecutor>,
}
impl StagedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Stages an undiscoverable private control face before durable publication.
///
/// # Errors
/// Refuses existing artifacts, changed paths, ownership/mode errors, or TLS failure.
pub fn stage(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
registry: Arc<SessionRegistry>,
) -> Result<Self, ControlError> {
Self::stage_with_executor(
runtime_domain_root,
uid,
anchor,
registry,
Arc::new(ExecutionUnavailable),
)
}
/// Stages a control face with its generation-bound non-interactive executor.
///
/// # Errors
/// Refuses the same invalid staging conditions as [`Self::stage`].
pub fn stage_with_executor(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
registry: Arc<SessionRegistry>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
) -> Result<Self, ControlError> {
Self::stage_with_executors(
runtime_domain_root,
uid,
anchor,
registry,
executor,
Arc::new(InteractiveExecutionUnavailable),
)
}
/// Stages a control face with generation-bound command and terminal execution.
///
/// # Errors
/// Refuses the same invalid staging conditions as [`Self::stage`].
pub fn stage_with_executors(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
registry: Arc<SessionRegistry>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
interactive_executor: Arc<dyn InteractiveSessionExecutor>,
) -> Result<Self, ControlError> {
if registry.generation_identity != anchor.generation_identity {
return Err(ControlError::StaleGeneration);
}
let paths = ControlPaths::open(runtime_domain_root, uid)?;
if [
&paths.socket,
&paths.identity,
&paths.staged_socket,
&paths.staged_identity,
]
.into_iter()
.any(|path| path.try_exists().unwrap_or(true))
{
return Err(ControlError::InvalidPublication(
"publication path already exists",
));
}
let publication_identity = anchor.publication_identity();
let publication_bytes = encode(&publication_identity)?;
let server_config = Arc::new(anchor.server_config()?);
let (identity_file, identity_file_identity) = paths.create_named_identity(
STAGED_PUBLICATION_IDENTITY_NAME,
&publication_bytes,
uid,
)?;
let listener = std::os::unix::net::UnixListener::bind(&paths.staged_socket)
.map_err(|_| ControlError::InvalidPublication("staged control socket bind failed"))?;
std::fs::set_permissions(&paths.staged_socket, std::fs::Permissions::from_mode(0o600))?;
let socket_metadata = std::fs::symlink_metadata(&paths.staged_socket)?;
let staged_socket_identity = FileIdentity::from_metadata(&socket_metadata);
if !socket_metadata.file_type().is_socket()
|| socket_metadata.uid() != uid
|| socket_metadata.mode() & 0o7777 != 0o600
{
return Err(ControlError::InvalidPublication(
"staged control socket ownership or mode is invalid",
));
}
paths.root_file.sync_all()?;
Ok(Self {
listener: Some(listener),
paths,
staged_socket_identity,
identity_file,
identity_file_identity,
publication_identity,
publication_bytes,
uid,
anchor: Some(anchor),
server_config,
registry,
executor,
interactive_executor,
})
}
/// Exposes the staged face after the exact generation is durably Published.
///
/// # Errors
/// Refuses stale durable state, changed staged artifacts, or exposure failure.
pub fn expose(mut self) -> Result<PublishedControlFace, ControlError> {
self.registry.require_serviceable_generation()?;
self.paths.validate_root(self.uid)?;
let staged_socket = std::fs::symlink_metadata(&self.paths.staged_socket)?;
if FileIdentity::from_metadata(&staged_socket) != self.staged_socket_identity {
return Err(ControlError::InvalidPublication(
"staged control socket changed",
));
}
self.paths.validate_identity_at(
&self.paths.staged_identity,
&self.identity_file,
self.identity_file_identity,
self.uid,
self.publication_bytes.len(),
)?;
std::fs::rename(&self.paths.staged_socket, &self.paths.socket)?;
if let Err(error) = std::fs::rename(&self.paths.staged_identity, &self.paths.identity) {
remove_if_identity_matches(&self.paths.socket, self.staged_socket_identity);
return Err(error.into());
}
self.paths.root_file.sync_all()?;
let socket_identity =
FileIdentity::from_metadata(&std::fs::symlink_metadata(&self.paths.socket)?);
self.paths.validate_identity(
&self.identity_file,
self.identity_file_identity,
self.uid,
self.publication_bytes.len(),
)?;
let paths = ControlPaths::open(&self.paths.root, self.uid)?;
Ok(PublishedControlFace {
listener: self
.listener
.take()
.ok_or(ControlError::InvalidPublication(
"staged listener is absent",
))?,
paths,
socket_identity,
identity_file: self.identity_file.try_clone()?,
identity_file_identity: self.identity_file_identity,
publication_identity: self.publication_identity.clone(),
publication_bytes: self.publication_bytes.clone(),
uid: self.uid,
anchor: self.anchor.take().ok_or(ControlError::InvalidPublication(
"staged Trust Anchor is absent",
))?,
server_config: Arc::clone(&self.server_config),
registry: Arc::clone(&self.registry),
executor: Arc::clone(&self.executor),
interactive_executor: Arc::clone(&self.interactive_executor),
})
}
}
impl Drop for StagedControlFace {
fn drop(&mut self) {
remove_if_identity_matches(&self.paths.staged_socket, self.staged_socket_identity);
remove_if_identity_matches(&self.paths.staged_identity, self.identity_file_identity);
}
}
fn publication_recovery_ownership(
paths: &ControlPaths,
) -> Result<PublicationOwnership, ControlError> {
let socket_path = if paths.staged_socket.exists() {
&paths.staged_socket
} else {
&paths.socket
};
let identity_path = if paths.staged_identity.exists() {
&paths.staged_identity
} else {
&paths.identity
};
Ok(PublicationOwnership {
staged_socket_path: Some(HostPath::new(
STAGED_CONTROL_SOCKET_NAME.as_bytes().to_vec(),
)),
staged_identity_path: Some(HostPath::new(
STAGED_PUBLICATION_IDENTITY_NAME.as_bytes().to_vec(),
)),
socket_path: Some(HostPath::new(CONTROL_SOCKET_NAME.as_bytes().to_vec())),
identity_path: Some(HostPath::new(PUBLICATION_IDENTITY_NAME.as_bytes().to_vec())),
socket_identity: Some(publication_path_identity(paths, socket_path)?),
identity_file_identity: Some(publication_path_identity(paths, identity_path)?),
})
}
fn publication_path_identity(
paths: &ControlPaths,
path: &Path,
) -> Result<MountIdentity, ControlError> {
let metadata = std::fs::symlink_metadata(path)?;
let root = mount_identity(&paths.root_file)?;
Ok(MountIdentity {
mount_id: root.mount_id,
device: DeviceId::new(metadata.dev()),
inode: InodeId::new(metadata.ino()),
})
}

193
crates/lazaret-host-coordinator/src/control/tests.rs Normal file
View file

@ -0,0 +1,193 @@
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use std::sync::Arc;
use std::thread;
use lazaret_core::identity::{
ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, SandboxId,
};
use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName};
use super::*;
fn generation() -> GenerationIdentity {
GenerationIdentity {
agent_profile: "trusted".into(),
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(7),
configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
coordinator_id: CoordinatorId::new([3; 32]),
sandbox_id: SandboxId::new([4; 32]),
}
}
fn client_config_with_certificate(
anchor: &TrustAnchor,
certificate: CertificateDer<'static>,
private_key: PrivatePkcs8KeyDer<'static>,
) -> ClientConfig {
let mut roots = RootCertStore::empty();
roots.add(anchor.tls_ca_der.clone()).unwrap();
let mut config = ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13])
.with_root_certificates(roots)
.with_client_auth_cert(vec![certificate], PrivateKeyDer::Pkcs8(private_key))
.unwrap();
config.alpn_protocols = vec![CONTROL_ALPN.to_vec()];
config.resumption = rustls::client::Resumption::disabled();
config
}
fn handshake_pair(
server_config: ServerConfig,
client_config: ClientConfig,
) -> (Result<(), rustls::Error>, Result<(), rustls::Error>) {
let (mut server_socket, mut client_socket) =
std::os::unix::net::UnixStream::pair().unwrap();
let server = thread::spawn(move || {
let mut connection = ServerConnection::new(Arc::new(server_config)).unwrap();
while connection.is_handshaking() {
connection
.complete_io(&mut server_socket)
.map_err(|error| rustls::Error::General(error.to_string()))?;
}
Ok(())
});
let mut connection = ClientConnection::new(
Arc::new(client_config),
ServerName::try_from(CONTROL_SERVER_NAME).unwrap(),
)
.unwrap();
let client = (|| {
while connection.is_handshaking() {
connection
.complete_io(&mut client_socket)
.map_err(|error| rustls::Error::General(error.to_string()))?;
}
Ok(())
})();
(server.join().unwrap(), client)
}
#[test]
fn exact_peer_uid_is_required() {
assert!(require_peer_uid(1_000, 1_000).is_ok());
assert!(matches!(
require_peer_uid(1_001, 1_000),
Err(ControlError::UnauthorizedPeer)
));
}
struct RecordingRunner {
observed: Mutex<Option<(OsString, Vec<OsString>)>>,
exit: ProcessExit,
}
impl NonInteractiveCommandRunner for RecordingRunner {
fn run(
&self,
target: &TargetCommand,
_session_id: SessionId,
_emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<ProcessExit, ControlError> {
*self.observed.lock().unwrap() =
Some((target.program().to_owned(), target.arguments().to_vec()));
Ok(self.exit)
}
}
#[test]
fn runsc_exec_uses_the_command_as_the_first_post_container_argument() {
let runner = Arc::new(RecordingRunner {
observed: Mutex::new(None),
exit: ProcessExit {
exit_code: Some(23),
terminating_signal: None,
},
});
let executor = RunscNonInteractiveExecutor {
program: "runsc".into(),
global_arguments: vec!["--root=/runtime".into()],
container: "container-id".into(),
runner: runner.clone(),
};
let session_id = SessionId::new([9; 32]);
let exit = executor
.execute(
&NonInteractiveSessionRequest {
protocol_version: ProtocolVersion::V1,
session_id,
argv: vec![b"printf".to_vec(), b"hello".to_vec()],
},
&mut |_| Ok(()),
)
.unwrap();
assert_eq!(exit.exit_code(), Some(23));
assert_eq!(
runner.observed.lock().unwrap().as_ref().unwrap(),
&(
OsString::from("runsc"),
vec![
OsString::from("--root=/runtime"),
OsString::from("exec"),
OsString::from("-env"),
OsString::from(format!(
"LAZARET_SESSION_ID={}",
hex_lower(session_id.as_bytes())
)),
OsString::from("container-id"),
OsString::from("printf"),
OsString::from("hello"),
]
)
);
}
#[test]
fn rustls_refuses_client_leaf_with_server_role() {
let anchor = TrustAnchor::create(generation()).unwrap();
let key = KeyPair::generate_for(&PKCS_ED25519).unwrap();
let certificate = leaf_parameters(
"wrong-client-role.invalid",
ExtendedKeyUsagePurpose::ServerAuth,
OffsetDateTime::now_utc(),
)
.unwrap()
.signed_by(&key, &anchor.tls_ca)
.unwrap();
let client_config = client_config_with_certificate(
&anchor,
certificate.der().clone(),
PrivatePkcs8KeyDer::from(key.serialize_der()),
);
let (server, client) = handshake_pair(anchor.server_config().unwrap(), client_config);
assert!(server.is_err() || client.is_err());
}
#[test]
fn rustls_refuses_server_leaf_with_client_role() {
let anchor = TrustAnchor::create(generation()).unwrap();
let key = KeyPair::generate_for(&PKCS_ED25519).unwrap();
let certificate = leaf_parameters(
CONTROL_SERVER_NAME,
ExtendedKeyUsagePurpose::ClientAuth,
OffsetDateTime::now_utc(),
)
.unwrap()
.signed_by(&key, &anchor.tls_ca)
.unwrap();
let server_config =
ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13])
.with_no_client_auth()
.with_single_cert(
vec![certificate.der().clone()],
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(key.serialize_der())),
)
.unwrap();
let client_config =
PublishedControlFace::client_config(&anchor.publication_identity(), None).unwrap();
let (server, client) = handshake_pair(server_config, client_config);
assert!(server.is_err() || client.is_err());
}
}

475
crates/lazaret-host-coordinator/src/escalation.rs
View file

@ -986,476 +986,5 @@ impl EscalationCoordinator {
}
}
struct SessionAuthorityFiles {
binding_name: OsString,
binding_identity: MailboxEntryIdentity,
registration_name: OsString,
registration_identity: MailboxEntryIdentity,
}
struct SessionEscalationAuthority {
session: VerifiedSession,
files: SessionAuthorityFiles,
}
struct RegistrationCommand {
binding: SessionBinding,
registration: SessionRegistrationResponse,
completed: mpsc::SyncSender<Result<(), String>>,
}
enum EscalationServiceCommand {
Register(Box<RegistrationCommand>),
Disconnect {
session_id: SessionId,
completed: mpsc::SyncSender<Result<(), String>>,
},
Stop,
}
#[derive(Clone)]
pub(crate) struct EscalationServiceHandle {
commands: mpsc::Sender<EscalationServiceCommand>,
}
impl EscalationServiceHandle {
pub(crate) fn register(
&self,
binding: SessionBinding,
registration: SessionRegistrationResponse,
) -> Result<(), EscalationError> {
let (completed, result) = mpsc::sync_channel(1);
self.commands
.send(EscalationServiceCommand::Register(Box::new(
RegistrationCommand {
binding,
registration,
completed,
},
)))
.map_err(|_| io::Error::other("escalation service stopped"))?;
result
.recv()
.map_err(|_| io::Error::other("escalation registration was abandoned"))?
.map_err(|error| io::Error::other(error).into())
}
pub(crate) fn disconnect(&self, session_id: SessionId) -> Result<(), EscalationError> {
let (completed, result) = mpsc::sync_channel(1);
self.commands
.send(EscalationServiceCommand::Disconnect {
session_id,
completed,
})
.map_err(|_| io::Error::other("escalation service stopped"))?;
result
.recv()
.map_err(|_| io::Error::other("escalation disconnect was abandoned"))?
.map_err(|error| io::Error::other(error).into())
}
pub(crate) fn stop(&self) {
let _ = self.commands.send(EscalationServiceCommand::Stop);
}
}
pub(crate) struct EscalationService<'a> {
configuration: &'a ResolvedDomainConfiguration,
publication: ProtocolPublicationIdentity,
credentials: SandboxCredentialBundle,
credential_files: RegularFileMailbox,
inbox: RegularFileMailbox,
prompt_transport: Arc<SessionPromptTransport>,
coordinator: EscalationCoordinator,
ownership: LocalExecutionOwnership,
sessions: BTreeMap<SessionId, SessionEscalationAuthority>,
active: BTreeMap<RequestId, SessionId>,
commands: mpsc::Receiver<EscalationServiceCommand>,
}
pub(crate) struct EscalationServiceInputs<'a> {
pub configuration: &'a ResolvedDomainConfiguration,
pub publication: ProtocolPublicationIdentity,
pub credentials: SandboxCredentialBundle,
pub credential_files: RegularFileMailbox,
pub inbox: RegularFileMailbox,
pub outbox: RegularFileMailbox,
pub prompt_transport: Arc<SessionPromptTransport>,
pub coordinator_key: SigningKey,
pub ownership: ProcessOwnershipContext,
pub invoking_uid: u32,
}
impl<'a> EscalationService<'a> {
pub(crate) fn new(
inputs: EscalationServiceInputs<'a>,
) -> Result<(Self, EscalationServiceHandle), EscalationError> {
inputs.credential_files.publish(
OsStr::new("publication.postcard"),
&lazaret_core::codec::encode(&inputs.publication)
.map_err(|error| io::Error::other(error.to_string()))?,
)?;
inputs.credential_files.publish(
OsStr::new("sandbox-credentials.postcard"),
&lazaret_core::codec::encode(&inputs.credentials)
.map_err(|error| io::Error::other(error.to_string()))?,
)?;
let (commands, receiver) = mpsc::channel();
Ok((
Self {
configuration: inputs.configuration,
publication: inputs.publication,
credentials: inputs.credentials,
credential_files: inputs.credential_files,
inbox: inputs.inbox,
prompt_transport: inputs.prompt_transport,
coordinator: EscalationCoordinator::new(
inputs.coordinator_key,
inputs.outbox,
inputs.ownership,
inputs.invoking_uid,
Duration::from_secs(30),
Duration::from_secs(2),
),
ownership: LocalExecutionOwnership::default(),
sessions: BTreeMap::new(),
active: BTreeMap::new(),
commands: receiver,
},
EscalationServiceHandle { commands },
))
}
pub(crate) fn run(mut self) -> Result<(), EscalationError> {
loop {
while let Ok(command) = self.commands.try_recv() {
if self.apply_command(command)? {
return Ok(());
}
}
self.process_requests()?;
self.process_active()?;
std::thread::sleep(Duration::from_millis(5));
}
}
fn apply_command(
&mut self,
command: EscalationServiceCommand,
) -> Result<bool, EscalationError> {
match command {
EscalationServiceCommand::Register(command) => {
let RegistrationCommand {
binding,
registration,
completed,
} = *command;
let result = self
.register(&binding, &registration)
.map_err(|error| error.to_string());
let failed = result.is_err();
let _ = completed.send(result);
if failed {
return Err(io::Error::other("session escalation registration failed").into());
}
Ok(false)
}
EscalationServiceCommand::Disconnect {
session_id,
completed,
} => {
let result = self
.disconnect(session_id)
.map_err(|error| error.to_string());
let failed = result.is_err();
let _ = completed.send(result);
if failed {
return Err(io::Error::other("session escalation cleanup failed").into());
}
Ok(false)
}
EscalationServiceCommand::Stop => {
let sessions = self.sessions.keys().copied().collect::<Vec<_>>();
for session_id in sessions {
self.disconnect(session_id)?;
}
Ok(true)
}
}
}
fn register(
&mut self,
binding: &SessionBinding,
registration: &SessionRegistrationResponse,
) -> Result<(), EscalationError> {
let session_id = binding.session_id;
if self.sessions.contains_key(&session_id) {
return Err(EscalationError::DuplicateSessionRequest);
}
let session = VerifiedSession::new(
self.publication.trust_manifest_sha256,
self.credentials.trust_manifest.tls_ca_sha256,
&self.credentials.trust_manifest,
binding.clone(),
self.credentials.coordinator_protocol_credential(),
self.credentials.sandbox_protocol_credential(),
registration.session_capability.clone(),
)?;
let prefix = mailbox_session_name(session_id);
let mut binding_name = prefix.clone();
binding_name.push("-binding.postcard");
let mut registration_name = prefix;
registration_name.push("-registration.postcard");
let binding_identity = self.credential_files.publish(
&binding_name,
&lazaret_core::codec::encode(binding)
.map_err(|error| io::Error::other(error.to_string()))?,
)?;
let registration_identity = match self.credential_files.publish(
&registration_name,
&lazaret_core::codec::encode(registration)
.map_err(|error| io::Error::other(error.to_string()))?,
) {
Ok(identity) => identity,
Err(error) => {
self.credential_files
.remove_if_matches(&binding_name, binding_identity)?;
return Err(error.into());
}
};
self.sessions.insert(
session_id,
SessionEscalationAuthority {
session,
files: SessionAuthorityFiles {
binding_name,
binding_identity,
registration_name,
registration_identity,
},
},
);
Ok(())
}
fn disconnect(&mut self, session_id: SessionId) -> Result<(), EscalationError> {
self.coordinator.session_lost(session_id)?;
self.active.retain(|_, active| *active != session_id);
let _ = self.prompt_transport.finish_escalation(session_id);
if let Some(authority) = self.sessions.remove(&session_id) {
self.credential_files.remove_if_matches(
&authority.files.binding_name,
authority.files.binding_identity,
)?;
self.credential_files.remove_if_matches(
&authority.files.registration_name,
authority.files.registration_identity,
)?;
}
Ok(())
}
fn process_requests(&mut self) -> Result<(), EscalationError> {
for name in self.inbox.names()? {
if !is_ready_record(&name) || !name.as_bytes().starts_with(b"request-") {
continue;
}
let Ok((bytes, identity)) = self.inbox.read(&name, MAX_SIGNED_RECORD_BYTES) else {
continue;
};
let matched = self.sessions.values().find_map(|authority| {
authority
.session
.verify_request(&bytes)
.ok()
.map(|(request, _)| (authority.session.clone(), request))
});
self.inbox.remove_if_matches(&name, identity)?;
let Some((session, request)) = matched else {
continue;
};
let session_id = session.session_binding().session_id;
let request_id = request.request_id();
let auto = self
.configuration
.auto_approval_rules()
.iter()
.find_map(|rule| {
rule.authorize(&request)
.ok()
.map(|executable| (executable, request.command_argv().to_vec()))
});
let phase = if let Some((executable, command_argv)) = auto {
self.coordinator.receive_auto_approved(
session,
&bytes,
executable,
command_argv,
&mut self.ownership,
Instant::now(),
)
} else {
let phase = self.coordinator.receive_for_developer_approval(
session.clone(),
&bytes,
Instant::now(),
);
let Ok(phase) = phase else {
continue;
};
if let Err(error) = self.prompt_transport.queue_signed_request(session, &bytes) {
self.coordinator.session_lost(session_id)?;
return Err(error.into());
}
Ok(phase)
};
let phase = match phase {
Ok(phase) => phase,
Err(error) if error.is_record_refusal() => continue,
Err(error) => return Err(error),
};
self.record_phase(request_id, session_id, phase)?;
}
Ok(())
}
fn process_active(&mut self) -> Result<(), EscalationError> {
let active = self
.active
.iter()
.map(|(id, session)| (*id, *session))
.collect::<Vec<_>>();
for (request_id, session_id) in &active {
if let Some(decision) = self.prompt_transport.take_decision(*session_id)? {
let phase = self.coordinator.apply_developer_decision(
*request_id,
&decision.signed_decision,
&mut DirectDeveloperExecutableResolver::open()?,
&mut self.ownership,
)?;
self.record_phase(*request_id, *session_id, phase)?;
}
if !self.active.contains_key(request_id) {
continue;
}
let inbox_names = self.inbox.names()?;
let prefix = acknowledgment_prefix(*request_id);
for name in &inbox_names {
if !name.as_bytes().starts_with(&prefix) {
continue;
}
let Ok(bytes) = self.inbox.consume(name, MAX_SIGNED_RECORD_BYTES) else {
continue;
};
match self
.coordinator
.acknowledge(*request_id, &bytes, Instant::now())
{
Ok(()) => {}
Err(error) if error.is_record_refusal() => {}
Err(error) => return Err(error),
}
}
let cancellation = cancellation_file_name(*request_id);
if inbox_names.contains(&cancellation) {
let Ok(bytes) = self.inbox.consume(&cancellation, MAX_SIGNED_RECORD_BYTES) else {
continue;
};
match self.coordinator.cancel(*request_id, &bytes) {
Ok(phase) => self.record_phase(*request_id, *session_id, phase)?,
Err(error) if error.is_record_refusal() => {}
Err(error) => return Err(error),
}
continue;
}
if self.active.contains_key(request_id) {
let phase = self.coordinator.progress(*request_id, Instant::now())?;
self.record_phase(*request_id, *session_id, phase)?;
}
}
for request_id in self.coordinator.expire(Instant::now())? {
if let Some(session_id) = self.active.get(&request_id).copied() {
self.record_phase(
request_id,
session_id,
EscalationPhase::Delivered(EscalationOutcome::TimedOut),
)?;
}
}
Ok(())
}
fn record_phase(
&mut self,
request_id: RequestId,
session_id: SessionId,
phase: EscalationPhase,
) -> Result<(), EscalationError> {
match phase {
EscalationPhase::AwaitingApproval => {
self.active.insert(request_id, session_id);
}
EscalationPhase::Executing => {
self.active.insert(request_id, session_id);
self.prompt_transport.begin_host_execution(session_id)?;
}
EscalationPhase::Delivered(_) => {
self.active.remove(&request_id);
self.prompt_transport.finish_escalation(session_id)?;
}
}
Ok(())
}
}
fn acknowledgment_prefix(request_id: RequestId) -> Vec<u8> {
let example = acknowledgment_file_name(request_id, 0);
let bytes = example.as_bytes();
bytes[..bytes.len() - 26].to_vec()
}
fn validate_executable(executable: impl AsFd, invoking_uid: u32) -> io::Result<()> {
let stat = fstat(executable)?;
let file_type = FileType::from_raw_mode(stat.st_mode);
let mode = Mode::from_raw_mode(stat.st_mode);
if !file_type.is_file()
|| stat.st_nlink != 1
|| (stat.st_uid != 0 && stat.st_uid != invoking_uid)
|| mode.intersects(Mode::WGRP | Mode::WOTH)
|| !mode.contains(Mode::XUSR)
{
return Err(io::Error::other(
"approved executable changed type, links, ownership, mode, or executability",
));
}
Ok(())
}
#[derive(Default)]
pub struct LocalExecutionOwnership {
staged: Vec<StagedProcessOwnership>,
}
impl ExecutionOwnershipRecorder for LocalExecutionOwnership {
fn stage(&mut self, ownership: StagedProcessOwnership) -> io::Result<()> {
if self.staged.contains(&ownership) {
Err(io::Error::other("execution ownership was already staged"))
} else {
self.staged.push(ownership);
Ok(())
}
}
fn acknowledge(
&mut self,
ownership: StagedProcessOwnership,
) -> io::Result<ProcessOwnershipCommitAck> {
if let Some(index) = self.staged.iter().position(|staged| *staged == ownership) {
self.staged.swap_remove(index);
Ok(ProcessOwnershipCommitAck::v1(ownership))
} else {
Err(io::Error::other("execution ownership is not staged"))
}
}
}
include!("escalation/service.rs");
include!("escalation/local_ownership.rs");

50
crates/lazaret-host-coordinator/src/escalation/local_ownership.rs Normal file
View file

@ -0,0 +1,50 @@
fn acknowledgment_prefix(request_id: RequestId) -> Vec<u8> {
let example = acknowledgment_file_name(request_id, 0);
let bytes = example.as_bytes();
bytes[..bytes.len() - 26].to_vec()
}
fn validate_executable(executable: impl AsFd, invoking_uid: u32) -> io::Result<()> {
let stat = fstat(executable)?;
let file_type = FileType::from_raw_mode(stat.st_mode);
let mode = Mode::from_raw_mode(stat.st_mode);
if !file_type.is_file()
|| stat.st_nlink != 1
|| (stat.st_uid != 0 && stat.st_uid != invoking_uid)
|| mode.intersects(Mode::WGRP | Mode::WOTH)
|| !mode.contains(Mode::XUSR)
{
return Err(io::Error::other(
"approved executable changed type, links, ownership, mode, or executability",
));
}
Ok(())
}
#[derive(Default)]
pub struct LocalExecutionOwnership {
staged: Vec<StagedProcessOwnership>,
}
impl ExecutionOwnershipRecorder for LocalExecutionOwnership {
fn stage(&mut self, ownership: StagedProcessOwnership) -> io::Result<()> {
if self.staged.contains(&ownership) {
Err(io::Error::other("execution ownership was already staged"))
} else {
self.staged.push(ownership);
Ok(())
}
}
fn acknowledge(
&mut self,
ownership: StagedProcessOwnership,
) -> io::Result<ProcessOwnershipCommitAck> {
if let Some(index) = self.staged.iter().position(|staged| *staged == ownership) {
self.staged.swap_remove(index);
Ok(ProcessOwnershipCommitAck::v1(ownership))
} else {
Err(io::Error::other("execution ownership is not staged"))
}
}
}

423
crates/lazaret-host-coordinator/src/escalation/service.rs Normal file
View file

@ -0,0 +1,423 @@
struct SessionAuthorityFiles {
binding_name: OsString,
binding_identity: MailboxEntryIdentity,
registration_name: OsString,
registration_identity: MailboxEntryIdentity,
}
struct SessionEscalationAuthority {
session: VerifiedSession,
files: SessionAuthorityFiles,
}
struct RegistrationCommand {
binding: SessionBinding,
registration: SessionRegistrationResponse,
completed: mpsc::SyncSender<Result<(), String>>,
}
enum EscalationServiceCommand {
Register(Box<RegistrationCommand>),
Disconnect {
session_id: SessionId,
completed: mpsc::SyncSender<Result<(), String>>,
},
Stop,
}
#[derive(Clone)]
pub(crate) struct EscalationServiceHandle {
commands: mpsc::Sender<EscalationServiceCommand>,
}
impl EscalationServiceHandle {
pub(crate) fn register(
&self,
binding: SessionBinding,
registration: SessionRegistrationResponse,
) -> Result<(), EscalationError> {
let (completed, result) = mpsc::sync_channel(1);
self.commands
.send(EscalationServiceCommand::Register(Box::new(
RegistrationCommand {
binding,
registration,
completed,
},
)))
.map_err(|_| io::Error::other("escalation service stopped"))?;
result
.recv()
.map_err(|_| io::Error::other("escalation registration was abandoned"))?
.map_err(|error| io::Error::other(error).into())
}
pub(crate) fn disconnect(&self, session_id: SessionId) -> Result<(), EscalationError> {
let (completed, result) = mpsc::sync_channel(1);
self.commands
.send(EscalationServiceCommand::Disconnect {
session_id,
completed,
})
.map_err(|_| io::Error::other("escalation service stopped"))?;
result
.recv()
.map_err(|_| io::Error::other("escalation disconnect was abandoned"))?
.map_err(|error| io::Error::other(error).into())
}
pub(crate) fn stop(&self) {
let _ = self.commands.send(EscalationServiceCommand::Stop);
}
}
pub(crate) struct EscalationService<'a> {
configuration: &'a ResolvedDomainConfiguration,
publication: ProtocolPublicationIdentity,
credentials: SandboxCredentialBundle,
credential_files: RegularFileMailbox,
inbox: RegularFileMailbox,
prompt_transport: Arc<SessionPromptTransport>,
coordinator: EscalationCoordinator,
ownership: LocalExecutionOwnership,
sessions: BTreeMap<SessionId, SessionEscalationAuthority>,
active: BTreeMap<RequestId, SessionId>,
commands: mpsc::Receiver<EscalationServiceCommand>,
}
pub(crate) struct EscalationServiceInputs<'a> {
pub configuration: &'a ResolvedDomainConfiguration,
pub publication: ProtocolPublicationIdentity,
pub credentials: SandboxCredentialBundle,
pub credential_files: RegularFileMailbox,
pub inbox: RegularFileMailbox,
pub outbox: RegularFileMailbox,
pub prompt_transport: Arc<SessionPromptTransport>,
pub coordinator_key: SigningKey,
pub ownership: ProcessOwnershipContext,
pub invoking_uid: u32,
}
impl<'a> EscalationService<'a> {
pub(crate) fn new(
inputs: EscalationServiceInputs<'a>,
) -> Result<(Self, EscalationServiceHandle), EscalationError> {
inputs.credential_files.publish(
OsStr::new("publication.postcard"),
&lazaret_core::codec::encode(&inputs.publication)
.map_err(|error| io::Error::other(error.to_string()))?,
)?;
inputs.credential_files.publish(
OsStr::new("sandbox-credentials.postcard"),
&lazaret_core::codec::encode(&inputs.credentials)
.map_err(|error| io::Error::other(error.to_string()))?,
)?;
let (commands, receiver) = mpsc::channel();
Ok((
Self {
configuration: inputs.configuration,
publication: inputs.publication,
credentials: inputs.credentials,
credential_files: inputs.credential_files,
inbox: inputs.inbox,
prompt_transport: inputs.prompt_transport,
coordinator: EscalationCoordinator::new(
inputs.coordinator_key,
inputs.outbox,
inputs.ownership,
inputs.invoking_uid,
Duration::from_secs(30),
Duration::from_secs(2),
),
ownership: LocalExecutionOwnership::default(),
sessions: BTreeMap::new(),
active: BTreeMap::new(),
commands: receiver,
},
EscalationServiceHandle { commands },
))
}
pub(crate) fn run(mut self) -> Result<(), EscalationError> {
loop {
while let Ok(command) = self.commands.try_recv() {
if self.apply_command(command)? {
return Ok(());
}
}
self.process_requests()?;
self.process_active()?;
std::thread::sleep(Duration::from_millis(5));
}
}
fn apply_command(
&mut self,
command: EscalationServiceCommand,
) -> Result<bool, EscalationError> {
match command {
EscalationServiceCommand::Register(command) => {
let RegistrationCommand {
binding,
registration,
completed,
} = *command;
let result = self
.register(&binding, &registration)
.map_err(|error| error.to_string());
let failed = result.is_err();
let _ = completed.send(result);
if failed {
return Err(io::Error::other("session escalation registration failed").into());
}
Ok(false)
}
EscalationServiceCommand::Disconnect {
session_id,
completed,
} => {
let result = self
.disconnect(session_id)
.map_err(|error| error.to_string());
let failed = result.is_err();
let _ = completed.send(result);
if failed {
return Err(io::Error::other("session escalation cleanup failed").into());
}
Ok(false)
}
EscalationServiceCommand::Stop => {
let sessions = self.sessions.keys().copied().collect::<Vec<_>>();
for session_id in sessions {
self.disconnect(session_id)?;
}
Ok(true)
}
}
}
fn register(
&mut self,
binding: &SessionBinding,
registration: &SessionRegistrationResponse,
) -> Result<(), EscalationError> {
let session_id = binding.session_id;
if self.sessions.contains_key(&session_id) {
return Err(EscalationError::DuplicateSessionRequest);
}
let session = VerifiedSession::new(
self.publication.trust_manifest_sha256,
self.credentials.trust_manifest.tls_ca_sha256,
&self.credentials.trust_manifest,
binding.clone(),
self.credentials.coordinator_protocol_credential(),
self.credentials.sandbox_protocol_credential(),
registration.session_capability.clone(),
)?;
let prefix = mailbox_session_name(session_id);
let mut binding_name = prefix.clone();
binding_name.push("-binding.postcard");
let mut registration_name = prefix;
registration_name.push("-registration.postcard");
let binding_identity = self.credential_files.publish(
&binding_name,
&lazaret_core::codec::encode(binding)
.map_err(|error| io::Error::other(error.to_string()))?,
)?;
let registration_identity = match self.credential_files.publish(
&registration_name,
&lazaret_core::codec::encode(registration)
.map_err(|error| io::Error::other(error.to_string()))?,
) {
Ok(identity) => identity,
Err(error) => {
self.credential_files
.remove_if_matches(&binding_name, binding_identity)?;
return Err(error.into());
}
};
self.sessions.insert(
session_id,
SessionEscalationAuthority {
session,
files: SessionAuthorityFiles {
binding_name,
binding_identity,
registration_name,
registration_identity,
},
},
);
Ok(())
}
fn disconnect(&mut self, session_id: SessionId) -> Result<(), EscalationError> {
self.coordinator.session_lost(session_id)?;
self.active.retain(|_, active| *active != session_id);
let _ = self.prompt_transport.finish_escalation(session_id);
if let Some(authority) = self.sessions.remove(&session_id) {
self.credential_files.remove_if_matches(
&authority.files.binding_name,
authority.files.binding_identity,
)?;
self.credential_files.remove_if_matches(
&authority.files.registration_name,
authority.files.registration_identity,
)?;
}
Ok(())
}
fn process_requests(&mut self) -> Result<(), EscalationError> {
for name in self.inbox.names()? {
if !is_ready_record(&name) || !name.as_bytes().starts_with(b"request-") {
continue;
}
let Ok((bytes, identity)) = self.inbox.read(&name, MAX_SIGNED_RECORD_BYTES) else {
continue;
};
let matched = self.sessions.values().find_map(|authority| {
authority
.session
.verify_request(&bytes)
.ok()
.map(|(request, _)| (authority.session.clone(), request))
});
self.inbox.remove_if_matches(&name, identity)?;
let Some((session, request)) = matched else {
continue;
};
let session_id = session.session_binding().session_id;
let request_id = request.request_id();
let auto = self
.configuration
.auto_approval_rules()
.iter()
.find_map(|rule| {
rule.authorize(&request)
.ok()
.map(|executable| (executable, request.command_argv().to_vec()))
});
let phase = if let Some((executable, command_argv)) = auto {
self.coordinator.receive_auto_approved(
session,
&bytes,
executable,
command_argv,
&mut self.ownership,
Instant::now(),
)
} else {
let phase = self.coordinator.receive_for_developer_approval(
session.clone(),
&bytes,
Instant::now(),
);
let Ok(phase) = phase else {
continue;
};
if let Err(error) = self.prompt_transport.queue_signed_request(session, &bytes) {
self.coordinator.session_lost(session_id)?;
return Err(error.into());
}
Ok(phase)
};
let phase = match phase {
Ok(phase) => phase,
Err(error) if error.is_record_refusal() => continue,
Err(error) => return Err(error),
};
self.record_phase(request_id, session_id, phase)?;
}
Ok(())
}
fn process_active(&mut self) -> Result<(), EscalationError> {
let active = self
.active
.iter()
.map(|(id, session)| (*id, *session))
.collect::<Vec<_>>();
for (request_id, session_id) in &active {
if let Some(decision) = self.prompt_transport.take_decision(*session_id)? {
let phase = self.coordinator.apply_developer_decision(
*request_id,
&decision.signed_decision,
&mut DirectDeveloperExecutableResolver::open()?,
&mut self.ownership,
)?;
self.record_phase(*request_id, *session_id, phase)?;
}
if !self.active.contains_key(request_id) {
continue;
}
let inbox_names = self.inbox.names()?;
let prefix = acknowledgment_prefix(*request_id);
for name in &inbox_names {
if !name.as_bytes().starts_with(&prefix) {
continue;
}
let Ok(bytes) = self.inbox.consume(name, MAX_SIGNED_RECORD_BYTES) else {
continue;
};
match self
.coordinator
.acknowledge(*request_id, &bytes, Instant::now())
{
Ok(()) => {}
Err(error) if error.is_record_refusal() => {}
Err(error) => return Err(error),
}
}
let cancellation = cancellation_file_name(*request_id);
if inbox_names.contains(&cancellation) {
let Ok(bytes) = self.inbox.consume(&cancellation, MAX_SIGNED_RECORD_BYTES) else {
continue;
};
match self.coordinator.cancel(*request_id, &bytes) {
Ok(phase) => self.record_phase(*request_id, *session_id, phase)?,
Err(error) if error.is_record_refusal() => {}
Err(error) => return Err(error),
}
continue;
}
if self.active.contains_key(request_id) {
let phase = self.coordinator.progress(*request_id, Instant::now())?;
self.record_phase(*request_id, *session_id, phase)?;
}
}
for request_id in self.coordinator.expire(Instant::now())? {
if let Some(session_id) = self.active.get(&request_id).copied() {
self.record_phase(
request_id,
session_id,
EscalationPhase::Delivered(EscalationOutcome::TimedOut),
)?;
}
}
Ok(())
}
fn record_phase(
&mut self,
request_id: RequestId,
session_id: SessionId,
phase: EscalationPhase,
) -> Result<(), EscalationError> {
match phase {
EscalationPhase::AwaitingApproval => {
self.active.insert(request_id, session_id);
}
EscalationPhase::Executing => {
self.active.insert(request_id, session_id);
self.prompt_transport.begin_host_execution(session_id)?;
}
EscalationPhase::Delivered(_) => {
self.active.remove(&request_id);
self.prompt_transport.finish_escalation(session_id)?;
}
}
Ok(())
}
}

1751
crates/lazaret-host-coordinator/src/recovery.rs
View file

File diff suppressed because it is too large Load diff

841
crates/lazaret-host-coordinator/src/recovery/reconciliation.rs Normal file
View file

@ -0,0 +1,841 @@
#[must_use]
pub fn recovery_role_order() -> [ParticipantRole; 7] {
[
ParticipantRole::Publication,
ParticipantRole::Sandbox,
ParticipantRole::Helper,
ParticipantRole::Network,
ParticipantRole::GitRouting,
ParticipantRole::StateAnchor,
ParticipantRole::PreparedLower,
]
}
fn reconcile_runtime_before_state_anchor(owners: &mut SealedRuntimeOwners) -> io::Result<()> {
for role in [
RuntimeProcessRole::AttachedRunsc,
RuntimeProcessRole::ConnectivityHelper,
RuntimeProcessRole::GitRoutingDaemon,
RuntimeProcessRole::RuntimeSupervisor,
] {
owners.reconcile(role)?;
}
Ok(())
}
/// Proves every recorded generation runtime holder and filesystem mount absent.
///
/// # Errors
/// Refuses empty or partial inventory, live HC authority, exact-pidfd action failure, unreadable
/// evidence, or any recorded State Anchor/Git-routing mount that remains present.
pub fn prove_runtime_owners_absent(
ownership: &RuntimeRecoveryOwnership,
) -> Result<RuntimeOwnersAbsent, io::Error> {
if ownership.participants_present == 0
|| (ownership.processes.is_empty()
&& ownership.state_anchor_mount.is_none()
&& ownership.git_routing_mount.is_none())
{
return Err(io::Error::other("runtime recovery inventory is empty"));
}
seal_runtime_owners(ownership, match_recovery_process)?.prove_absent()
}
fn seal_runtime_owners<Open>(
ownership: &RuntimeRecoveryOwnership,
mut open: Open,
) -> io::Result<SealedRuntimeOwners>
where
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
{
let mut processes = Vec::with_capacity(ownership.processes.len());
for owner in &ownership.processes {
let matched = match open(owner.identity) {
Ok(PidfsRecoveryMatch::BootMismatch) => SealedProcessMatch::BootMismatch,
Ok(PidfsRecoveryMatch::Matched(process)) => SealedProcessMatch::Matched(process),
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
SealedProcessMatch::Esrch
}
Err(error) => return Err(error),
};
processes.push(SealedProcessOwner {
role: owner.role,
matched,
reconciled: false,
});
}
Ok(SealedRuntimeOwners {
ownership: ownership.clone(),
processes,
})
}
fn require_mount_absent(
path: Option<&HostPath>,
mount: Option<MountIdentity>,
label: &str,
) -> io::Result<()> {
match (path, mount) {
(None, None) => Ok(()),
(Some(path), Some(expected)) => {
let observed =
current_mount_identity(Path::new(std::ffi::OsStr::from_bytes(path.as_bytes())))?;
if observed == expected {
Err(io::Error::other(format!(
"{label} exact mount remains present"
)))
} else {
Ok(())
}
}
_ => Err(io::Error::other(format!(
"{label} recovery mount evidence is incomplete"
))),
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum RecoveryBoundary {
Snapshot,
Publication,
Sandbox,
HelperAndNetwork,
GitRouting,
StateAnchor,
PreparedLower,
PrepareOperations,
ArtifactGc,
FinishGeneration,
}
pub(crate) fn recover_before_cold_start(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
prepared: &PreparedLowerStore,
) -> Result<(), String> {
recover_before_cold_start_with(configuration, store, catalog, prepared, |_, _| Ok(()))
}
fn recover_before_cold_start_with<Failpoint>(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
prepared: &PreparedLowerStore,
failpoint: Failpoint,
) -> Result<(), String>
where
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
{
recover_before_cold_start_with_opener(
configuration,
store,
catalog,
prepared,
failpoint,
match_recovery_process,
|root, container| delete_runsc_container(root, container, std::ffi::OsStr::new("runsc")),
)
}
fn recover_before_cold_start_with_opener<Failpoint, Open, Delete>(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
prepared: &PreparedLowerStore,
mut failpoint: Failpoint,
mut open: Open,
mut delete: Delete,
) -> Result<(), String>
where
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
failpoint(RecoveryBoundary::Snapshot, GenerationId::new(0))?;
let snapshot = store
.snapshot()
.map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
if snapshot.domain.domain_id != configuration.domain_id() {
return Err("recovery Domain differs from resolved configuration".into());
}
let mut generations = snapshot.generations.clone();
generations.sort_by_key(|generation| generation.generation_id.get());
for generation in generations {
if generation.durable_decision == DurableDecision::Clean {
continue;
}
if generation.configuration_fingerprint != configuration.configuration_fingerprint() {
return Err("unfinished generation configuration differs".into());
}
recover_generation(
configuration,
store,
catalog,
&generation,
&mut failpoint,
&mut open,
&mut delete,
)
.map_err(|error| {
format!(
"generation {} recovery failed: {error}",
generation.generation_id.get()
)
})?;
}
failpoint(RecoveryBoundary::PrepareOperations, GenerationId::new(0))?;
prepared
.recover_preparations()
.map_err(|error| error.to_string())?;
failpoint(RecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
let selected = catalog_snapshot
.prepared_artifacts
.iter()
.filter(|artifact| {
!catalog_snapshot
.artifact_leases
.iter()
.any(|lease| lease.prepared_artifact_id == artifact.prepared_artifact_id)
})
.map(|artifact| artifact.prepared_artifact_id)
.collect::<Vec<_>>();
prepared
.gc_artifacts(&selected)
.map_err(|error| error.to_string())?;
Ok(())
}
fn recover_generation<Failpoint, Open, Delete>(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
generation: &Generation,
failpoint: &mut Failpoint,
open: &mut Open,
delete: &mut Delete,
) -> Result<(), String>
where
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
let generation_id = generation.generation_id;
let snapshot = store.snapshot().map_err(|error| error.to_string())?;
if snapshot
.session_terminals
.iter()
.any(|terminal| terminal.generation_id() == generation_id)
|| snapshot
.mailboxes
.iter()
.any(|mailbox| mailbox.generation_id == generation_id)
{
return Err("unfinished terminal or mailbox state cannot be recovered safely".into());
}
let session_updates = snapshot
.sessions
.iter()
.filter(|session| {
session.generation_id == generation_id
&& !matches!(
session.session_phase,
SessionPhase::Closed | SessionPhase::Failed
)
})
.map(|session| {
let mut failed = session.clone();
failed.session_phase = SessionPhase::Failed;
DomainStoreMutation::PutSession(failed)
})
.collect::<Vec<_>>();
if !session_updates.is_empty() {
store
.apply(&session_updates)
.map_err(|error| format!("Session failure update failed: {error}"))?;
}
let participants = store
.snapshot()
.map_err(|error| format!("Participant snapshot failed: {error}"))?
.participants;
let ownership = runtime_recovery_ownership(&participants, generation)
.map_err(|error| format!("runtime ownership inventory failed: {error}"))?;
let mut owners = seal_runtime_owners(&ownership, open)
.map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
failpoint(RecoveryBoundary::Publication, generation_id)?;
recover_publication(configuration, store, generation_id)
.map_err(|error| format!("Publication recovery failed: {error}"))?;
failpoint(RecoveryBoundary::Sandbox, generation_id)?;
reconcile_runtime_before_state_anchor(&mut owners)
.map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
require_sandbox_absent(configuration, store, generation_id, delete)
.map_err(|error| format!("Sandbox absence failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
.map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
failpoint(RecoveryBoundary::HelperAndNetwork, generation_id)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
.map_err(|error| format!("Helper cleanup failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Network)
.map_err(|error| format!("Network cleanup failed: {error}"))?;
failpoint(RecoveryBoundary::GitRouting, generation_id)?;
require_mount_absent(
ownership.git_routing_path.as_ref(),
ownership.git_routing_mount,
"Git-routing",
)
.map_err(|error| error.to_string())?;
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
.map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
failpoint(RecoveryBoundary::StateAnchor, generation_id)?;
recover_state_anchor(configuration, store, catalog, generation, &mut owners)
.map_err(|error| format!("State Anchor recovery failed: {error}"))?;
failpoint(RecoveryBoundary::PreparedLower, generation_id)?;
recover_partial_prepared_lower(configuration, store, catalog, generation_id)
.map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
let snapshot = store.snapshot().map_err(|error| error.to_string())?;
if snapshot.participants.iter().any(|participant| {
participant.generation_id() == generation_id
&& participant.state() != ParticipantState::Clean
}) {
return Err("generation retains non-Clean participants after recovery".into());
}
failpoint(RecoveryBoundary::FinishGeneration, generation_id)?;
let mut clean = generation.clone();
clean.durable_decision = DurableDecision::Clean;
store
.apply(&[DomainStoreMutation::PutGeneration(clean)])
.map_err(|error| error.to_string())
}
fn recover_publication(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
generation_id: GenerationId,
) -> Result<(), String> {
let Some(participant) = role(store, generation_id, ParticipantRole::Publication)? else {
return Ok(());
};
if participant.state() == ParticipantState::Clean {
return Ok(());
}
let root = configuration
.runtime_domain_root()
.open_authority()
.map_err(|error| error.to_string())?;
let Some(ParticipantOwnership::Publication(publication)) = participant.ownership() else {
if participant.ownership().is_none() {
require_bare_publication_absent(&root)?;
return clean_role_after_absence(store, generation_id, ParticipantRole::Publication);
}
return Err("publication ownership shape is invalid".into());
};
require_publication_labels(publication)?;
remove_exact_publication_resource(
&root,
[STAGED_CONTROL_SOCKET_NAME, CONTROL_SOCKET_NAME],
publication
.socket_identity
.ok_or("publication socket identity is absent")?,
true,
)?;
remove_exact_publication_resource(
&root,
[STAGED_PUBLICATION_IDENTITY_NAME, PUBLICATION_IDENTITY_NAME],
publication
.identity_file_identity
.ok_or("publication identity-file identity is absent")?,
false,
)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Publication)
}
fn require_bare_publication_absent(root: &std::os::fd::OwnedFd) -> Result<(), String> {
for label in [
STAGED_CONTROL_SOCKET_NAME,
STAGED_PUBLICATION_IDENTITY_NAME,
CONTROL_SOCKET_NAME,
PUBLICATION_IDENTITY_NAME,
] {
match openat(
root,
label,
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
) {
Err(rustix::io::Errno::NOENT) => {}
Err(error) => return Err(error.to_string()),
Ok(_) => return Err("bare publication intent has a fixed-label artifact".into()),
}
}
Ok(())
}
fn require_publication_labels(
ownership: &lazaret_core::records::PublicationOwnership,
) -> Result<(), String> {
for (observed, expected) in [
(&ownership.staged_socket_path, STAGED_CONTROL_SOCKET_NAME),
(
&ownership.staged_identity_path,
STAGED_PUBLICATION_IDENTITY_NAME,
),
(&ownership.socket_path, CONTROL_SOCKET_NAME),
(&ownership.identity_path, PUBLICATION_IDENTITY_NAME),
] {
if observed.as_ref().map(HostPath::as_bytes) != Some(expected.as_bytes()) {
return Err("publication durable label differs from the fixed runtime label".into());
}
}
Ok(())
}
fn remove_exact_publication_resource(
root: &std::os::fd::OwnedFd,
labels: [&str; 2],
expected: MountIdentity,
socket: bool,
) -> Result<(), String> {
let mut exact = None;
for label in labels {
let descriptor = match openat(
root,
label,
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
) {
Ok(descriptor) => descriptor,
Err(rustix::io::Errno::NOENT) => continue,
Err(error) => return Err(error.to_string()),
};
let stat = fstat(&descriptor).map_err(|error| error.to_string())?;
let file_type = rustix::fs::FileType::from_raw_mode(stat.st_mode);
if mount_identity(&descriptor).map_err(|error| error.to_string())? != expected
|| (socket && !file_type.is_socket())
|| (!socket && !file_type.is_file())
|| exact.replace(label).is_some()
{
return Err("publication resource changed identity, type, or location".into());
}
}
if let Some(label) = exact {
let current = openat(
root,
label,
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(|error| error.to_string())?;
let stat = fstat(&current).map_err(|error| error.to_string())?;
let file_type = rustix::fs::FileType::from_raw_mode(stat.st_mode);
if mount_identity(&current).map_err(|error| error.to_string())? != expected
|| (socket && !file_type.is_socket())
|| (!socket && !file_type.is_file())
{
return Err("publication resource changed before exact removal".into());
}
unlinkat(root, label, AtFlags::empty()).map_err(|error| error.to_string())?;
}
Ok(())
}
fn require_sandbox_absent(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
generation_id: GenerationId,
delete: &mut impl FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
) -> Result<(), String> {
let Some(participant) = role(store, generation_id, ParticipantRole::Sandbox)? else {
return Ok(());
};
if participant.state() == ParticipantState::Clean {
return Ok(());
}
let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else {
return Ok(());
};
let (Some(runtime_root), Some(runtime_root_identity), Some(container_id)) = (
&sandbox.runtime_root,
sandbox.runtime_root_identity,
sandbox.container_id,
) else {
return Err("Sandbox runsc-root recovery authority is incomplete".into());
};
let expected_label = Path::new(std::ffi::OsStr::from_bytes(
configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
))
.join("runsc");
if runtime_root.as_bytes() != expected_label.as_os_str().as_bytes() {
return Err("Sandbox runsc root differs from the resolved Runtime Domain".into());
}
let root = configuration
.runtime_domain_root()
.open_beneath(Path::new("runsc"), ConfinedPathType::Directory)
.map_err(|error| error.to_string())?;
if mount_identity(&root).map_err(|error| error.to_string())? != runtime_root_identity {
return Err("Sandbox runsc-root physical identity changed".into());
}
let container = hex_lower(container_id.as_bytes());
match statat(&root, &container, AtFlags::SYMLINK_NOFOLLOW) {
Err(rustix::io::Errno::NOENT) => {}
Err(error) => return Err(error.to_string()),
Ok(stat) => {
if !rustix::fs::FileType::from_raw_mode(stat.st_mode).is_dir() {
return Err("recorded runsc container changed type".into());
}
delete(&root, &container)?;
match statat(&root, &container, AtFlags::SYMLINK_NOFOLLOW) {
Err(rustix::io::Errno::NOENT) => {}
Err(error) => return Err(error.to_string()),
Ok(_) => return Err("recorded runsc container remains after exact cleanup".into()),
}
}
}
Ok(())
}
/// Deletes one exact container beneath a retained, already-verified runsc root descriptor.
///
/// # Errors
/// Returns launch, wrapper, or nonzero runsc failure without weakening the retained root.
pub fn delete_runsc_container(
root: &std::os::fd::OwnedFd,
container: &str,
program: &std::ffi::OsStr,
) -> Result<(), String> {
fcntl_setfd(root, FdFlags::empty()).map_err(|error| error.to_string())?;
let root_label = format!("/proc/self/fd/{}", root.as_raw_fd());
child_broker()
.run_transient(
&TargetCommand::new(program)
.argument(format!("-root={root_label}"))
.argument("delete")
.argument("--force")
.argument(container),
)
.map_err(|error| error.to_string())?;
Ok(())
}
fn recover_state_anchor(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
generation: &Generation,
owners: &mut SealedRuntimeOwners,
) -> Result<(), String> {
let Some(anchor) = role(
store,
generation.generation_id,
ParticipantRole::StateAnchor,
)?
else {
return Ok(());
};
if anchor.state() == ParticipantState::Clean {
let prepared = role(
store,
generation.generation_id,
ParticipantRole::PreparedLower,
)?;
if prepared
.is_some_and(|participant| participant.state() == ParticipantState::ReleaseIntent)
{
let layout =
StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
let coordinator = StateAnchorCoordinator::new(store, catalog);
return coordinator
.resume_release_after_checkpoint(generation.generation_id, &layout, || {
fresh_absence(store, generation.generation_id, &layout)
})
.map_err(|error| error.to_string());
}
return Ok(());
}
let full_mount = matches!(
anchor.state(),
ParticipantState::Prepared | ParticipantState::ReleaseIntent
) && owners.ownership.state_anchor_mount.is_some();
if !full_mount {
let _ = owners.prove_absent().map_err(|error| error.to_string())?;
return clean_role_after_absence(
store,
generation.generation_id,
ParticipantRole::StateAnchor,
);
}
let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
let coordinator = StateAnchorCoordinator::new(store, catalog);
owners
.ownership
.process(RuntimeProcessRole::StateAnchorDaemon)
.ok_or("State Anchor daemon authority is absent")?;
let recovered = if let Some(SealedProcessMatch::Matched(process)) =
owners.process(RuntimeProcessRole::StateAnchorDaemon)
{
if process.has_exited().map_err(|error| error.to_string())? {
let witness = owners.prove_absent().map_err(|error| error.to_string())?;
coordinator
.recover_after_runtime_owner_loss(
generation.generation_id,
&layout,
StateAnchorRecoveryOwner::Absent(&witness),
)
.map_err(|error| error.to_string())?
} else {
coordinator
.recover_after_runtime_owner_loss(
generation.generation_id,
&layout,
StateAnchorRecoveryOwner::Matched(process),
)
.map_err(|error| error.to_string())?
}
} else {
let witness = owners.prove_absent().map_err(|error| error.to_string())?;
coordinator
.recover_after_runtime_owner_loss(
generation.generation_id,
&layout,
StateAnchorRecoveryOwner::Absent(&witness),
)
.map_err(|error| error.to_string())?
};
let (authority, first) = recovered;
let complete_witness = owners.prove_absent().map_err(|error| error.to_string())?;
if !complete_witness.ownership().state_anchor_matches(
authority.daemon(),
authority.user_namespace(),
authority.mount_namespace(),
authority.mount(),
) {
return Err("completed recovery witness changed State Anchor authority".into());
}
coordinator
.complete_release_after_unmount(&authority, &layout, first, || {
fresh_absence(store, generation.generation_id, &layout)
})
.map_err(|error| error.to_string())
}
fn recover_partial_prepared_lower(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
generation_id: GenerationId,
) -> Result<(), String> {
let Some(prepared) = role(store, generation_id, ParticipantRole::PreparedLower)? else {
return Ok(());
};
if prepared.state() == ParticipantState::Clean {
return Ok(());
}
if prepared.state() == ParticipantState::ReleaseIntent {
return Err("prepared-lower ReleaseIntent lacks complete State Anchor evidence".into());
}
let Some(ParticipantOwnership::PreparedLower(ownership)) = prepared.ownership() else {
return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower);
};
if prepared.state() != ParticipantState::LeaseHeld {
return clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower);
}
let artifact = ownership
.prepared_artifact_id
.ok_or("LeaseHeld prepared artifact is absent")?;
let domain_id = store
.snapshot()
.map_err(|error| error.to_string())?
.domain
.domain_id;
let lease = ArtifactLease {
prepared_artifact_id: artifact,
domain_id,
generation_id,
};
require_exact_lease(catalog, &lease)?;
let layout = StateAnchorLayout::create(configuration).map_err(|error| error.to_string())?;
lazaret_state_anchor::require_mount_absent(&layout).map_err(|error| error.to_string())?;
let release = Participant::snapshot(
generation_id,
ParticipantRole::PreparedLower,
ParticipantState::ReleaseIntent,
Some(ParticipantOwnership::PreparedLower(
lazaret_core::records::PreparedLowerOwnership {
lease_state: Some(ParticipantState::ReleaseIntent),
..ownership.clone()
},
)),
)
.map_err(|_| "partial prepared-lower release snapshot is invalid")?;
store
.apply(&[DomainStoreMutation::PutParticipant(release)])
.map_err(|error| error.to_string())?;
catalog
.release_lease_checked(&lease, || {
let snapshot = store
.snapshot()
.map_err(|error| CatalogError::Io(io::Error::other(error)))?;
let prepared = snapshot
.participants
.iter()
.find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::PreparedLower
})
.ok_or_else(|| CatalogError::Io(io::Error::other("prepared participant absent")))?;
if prepared.state() != ParticipantState::ReleaseIntent {
return Err(CatalogError::Io(io::Error::other(
"prepared release intent is absent",
)));
}
lazaret_state_anchor::require_mount_absent(&layout)
.map_err(|error| CatalogError::Io(io::Error::other(error)))
})
.map_err(|error| error.to_string())?;
clean_role_after_absence(store, generation_id, ParticipantRole::PreparedLower)
}
fn require_exact_lease(catalog: &Catalog, expected: &ArtifactLease) -> Result<(), String> {
match catalog
.snapshot()
.map_err(|error| error.to_string())?
.artifact_leases
.iter()
.find(|lease| {
lease.domain_id == expected.domain_id && lease.generation_id == expected.generation_id
}) {
Some(observed) if observed == expected => Ok(()),
Some(_) => Err("generation catalog lease mismatches durable ownership".into()),
None => Err("LeaseHeld prepared lower lacks its exact catalog lease".into()),
}
}
fn fresh_absence(
store: &HostCoordinatorDomainStore,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
) -> Result<StateAnchorAbsenceObservation, StateAnchorCoordinatorError> {
let snapshot = store.snapshot()?;
let anchor = snapshot
.participants
.iter()
.find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::StateAnchor
})
.ok_or(StateAnchorCoordinatorError::InvalidState(
"State Anchor participant is absent",
))?;
let mut evidence = match anchor.ownership() {
Some(ParticipantOwnership::StateAnchor(ownership)) => (
ownership.user_namespace,
ownership.mount_namespace,
ownership.mount,
),
_ => (None, None, None),
};
if evidence == (None, None, None) {
evidence = snapshot
.participants
.iter()
.find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::PreparedLower
})
.and_then(Participant::ownership)
.and_then(|ownership| match ownership {
ParticipantOwnership::PreparedLower(ownership) => ownership.released_state_anchor,
_ => None,
})
.map_or((None, None, None), |evidence| {
(
Some(evidence.user_namespace),
Some(evidence.mount_namespace),
Some(evidence.mount),
)
});
}
let (Some(user_namespace), Some(mount_namespace), Some(mount)) = evidence else {
return Err(StateAnchorCoordinatorError::InvalidState(
"State Anchor absence evidence is incomplete",
));
};
let observed = lazaret_state_anchor::observe_mount_absence(layout, mount)?;
Ok(StateAnchorAbsenceObservation {
user_namespace,
mount_namespace,
mount,
observed,
})
}
fn clean_role_after_absence(
store: &HostCoordinatorDomainStore,
generation_id: GenerationId,
role_name: ParticipantRole,
) -> Result<(), String> {
let Some(current) = role(store, generation_id, role_name)? else {
return Ok(());
};
if current.state() == ParticipantState::Clean {
return Ok(());
}
if current.state() != ParticipantState::ReleaseIntent && current.ownership().is_some() {
let release = Participant::snapshot(
generation_id,
role_name,
ParticipantState::ReleaseIntent,
current.ownership().cloned(),
)
.map_err(|_| "participant release snapshot is invalid")?;
store
.apply(&[DomainStoreMutation::PutParticipant(release)])
.map_err(|error| error.to_string())?;
}
let clean = Participant::snapshot(
generation_id,
role_name,
ParticipantState::Clean,
Some(ParticipantOwnership::empty(role_name)),
)
.map_err(|_| "participant Clean snapshot is invalid")?;
store
.apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(clean)])
.map_err(|error| error.to_string())
}
fn role(
store: &HostCoordinatorDomainStore,
generation_id: GenerationId,
role_name: ParticipantRole,
) -> Result<Option<Participant>, String> {
Ok(store
.snapshot()
.map_err(|error| error.to_string())?
.participants
.into_iter()
.find(|participant| {
participant.generation_id() == generation_id && participant.role() == role_name
}))
}
fn hex_lower(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(bytes.len() * 2);
for byte in bytes {
output.push(char::from(HEX[usize::from(byte >> 4)]));
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
output
}

907
crates/lazaret-host-coordinator/src/recovery/tests.rs Normal file
View file

@ -0,0 +1,907 @@
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::fs::{self, File, OpenOptions};
use std::os::unix::fs::symlink;
use std::os::unix::fs::{FileTypeExt, MetadataExt};
use std::os::unix::net::UnixListener;
use std::path::PathBuf;
use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration};
use lazaret_core::identity::{
BootId, BundleDigest, ContainerId, CoordinatorId, DeviceId, HostPidLocator, IdMapDigest,
InodeId, MountId, NetworkSidecarId, PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1,
PolicyId, PrepareOperationId, PreparedArtifactId, ProcessStartTimeTicks, SandboxId,
SelectedManifestDigest, SourceIdentity,
};
use lazaret_core::records::{
ArtifactState, DurableDecision, GitRoutingOwnership, HelperOwnership, PrepareOperation,
PrepareStageState, PreparedArtifact, PreparedLowerOwnership, PublicationOwnership,
SandboxOwnership, StateAnchorOwnership,
};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
use super::*;
fn process(value: u64) -> PidfsProcessIdentityV1 {
PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new([1; 16]),
pidfs_st_dev: PidfsDeviceId::new(value),
pidfs_st_ino: PidfsInodeId::new(value),
},
HostPidLocator::new(u32::try_from(value).unwrap()),
ProcessStartTimeTicks::new(value),
)
}
fn generation() -> Generation {
Generation {
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(1),
configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
coordinator_id: CoordinatorId::new([3; 32]),
network_sidecar_id: NetworkSidecarId::new([4; 32]),
sandbox_id: SandboxId::new([5; 32]),
durable_decision: DurableDecision::Preparing,
}
}
fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant {
Participant::snapshot(
GenerationId::new(1),
role,
ParticipantState::Intent,
Some(ownership),
)
.unwrap()
}
#[test]
fn inventory_rejects_default_partial_and_helper_or_fuse_omission() {
let generation = generation();
let empty = runtime_recovery_ownership(&[], &generation).unwrap();
assert!(prove_runtime_owners_absent(&empty).is_err());
let helper = participant(
ParticipantRole::Helper,
ParticipantOwnership::Helper(HelperOwnership {
network_sidecar_id: None,
connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
}),
);
assert!(runtime_recovery_ownership(&[helper], &generation).is_err());
let incomplete_fuse = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: None,
work_path: None,
merged_path: Some(HostPath::new(b"/tmp/missing".to_vec())),
daemon: PersistedHostProcessOwnership::unassigned(),
user_namespace: Some(NamespaceIdentity {
device: DeviceId::new(6),
inode: InodeId::new(7),
}),
mount_namespace: Some(NamespaceIdentity {
device: DeviceId::new(8),
inode: InodeId::new(9),
}),
mount: Some(MountIdentity {
mount_id: MountId::new(10),
device: DeviceId::new(11),
inode: InodeId::new(12),
}),
}),
);
let prepared =
Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower);
assert!(
runtime_recovery_ownership(&[prepared.clone(), incomplete_fuse], &generation).is_err()
);
let staged_state = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: None,
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: None,
mount_namespace: None,
mount: None,
}),
);
let mut opened = 0;
let partial = runtime_recovery_ownership(&[prepared, staged_state], &generation);
assert!(partial.is_err());
if let Ok(partial) = partial {
let _ = seal_runtime_owners(&partial, |_| {
opened += 1;
Err(io::Error::from_raw_os_error(
rustix::io::Errno::SRCH.raw_os_error(),
))
});
}
assert_eq!(opened, 0);
}
#[test]
fn complete_boot_mismatch_inventory_proves_only_after_every_holder_and_mount() {
let generation = generation();
let prepared =
Participant::bare_intent(GenerationId::new(1), ParticipantRole::PreparedLower);
let state = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: None,
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: None,
mount_namespace: None,
mount: None,
}),
);
let helper = participant(
ParticipantRole::Helper,
ParticipantOwnership::Helper(HelperOwnership {
network_sidecar_id: None,
connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
}),
);
let network = Participant::bare_intent(GenerationId::new(1), ParticipantRole::Network);
let sandbox = participant(
ParticipantRole::Sandbox,
ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new([3; 32])),
runtime_root: Some(HostPath::new(b"/runtime".to_vec())),
runtime_root_identity: Some(MountIdentity {
mount_id: MountId::new(13),
device: DeviceId::new(14),
inode: InodeId::new(15),
}),
bundle_digest: Some(BundleDigest::new([4; 32])),
host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)),
attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
}),
);
let inventory =
runtime_recovery_ownership(&[prepared, state, helper, network, sandbox], &generation)
.unwrap();
assert_eq!(
prove_runtime_owners_absent(&inventory).unwrap().ownership(),
&inventory
);
}
#[test]
fn recovery_preflight_recovers_full_runtime_then_is_idempotent() {
let fixture = RecoveryFixture::new();
let mut boundaries = Vec::new();
fixture
.recover_with(|boundary, generation| {
boundaries.push((boundary, generation));
Ok(())
})
.unwrap();
assert!(
boundaries
.iter()
.any(|(boundary, _)| { *boundary == RecoveryBoundary::StateAnchor })
);
assert_eq!(
fixture.store.snapshot().unwrap().generations[0].durable_decision,
DurableDecision::Clean
);
assert!(
fixture
.catalog
.snapshot()
.unwrap()
.artifact_leases
.is_empty()
);
assert!(
fixture
.catalog
.snapshot()
.unwrap()
.prepared_artifacts
.is_empty()
);
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
));
assert!(
!runtime_domain
.join("runsc")
.join(hex_lower(&[6; 32]))
.exists()
);
recover_before_cold_start(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
&fixture.prepared,
)
.unwrap();
}
#[derive(Clone, Copy)]
enum PublicationFixtureBoundary {
TrulyAbsent,
AfterStagedIdentity,
AfterStagedSocket,
}
#[test]
fn bare_publication_intent_requires_every_fixed_label_to_be_absent() {
let absent = RecoveryFixture::new();
absent.prepare_bare_publication(PublicationFixtureBoundary::TrulyAbsent);
recover_publication(&absent.configuration, &absent.store, GenerationId::new(2)).unwrap();
assert_eq!(absent.publication(2).state(), ParticipantState::Clean);
let identity = RecoveryFixture::new();
identity.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedIdentity);
assert!(
recover_publication(
&identity.configuration,
&identity.store,
GenerationId::new(2)
)
.is_err()
);
assert!(
identity
.runtime_domain()
.join(STAGED_PUBLICATION_IDENTITY_NAME)
.is_file()
);
assert_eq!(identity.publication(2).state(), ParticipantState::Intent);
assert!(identity.publication(2).ownership().is_none());
let socket = RecoveryFixture::new();
socket.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedSocket);
assert!(
recover_publication(&socket.configuration, &socket.store, GenerationId::new(2))
.is_err()
);
assert!(
socket
.runtime_domain()
.join(STAGED_PUBLICATION_IDENTITY_NAME)
.is_file()
);
assert!(
fs::symlink_metadata(socket.runtime_domain().join(STAGED_CONTROL_SOCKET_NAME))
.unwrap()
.file_type()
.is_socket()
);
assert_eq!(socket.publication(2).state(), ParticipantState::Intent);
assert!(socket.publication(2).ownership().is_none());
}
#[test]
fn recovery_preflight_retries_every_recovery_boundary() {
for failed in [
RecoveryBoundary::Snapshot,
RecoveryBoundary::Publication,
RecoveryBoundary::Sandbox,
RecoveryBoundary::HelperAndNetwork,
RecoveryBoundary::GitRouting,
RecoveryBoundary::StateAnchor,
RecoveryBoundary::PreparedLower,
RecoveryBoundary::PrepareOperations,
RecoveryBoundary::ArtifactGc,
RecoveryBoundary::FinishGeneration,
] {
let fixture = RecoveryFixture::new();
let mut injected = false;
assert!(
fixture
.recover_with(|boundary, _| {
if boundary == failed && !injected {
injected = true;
Err("injected recovery boundary".into())
} else {
Ok(())
}
},)
.is_err()
);
assert!(injected);
fixture.recover_with(|_, _| Ok(())).unwrap();
assert_eq!(
fixture.store.snapshot().unwrap().generations[0].durable_decision,
DurableDecision::Clean
);
}
}
#[test]
fn recovery_preflight_opens_every_exact_owner_once() {
let fixture = RecoveryFixture::new();
let mut opened = BTreeMap::new();
recover_before_cold_start_with_opener(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
&fixture.prepared,
|_, _| Ok(()),
|identity| {
*opened.entry(identity.host_pid().get()).or_insert(0) += 1;
Ok(PidfsRecoveryMatch::BootMismatch)
},
|root, container| {
unlinkat(root, container, AtFlags::REMOVEDIR).map_err(|error| error.to_string())
},
)
.unwrap();
assert_eq!(opened.len(), 6);
assert!(opened.values().all(|count| *count == 1));
}
#[test]
fn publication_replacement_and_runsc_root_replacement_refuse_untouched() {
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
));
let socket = runtime_domain.join(CONTROL_SOCKET_NAME);
let old_socket = runtime_domain.join("old-control.sock");
fs::rename(&socket, &old_socket).unwrap();
let root = File::open(runtime_domain).unwrap();
drop(
UnixListener::bind(format!(
"/proc/self/fd/{}/{}",
root.as_raw_fd(),
CONTROL_SOCKET_NAME
))
.unwrap(),
);
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert!(socket.exists());
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
));
let identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME);
fs::rename(&identity, runtime_domain.join("old-identity")).unwrap();
symlink("old-identity", &identity).unwrap();
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert!(
fs::symlink_metadata(&identity)
.unwrap()
.file_type()
.is_symlink()
);
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
));
let identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME);
fs::rename(&identity, runtime_domain.join("old-identity")).unwrap();
fs::write(&identity, b"replacement").unwrap();
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert_eq!(fs::read(&identity).unwrap(), b"replacement");
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
));
let runsc = runtime_domain.join("runsc");
let old_runsc = runtime_domain.join("old-runsc");
fs::rename(&runsc, &old_runsc).unwrap();
fs::create_dir(&runsc).unwrap();
let mut deletes = 0;
let result = recover_before_cold_start_with_opener(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
&fixture.prepared,
|_, _| Ok(()),
|_| Ok(PidfsRecoveryMatch::BootMismatch),
|_, _| {
deletes += 1;
Ok(())
},
);
assert!(result.is_err());
assert_eq!(deletes, 0);
assert!(runsc.exists());
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
));
let runsc = runtime_domain.join("runsc");
fs::rename(&runsc, runtime_domain.join("old-runsc")).unwrap();
symlink("old-runsc", &runsc).unwrap();
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert!(
fs::symlink_metadata(&runsc)
.unwrap()
.file_type()
.is_symlink()
);
}
struct RecoveryFixture {
_tree: tempfile::TempDir,
configuration: ResolvedDomainConfiguration,
store: HostCoordinatorDomainStore,
catalog: Catalog,
prepared: PreparedLowerStore,
}
impl RecoveryFixture {
fn recover_with<Failpoint>(&self, failpoint: Failpoint) -> Result<(), String>
where
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
{
recover_before_cold_start_with_opener(
&self.configuration,
&self.store,
&self.catalog,
&self.prepared,
failpoint,
match_recovery_process,
|root, container| {
unlinkat(root, container, AtFlags::REMOVEDIR).map_err(|error| error.to_string())
},
)
}
#[allow(clippy::too_many_lines)]
fn new() -> Self {
let tree = tempfile::tempdir().unwrap();
let configuration = test_configuration(tree.path());
let catalog = Catalog::under_storage_root(configuration.storage_root()).unwrap();
catalog.initialize().unwrap();
let prepared = PreparedLowerStore::under_storage_root(
configuration.storage_root(),
catalog.clone(),
)
.unwrap();
let artifact_name = hex_lower(&[21; 32]);
let artifact_path = tree
.path()
.join("data/prepared-lower/artifacts")
.join(&artifact_name);
fs::create_dir(&artifact_path).unwrap();
let metadata = artifact_path.metadata().unwrap();
let artifact = PreparedArtifact {
prepared_artifact_id: PreparedArtifactId::new([21; 32]),
source_identity: SourceIdentity::new(vec![22]),
selected_manifest_digest: SelectedManifestDigest::new([23; 32]),
policy_id: PolicyId::new([24; 32]),
id_map_digest: IdMapDigest::new([25; 32]),
path: HostPath::new(format!("artifacts/{artifact_name}").into_bytes()),
device: DeviceId::new(metadata.dev()),
inode: InodeId::new(metadata.ino()),
state: ArtifactState::Available,
};
publish_artifact(&catalog, &artifact);
let domain = lazaret_core::records::Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
};
let lock = OpenOptions::new()
.create(true)
.read(true)
.truncate(false)
.write(true)
.open(tree.path().join("domain.lock"))
.unwrap();
let store = HostCoordinatorDomainStore::open(
try_exclusive_flock(lock).unwrap(),
&tree.path().join("domain.redb"),
&domain,
)
.unwrap();
let generation = Generation {
domain_id: domain.domain_id,
generation_id: GenerationId::new(1),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([3; 32]),
network_sidecar_id: NetworkSidecarId::new([4; 32]),
sandbox_id: SandboxId::new([5; 32]),
durable_decision: DurableDecision::None,
};
store
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
.unwrap();
let mut generation = generation;
generation.durable_decision = DurableDecision::Preparing;
store
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
.unwrap();
let layout = StateAnchorLayout::create(&configuration).unwrap();
let mut anchor = layout.ownership_intent(artifact.prepared_artifact_id);
let mut staged_anchor = anchor.clone();
staged_anchor.daemon = PersistedHostProcessOwnership::staged(process(5));
anchor.daemon = PersistedHostProcessOwnership::committed(process(5));
anchor.user_namespace = Some(NamespaceIdentity {
device: DeviceId::new(30),
inode: InodeId::new(31),
});
anchor.mount_namespace = Some(NamespaceIdentity {
device: DeviceId::new(32),
inode: InodeId::new(33),
});
anchor.mount = Some(MountIdentity {
mount_id: MountId::new(u64::MAX),
device: DeviceId::new(34),
inode: InodeId::new(35),
});
let prepared_ownership = PreparedLowerOwnership {
prepared_artifact_id: Some(artifact.prepared_artifact_id),
source_identity: Some(artifact.source_identity.clone()),
lease_state: Some(ParticipantState::LeaseHeld),
path: Some(artifact.path.clone()),
device: Some(artifact.device),
inode: Some(artifact.inode),
released_state_anchor: None,
};
let helper = HelperOwnership {
network_sidecar_id: None,
connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
runtime_supervisor: PersistedHostProcessOwnership::committed(process(2)),
};
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
));
let runsc_root = runtime_domain.join("runsc");
fs::create_dir(&runsc_root).unwrap();
let container_id = ContainerId::new([6; 32]);
fs::create_dir(runsc_root.join(hex_lower(container_id.as_bytes()))).unwrap();
let sandbox = SandboxOwnership {
container_id: Some(container_id),
runtime_root: Some(HostPath::new(runsc_root.as_os_str().as_bytes().to_vec())),
runtime_root_identity: Some(current_mount_identity(&runsc_root).unwrap()),
bundle_digest: Some(BundleDigest::new([7; 32])),
host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
runtime_supervisor: PersistedHostProcessOwnership::staged(process(2)),
attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
};
let publication_socket = runtime_domain.join(CONTROL_SOCKET_NAME);
let publication_root = File::open(runtime_domain).unwrap();
drop(
UnixListener::bind(format!(
"/proc/self/fd/{}/{}",
publication_root.as_raw_fd(),
CONTROL_SOCKET_NAME
))
.unwrap(),
);
let publication_identity = runtime_domain.join(PUBLICATION_IDENTITY_NAME);
fs::write(&publication_identity, b"identity").unwrap();
let publication = PublicationOwnership {
staged_socket_path: Some(HostPath::new(
STAGED_CONTROL_SOCKET_NAME.as_bytes().to_vec(),
)),
staged_identity_path: Some(HostPath::new(
STAGED_PUBLICATION_IDENTITY_NAME.as_bytes().to_vec(),
)),
socket_path: Some(HostPath::new(CONTROL_SOCKET_NAME.as_bytes().to_vec())),
identity_path: Some(HostPath::new(PUBLICATION_IDENTITY_NAME.as_bytes().to_vec())),
socket_identity: Some(path_identity(runtime_domain, &publication_socket)),
identity_file_identity: Some(path_identity(runtime_domain, &publication_identity)),
};
let git_path = runtime_domain.join("git-routing-fixture");
fs::create_dir(&git_path).unwrap();
let git_routing = GitRoutingOwnership {
mountpoint: Some(HostPath::new(git_path.as_os_str().as_bytes().to_vec())),
host_root: Some(HostPath::new(b"/git-host".to_vec())),
isolated_root: Some(HostPath::new(b"/git-isolated".to_vec())),
daemon: PersistedHostProcessOwnership::staged(process(6)),
mount: Some(MountIdentity {
mount_id: MountId::new(u64::MAX - 1),
device: DeviceId::new(40),
inode: InodeId::new(41),
}),
};
store
.apply(&[
DomainStoreMutation::PutParticipant(Participant::bare_intent(
generation.generation_id,
ParticipantRole::PreparedLower,
)),
DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation.generation_id,
ParticipantRole::PreparedLower,
ParticipantState::Intent,
Some(ParticipantOwnership::PreparedLower(
PreparedLowerOwnership {
lease_state: Some(ParticipantState::Intent),
..prepared_ownership.clone()
},
)),
)
.unwrap(),
),
])
.unwrap();
let lease = ArtifactLease {
prepared_artifact_id: artifact.prepared_artifact_id,
domain_id: domain.domain_id,
generation_id: generation.generation_id,
};
catalog.acquire_lease_checked(&artifact, &lease).unwrap();
store
.apply(&[
DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation.generation_id,
ParticipantRole::PreparedLower,
ParticipantState::LeaseHeld,
Some(ParticipantOwnership::PreparedLower(prepared_ownership)),
)
.unwrap(),
),
DomainStoreMutation::PutParticipant(Participant::bare_intent(
generation.generation_id,
ParticipantRole::StateAnchor,
)),
DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation.generation_id,
ParticipantRole::StateAnchor,
ParticipantState::Intent,
Some(ParticipantOwnership::StateAnchor(staged_anchor)),
)
.unwrap(),
),
DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation.generation_id,
ParticipantRole::StateAnchor,
ParticipantState::Intent,
Some(ParticipantOwnership::StateAnchor(anchor.clone())),
)
.unwrap(),
),
DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation.generation_id,
ParticipantRole::StateAnchor,
ParticipantState::Prepared,
Some(ParticipantOwnership::StateAnchor(anchor)),
)
.unwrap(),
),
DomainStoreMutation::PutParticipant(Participant::bare_intent(
generation.generation_id,
ParticipantRole::Helper,
)),
DomainStoreMutation::PutParticipant(participant(
ParticipantRole::Helper,
ParticipantOwnership::Helper(helper),
)),
DomainStoreMutation::PutParticipant(Participant::bare_intent(
generation.generation_id,
ParticipantRole::Network,
)),
DomainStoreMutation::PutParticipant(Participant::bare_intent(
generation.generation_id,
ParticipantRole::Sandbox,
)),
DomainStoreMutation::PutParticipant(participant(
ParticipantRole::Sandbox,
ParticipantOwnership::Sandbox(sandbox),
)),
DomainStoreMutation::PutParticipant(Participant::bare_intent(
generation.generation_id,
ParticipantRole::GitRouting,
)),
DomainStoreMutation::PutParticipant(participant(
ParticipantRole::GitRouting,
ParticipantOwnership::GitRouting(git_routing),
)),
DomainStoreMutation::PutParticipant(Participant::bare_intent(
generation.generation_id,
ParticipantRole::Publication,
)),
DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation.generation_id,
ParticipantRole::Publication,
ParticipantState::Intent,
Some(ParticipantOwnership::Publication(publication.clone())),
)
.unwrap(),
),
DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation.generation_id,
ParticipantRole::Publication,
ParticipantState::Committed,
Some(ParticipantOwnership::Publication(publication)),
)
.unwrap(),
),
])
.unwrap();
Self {
_tree: tree,
configuration,
store,
catalog,
prepared,
}
}
fn runtime_domain(&self) -> PathBuf {
Path::new(std::ffi::OsStr::from_bytes(
self.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
))
.to_path_buf()
}
fn prepare_bare_publication(&self, boundary: PublicationFixtureBoundary) {
let runtime_domain = self.runtime_domain();
fs::remove_file(runtime_domain.join(CONTROL_SOCKET_NAME)).unwrap();
fs::remove_file(runtime_domain.join(PUBLICATION_IDENTITY_NAME)).unwrap();
let generation = Generation {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(2),
configuration_fingerprint: self.configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([13; 32]),
network_sidecar_id: NetworkSidecarId::new([14; 32]),
sandbox_id: SandboxId::new([15; 32]),
durable_decision: DurableDecision::None,
};
self.store
.apply(&[DomainStoreMutation::PutGeneration(generation.clone())])
.unwrap();
self.store
.apply(&[
DomainStoreMutation::PutGeneration(Generation {
durable_decision: DurableDecision::Preparing,
..generation
}),
DomainStoreMutation::PutParticipant(Participant::bare_intent(
GenerationId::new(2),
ParticipantRole::Publication,
)),
])
.unwrap();
if !matches!(boundary, PublicationFixtureBoundary::TrulyAbsent) {
fs::write(
runtime_domain.join(STAGED_PUBLICATION_IDENTITY_NAME),
b"identity",
)
.unwrap();
}
if matches!(boundary, PublicationFixtureBoundary::AfterStagedSocket) {
let root = File::open(runtime_domain).unwrap();
drop(
UnixListener::bind(format!(
"/proc/self/fd/{}/{}",
root.as_raw_fd(),
STAGED_CONTROL_SOCKET_NAME
))
.unwrap(),
);
}
}
fn publication(&self, generation: u64) -> Participant {
role(
&self.store,
GenerationId::new(generation),
ParticipantRole::Publication,
)
.unwrap()
.unwrap()
}
}
fn path_identity(root: &Path, path: &Path) -> MountIdentity {
let root_identity = current_mount_identity(root).unwrap();
let metadata = fs::symlink_metadata(path).unwrap();
MountIdentity {
mount_id: root_identity.mount_id,
device: DeviceId::new(metadata.dev()),
inode: InodeId::new(metadata.ino()),
}
}
fn publish_artifact(catalog: &Catalog, artifact: &PreparedArtifact) {
let intent = PrepareOperation {
operation_id: PrepareOperationId::new([12; 32]),
expected_artifact_id: artifact.prepared_artifact_id,
source_identity: artifact.source_identity.clone(),
selected_manifest_digest: artifact.selected_manifest_digest,
policy_id: artifact.policy_id,
id_map_digest: artifact.id_map_digest,
stage_path: HostPath::new(b"stages/fixture".to_vec()),
stage_state: PrepareStageState::Intent,
device: None,
inode: None,
};
catalog.record_prepare(&intent).unwrap();
let owned = PrepareOperation {
stage_state: PrepareStageState::Owned,
device: Some(artifact.device),
inode: Some(artifact.inode),
..intent.clone()
};
catalog.bind_prepare_stage(&intent, &owned).unwrap();
catalog.publish(&owned, artifact).unwrap();
}
fn test_configuration(root: &Path) -> ResolvedDomainConfiguration {
let configuration_directory = root.join("configuration");
let workspace = root.join("workspace");
let runtime = root.join("runtime");
fs::create_dir(&configuration_directory).unwrap();
fs::create_dir(&workspace).unwrap();
fs::create_dir(&runtime).unwrap();
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 8443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
root.join("data").display()
),
)
.unwrap();
load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace,
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: default_sandbox_layout().unwrap(),
})
.unwrap()
}
}

876
crates/lazaret-host-coordinator/src/runtime.rs
View file

@ -372,879 +372,7 @@ impl<'store> SandboxCoordinator<'store> {
}
}
pub struct AwaitingAttachedRunsc {
supervisor: Option<RetainedProcess>,
control: Option<CoordinatorChannel>,
lifetime: Option<std::os::fd::OwnedFd>,
staged: Option<StagedProcessOwnership>,
helper_staged: Option<StagedProcessOwnership>,
git_routing_required: bool,
git_routing_mount: Option<MountIdentity>,
network_evidence: Option<NetworkReadinessEvidence>,
runsc_command: lazaret_runtime_supervisor::sandbox::RunscCommand,
}
impl AwaitingAttachedRunsc {
#[must_use]
pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
self.supervisor.as_ref().map(RetainedProcess::identity)
}
/// Requests RS-owned State Anchor acquisition and commits exact staged and mount evidence.
///
/// # Errors
/// Refuses missing intent, event mismatch, changed ownership, or channel failure.
pub fn mount_state_anchor(
&mut self,
coordinator: &StateAnchorCoordinator<'_>,
prepared_artifact_id: PreparedArtifactId,
) -> Result<MountIdentity, SandboxCoordinatorError> {
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
control.mount_state_anchor()?;
let staged = match control.receive()? {
SupervisorEvent::StateAnchorStaged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stage the State Anchor daemon",
));
}
};
coordinator
.stage_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
let acknowledgment = coordinator
.acknowledge_staged_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
control.acknowledge_state_anchor(acknowledgment)?;
let (ready_staged, user_namespace, mount_namespace, mount) = match control.receive()? {
SupervisorEvent::StateAnchorReady {
staged,
user_namespace,
mount_namespace,
mount,
} => (staged, user_namespace, mount_namespace, mount),
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report State Anchor readiness",
));
}
};
if ready_staged != staged {
return Err(SandboxCoordinatorError::InvalidState(
"State Anchor readiness changed daemon ownership",
));
}
let readiness =
lazaret_state_anchor::VerifiedStateAnchorReadiness::from_parts(ready_staged, mount);
coordinator
.commit_daemon_ready(readiness, user_namespace, mount_namespace)
.and_then(|()| {
coordinator.commit_prepared(staged.generation_id(), prepared_artifact_id, mount)
})
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
Ok(mount)
}
/// Requests generation-local routing and commits exact staged and mount evidence.
///
/// # Errors
/// Refuses missing intent, event mismatch, changed ownership, or channel failure.
pub fn start_git_routing(
&mut self,
coordinator: &GitRoutingCoordinator<'_>,
) -> Result<MountIdentity, SandboxCoordinatorError> {
if !self.git_routing_required || self.git_routing_mount.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"Git routing is not awaiting start",
));
}
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
control.start_git_routing()?;
let staged = match control.receive()? {
SupervisorEvent::GitRoutingStaged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stage the Git router",
));
}
};
coordinator
.stage_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
let acknowledgment = coordinator
.acknowledge_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
control.acknowledge_git_routing(acknowledgment)?;
let (ready_staged, mount) = match control.receive()? {
SupervisorEvent::GitRoutingReady { staged, mount } => (staged, mount),
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report Git-routing readiness",
));
}
};
if ready_staged != staged {
return Err(SandboxCoordinatorError::InvalidState(
"Git-routing readiness changed daemon ownership",
));
}
coordinator
.commit_ready(staged, mount)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
self.git_routing_mount = Some(mount);
Ok(mount)
}
/// Releases the RS-owned sidecar install path after exact State Anchor readiness.
///
/// # Errors
/// Refuses an unexpected RS response or channel failure.
pub fn start_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
control.start_sidecar()?;
if control.receive()? == SupervisorEvent::SidecarStarted {
Ok(())
} else {
Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not start the sidecar path",
))
}
}
/// Launches the RS through the HC broker and commits its staged ownership before execution.
///
/// # Errors
/// Refuses any pipe, channel, broker, staging, or acknowledgment failure.
pub fn launch(
coordinator: &SandboxCoordinator<'_>,
launch: &RuntimeSupervisorLaunch,
) -> Result<Self, SandboxCoordinatorError> {
if launch.network_sidecar_id().is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"networked Runtime Supervisor requires helper intent",
));
}
Self::launch_inner(coordinator, None, launch)
}
/// Launches the RS only after independent durable helper intent is recorded.
///
/// # Errors
/// Refuses a non-network launch or any sandbox/helper staging failure.
pub fn launch_networked(
coordinator: &SandboxCoordinator<'_>,
network: &NetworkCoordinator<'_>,
launch: &RuntimeSupervisorLaunch,
) -> Result<Self, SandboxCoordinatorError> {
if launch.network_sidecar_id().is_none() {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor launch has no network policy",
));
}
Self::launch_inner(coordinator, Some(network), launch)
}
fn launch_inner(
coordinator: &SandboxCoordinator<'_>,
network: Option<&NetworkCoordinator<'_>>,
launch: &RuntimeSupervisorLaunch,
) -> Result<Self, SandboxCoordinatorError> {
let parent = current_process_identity()?;
let (coordinator_control, supervisor_control) = private_seqpacket_pair()?;
let (supervisor_lifetime, coordinator_lifetime) =
pipe_with(PipeFlags::CLOEXEC).map_err(io::Error::from)?;
let process = child_broker().acquire_with_stdio(
parent,
launch.ownership(),
launch.target(),
launch.child_standard_io(supervisor_control, supervisor_lifetime)?,
|staged| {
coordinator
.stage_runtime_supervisor(staged)
.map_err(io::Error::other)?;
if let Some(network) = network {
network
.begin_helper_intent(
staged.generation_id(),
launch.network_sidecar_id().ok_or_else(|| {
io::Error::other("network sidecar identity is absent")
})?,
staged.identity(),
)
.map_err(io::Error::other)?;
}
Ok(())
},
|staged| {
coordinator
.acknowledge_runtime_supervisor(staged)
.map_err(io::Error::other)
},
)?;
Ok(Self {
supervisor: Some(process),
control: Some(CoordinatorChannel::new(coordinator_control)),
lifetime: Some(coordinator_lifetime),
staged: None,
helper_staged: None,
git_routing_required: launch.has_git_routing(),
git_routing_mount: None,
network_evidence: None,
runsc_command: launch.runsc_command().clone(),
})
}
/// Receives, persists, and acknowledges the exact helper child acquisition.
///
/// # Errors
/// Refuses unexpected events, changed ownership, or a missing network launch.
pub fn receive_helper_staged(
&mut self,
network: &NetworkCoordinator<'_>,
) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
if self.helper_staged.is_some() || self.network_evidence.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"helper stage was already received",
));
}
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
let staged = match control.receive()? {
SupervisorEvent::HelperStaged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stage the helper first",
));
}
};
let runtime_supervisor = self
.supervisor
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor handle is absent",
))?
.identity();
network.stage_helper(staged, runtime_supervisor)?;
let acknowledgment = network.acknowledge_helper(staged)?;
control.acknowledge_helper(acknowledgment)?;
self.helper_staged = Some(staged);
Ok(staged)
}
/// Commits helper readiness, independently observes the namespace, and validates attestation.
///
/// # Errors
/// Refuses event ordering, namespace mutation, or any attestation mismatch.
pub fn receive_network_readiness<BuildProbeRequest>(
&mut self,
network: &NetworkCoordinator<'_>,
policy: &CanonicalDerivedNetworkPolicy,
build_probe_request: BuildProbeRequest,
expected_reachability: &ReachabilityExpectation,
) -> Result<NetworkReadinessEvidence, SandboxCoordinatorError>
where
BuildProbeRequest: FnOnce(NamespaceIdentity) -> NetworkProbeRequest,
{
let staged = self
.helper_staged
.ok_or(SandboxCoordinatorError::InvalidState(
"helper ownership was not staged",
))?;
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
if control.receive()? != SupervisorEvent::HelperReady(staged) {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor helper readiness mismatched ownership",
));
}
network.commit_helper(staged)?;
let attestation = match control.receive()? {
SupervisorEvent::NetworkAttestation(attestation) => attestation,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not provide network attestation",
));
}
};
let namespace = network.begin_network_intent(
attestation.generation_id,
policy.digest(),
attestation.runtime_supervisor,
)?;
network.validate_attestation(policy, &attestation)?;
let probe_request = build_probe_request(namespace);
if probe_request.domain_id != attestation.domain_id
|| probe_request.generation_id != attestation.generation_id
|| probe_request.enforcement_namespace != attestation.enforcement_namespace
{
return Err(SandboxCoordinatorError::InvalidState(
"network probe request mismatches attestation",
));
}
control.acknowledge_network_attestation(probe_request)?;
let probe = match control.receive()? {
SupervisorEvent::NetworkProbe(probe) => probe,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not return the network probe",
));
}
};
let evidence =
network.commit_readiness(policy, attestation, probe, expected_reachability)?;
control.acknowledge_network_readiness()?;
self.network_evidence = Some(evidence.clone());
Ok(evidence)
}
/// Receives and durably stages the exact RS child without releasing its barrier.
///
/// # Errors
/// Refuses unexpected, duplicate, or mismatched messages and durable-state failures.
pub fn receive_staged(
&mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
if self.staged.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc stage was already received",
));
}
if self.helper_staged.is_some() && self.network_evidence.is_none() {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc cannot start before network attestation",
));
}
if self.git_routing_required && self.git_routing_mount.is_none() {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc cannot start before Git-routing readiness",
));
}
let event = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?
.receive()?;
let staged = match event {
SupervisorEvent::Staged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor sent readiness before staged ownership",
));
}
};
let acknowledgment = coordinator.stage_attached_runsc(staged)?;
self.staged = Some(acknowledgment.committed());
Ok(staged)
}
/// Releases the exact pending runsc barrier with its durable typed acknowledgment.
///
/// # Errors
/// Refuses a different or absent staged tuple or channel failure.
pub fn acknowledge(
mut self,
staged: StagedProcessOwnership,
handoff: &SandboxHandoffAuthority<'_>,
) -> Result<AwaitingSandboxReadiness, SandboxCoordinatorError> {
if self.staged != Some(staged) {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc acknowledgment does not match pending stage",
));
}
if staged.domain_id() != handoff.domain_id()
|| !handoff.matches_command(&self.runsc_command)
{
return Err(SandboxCoordinatorError::InvalidState(
"sandbox handoff authority does not match attached runsc",
));
}
handoff
.revalidate()
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
self.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?
.acknowledge_staged(ProcessOwnershipCommitAck::v1(staged))?;
Ok(AwaitingSandboxReadiness {
supervisor: self.supervisor.take(),
control: self.control.take(),
lifetime: self.lifetime.take(),
staged,
network_evidence: self.network_evidence.take(),
})
}
/// # Errors
/// Returns an error if the exact RS cannot be terminated and reaped.
pub fn abort(mut self) -> Result<(), SandboxCoordinatorError> {
self.lifetime.take();
cleanup_supervisor(&mut self.supervisor)?;
Ok(())
}
}
impl Drop for AwaitingAttachedRunsc {
fn drop(&mut self) {
self.lifetime.take();
let _ = cleanup_supervisor(&mut self.supervisor);
}
}
pub struct AwaitingSandboxReadiness {
supervisor: Option<RetainedProcess>,
control: Option<CoordinatorChannel>,
lifetime: Option<std::os::fd::OwnedFd>,
staged: StagedProcessOwnership,
network_evidence: Option<NetworkReadinessEvidence>,
}
impl AwaitingSandboxReadiness {
/// Commits only the exact ready attached runsc after RS capability removal.
///
/// # Errors
/// Refuses a mismatched readiness tuple, durable commit failure, or channel failure.
pub fn commit(
self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
if self.network_evidence.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"networked sandbox requires a successful sandbox-view probe",
));
}
self.commit_after_probe(coordinator, None)
}
/// Commits the Sandbox only after the enforcement-namespace probe is durably accepted.
///
/// # Errors
/// Refuses missing attestation, failed/mutated probe evidence, or any runtime mismatch.
pub fn commit_networked(
mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<(CommittedSandboxRuntime, NetworkReadinessEvidence), SandboxCoordinatorError> {
let evidence =
self.network_evidence
.take()
.ok_or(SandboxCoordinatorError::InvalidState(
"network readiness evidence is absent",
))?;
let runtime = self.commit_after_probe(coordinator, Some(evidence.clone()))?;
Ok((runtime, evidence))
}
fn commit_after_probe(
mut self,
coordinator: &SandboxCoordinator<'_>,
network_evidence: Option<NetworkReadinessEvidence>,
) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
let event = control.receive()?;
let staged = match event {
SupervisorEvent::Ready { staged, .. } => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report sandbox readiness",
));
}
};
if staged != self.staged {
return Err(SandboxCoordinatorError::InvalidState(
"sandbox readiness changed attached-runsc ownership",
));
}
coordinator.commit_ready(staged)?;
control.acknowledge_sandbox_committed()?;
Ok(CommittedSandboxRuntime {
generation_id: staged.generation_id(),
supervisor: self.supervisor.take(),
control: self
.control
.take()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?,
lifetime: self.lifetime.take(),
network_evidence,
})
}
}
impl Drop for AwaitingSandboxReadiness {
fn drop(&mut self) {
self.lifetime.take();
let _ = cleanup_supervisor(&mut self.supervisor);
}
}
pub struct CommittedSandboxRuntime {
generation_id: GenerationId,
supervisor: Option<RetainedProcess>,
control: CoordinatorChannel,
lifetime: Option<std::os::fd::OwnedFd>,
network_evidence: Option<NetworkReadinessEvidence>,
}
impl CommittedSandboxRuntime {
fn receive_runtime_event(&mut self) -> Result<SupervisorEvent, SandboxCoordinatorError> {
loop {
let event = self.control.receive()?;
if event == SupervisorEvent::HelperExited {
self.network_evidence = None;
} else {
return Ok(event);
}
}
}
#[must_use]
pub const fn network_evidence(&self) -> Option<&NetworkReadinessEvidence> {
self.network_evidence.as_ref()
}
/// Withdraws live readiness after the RS reports exact helper-only loss.
///
/// # Errors
/// Refuses any event other than connectivity-helper loss.
pub fn wait_for_helper_exit(&mut self) -> Result<(), SandboxCoordinatorError> {
if self.control.receive()? != SupervisorEvent::HelperExited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report helper-only loss",
));
}
self.network_evidence = None;
Ok(())
}
#[must_use]
pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
self.supervisor.as_ref().map(RetainedProcess::identity)
}
/// # Errors
/// Returns an error if the exact retained RS pidfd action fails.
pub fn signal_runtime_supervisor(&self, signal: Signal) -> Result<(), SandboxCoordinatorError> {
self.supervisor
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor handle is absent",
))?
.send_signal(signal)?;
Ok(())
}
/// # Errors
/// Returns an error if the RS cannot apply the exact retained-runsc pidfd action.
pub fn signal_attached_runsc(
&mut self,
signal: AttachedRunscSignal,
) -> Result<(), SandboxCoordinatorError> {
self.control.signal_attached_runsc(signal)?;
if self.receive_runtime_event()? == SupervisorEvent::SignalApplied(signal) {
Ok(())
} else {
Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not confirm the exact runsc action",
))
}
}
/// Stops and durably cleans only the exact attached sandbox runtime.
///
/// # Errors
/// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
pub fn stop_sandbox(
&mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<(), SandboxCoordinatorError> {
coordinator.begin_release(self.generation_id)?;
self.control.stop_sandbox()?;
if self.receive_runtime_event()? != SupervisorEvent::SandboxStopped {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stop the sandbox",
));
}
coordinator.complete_clean_after_absence(self.generation_id)
}
/// Stops exact generation-local Git routing after sandbox absence.
///
/// # Errors
/// Refuses missing release intent, changed mount evidence, or an unexpected RS response.
pub fn stop_git_routing(
&mut self,
coordinator: &GitRoutingCoordinator<'_>,
) -> Result<(), SandboxCoordinatorError> {
let expected = coordinator
.begin_release(self.generation_id)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
self.control.stop_git_routing()?;
match self.receive_runtime_event()? {
SupervisorEvent::GitRoutingStopped { mount } if mount == expected => coordinator
.complete_clean(self.generation_id, mount)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error))),
SupervisorEvent::Refused(detail) => {
Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
}
_ => Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stop exact Git routing",
)),
}
}
/// Stops only the RS-owned network resources after sandbox absence.
///
/// # Errors
/// Refuses an unexpected RS response or channel failure.
pub fn stop_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
self.control.stop_sidecar()?;
if self.receive_runtime_event()? == SupervisorEvent::SidecarStopped {
self.network_evidence = None;
Ok(())
} else {
Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stop the sidecar",
))
}
}
/// Requests exact RS-owned State Anchor unmount after dependent runtime resources stop.
///
/// # Errors
/// Refuses an unexpected RS response or channel failure.
pub fn unmount_state_anchor(
&mut self,
expected_mount: MountIdentity,
) -> Result<(), SandboxCoordinatorError> {
self.control.unmount_state_anchor()?;
match self.receive_runtime_event()? {
SupervisorEvent::StateAnchorUnmounted { mount } if mount == expected_mount => Ok(()),
_ => Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not confirm the exact State Anchor unmount",
)),
}
}
/// Requests a fresh namespace-local observation of exact State Anchor absence.
///
/// # Errors
/// Refuses an RS refusal, echoed mount, mismatched authority, or channel failure.
pub fn observe_state_anchor_absence(
&mut self,
user_namespace: NamespaceIdentity,
mount_namespace: NamespaceIdentity,
mount: MountIdentity,
) -> Result<StateAnchorAbsenceObservation, SandboxCoordinatorError> {
self.control
.observe_state_anchor_absence(user_namespace, mount_namespace, mount)?;
match self.receive_runtime_event()? {
SupervisorEvent::StateAnchorAbsenceObserved {
user_namespace: observed_user_namespace,
mount_namespace: observed_mount_namespace,
mount: observed_mount,
observed,
} if observed_user_namespace == user_namespace
&& observed_mount_namespace == mount_namespace
&& observed_mount == mount
&& observed != mount =>
{
Ok(StateAnchorAbsenceObservation {
user_namespace: observed_user_namespace,
mount_namespace: observed_mount_namespace,
mount: observed_mount,
observed,
})
}
SupervisorEvent::Refused(detail) => {
Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
}
_ => Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not provide exact State Anchor absence evidence",
)),
}
}
/// Gracefully exits and reaps the resource-empty exact Runtime Supervisor.
///
/// # Errors
/// Refuses an unexpected response or an RS that does not exit.
pub fn graceful_exit(mut self) -> Result<(), SandboxCoordinatorError> {
self.control.exit()?;
if self.receive_runtime_event()? != SupervisorEvent::Exited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not acknowledge graceful exit",
));
}
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)
}
/// Waits for an exact attached-runsc loss report and reaps the retained RS.
///
/// # Errors
/// Refuses any other event or an RS that does not then exit.
pub fn wait_for_attached_runsc_exit(mut self) -> Result<(), SandboxCoordinatorError> {
if self.receive_runtime_event()? != SupervisorEvent::AttachedRunscExited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report attached-runsc loss",
));
}
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)
}
/// Waits for an externally stopped exact RS handle to exit and reaps it.
///
/// # Errors
/// Refuses an RS that remains present beyond the bounded observation.
pub fn wait_for_runtime_supervisor_exit(mut self) -> Result<(), SandboxCoordinatorError> {
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)
}
/// Performs ordered normal stop and reaps the same retained RS handle.
///
/// # Errors
/// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
pub fn stop(
mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<(), SandboxCoordinatorError> {
coordinator.begin_release(self.generation_id)?;
self.control.stop()?;
if self.receive_runtime_event()? != SupervisorEvent::Stopped {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not complete sandbox stop",
));
}
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)?;
coordinator.complete_clean_after_absence(self.generation_id)
}
}
impl Drop for CommittedSandboxRuntime {
fn drop(&mut self) {
self.lifetime.take();
let _ = cleanup_supervisor(&mut self.supervisor);
}
}
fn sandbox_participant(
generation_id: GenerationId,
state: ParticipantState,
ownership: SandboxOwnership,
) -> Result<Participant, SandboxCoordinatorError> {
Participant::snapshot(
generation_id,
ParticipantRole::Sandbox,
state,
Some(ParticipantOwnership::Sandbox(ownership)),
)
.map_err(|_| SandboxCoordinatorError::InvalidState("Sandbox participant shape is invalid"))
}
fn sandbox_role(participants: &[Participant], generation_id: GenerationId) -> Option<&Participant> {
participants.iter().find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::Sandbox
})
}
fn cleanup_supervisor(
supervisor: &mut Option<RetainedProcess>,
) -> Result<(), SandboxCoordinatorError> {
let Some(process) = supervisor.take() else {
return Ok(());
};
if !process.has_exited()? {
process.send_signal(Signal::KILL)?;
}
child_broker().reap(process)?;
Ok(())
}
fn wait_and_reap_supervisor(
supervisor: &mut Option<RetainedProcess>,
) -> Result<(), SandboxCoordinatorError> {
let Some(process) = supervisor.take() else {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor handle is absent",
));
};
let deadline = Instant::now() + SUPERVISOR_EXIT_TIMEOUT;
while !process.has_exited()? && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(5));
}
if !process.has_exited()? {
process.send_signal(Signal::KILL)?;
child_broker().reap(process)?;
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not exit after sandbox stop",
));
}
child_broker().reap(process)?;
Ok(())
}
include!("runtime/acquisition.rs");
include!("runtime/committed.rs");
#[cfg(test)]
mod tests;

565
crates/lazaret-host-coordinator/src/runtime/acquisition.rs Normal file
View file

@ -0,0 +1,565 @@
pub struct AwaitingAttachedRunsc {
supervisor: Option<RetainedProcess>,
control: Option<CoordinatorChannel>,
lifetime: Option<std::os::fd::OwnedFd>,
staged: Option<StagedProcessOwnership>,
helper_staged: Option<StagedProcessOwnership>,
git_routing_required: bool,
git_routing_mount: Option<MountIdentity>,
network_evidence: Option<NetworkReadinessEvidence>,
runsc_command: lazaret_runtime_supervisor::sandbox::RunscCommand,
}
impl AwaitingAttachedRunsc {
#[must_use]
pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
self.supervisor.as_ref().map(RetainedProcess::identity)
}
/// Requests RS-owned State Anchor acquisition and commits exact staged and mount evidence.
///
/// # Errors
/// Refuses missing intent, event mismatch, changed ownership, or channel failure.
pub fn mount_state_anchor(
&mut self,
coordinator: &StateAnchorCoordinator<'_>,
prepared_artifact_id: PreparedArtifactId,
) -> Result<MountIdentity, SandboxCoordinatorError> {
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
control.mount_state_anchor()?;
let staged = match control.receive()? {
SupervisorEvent::StateAnchorStaged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stage the State Anchor daemon",
));
}
};
coordinator
.stage_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
let acknowledgment = coordinator
.acknowledge_staged_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
control.acknowledge_state_anchor(acknowledgment)?;
let (ready_staged, user_namespace, mount_namespace, mount) = match control.receive()? {
SupervisorEvent::StateAnchorReady {
staged,
user_namespace,
mount_namespace,
mount,
} => (staged, user_namespace, mount_namespace, mount),
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report State Anchor readiness",
));
}
};
if ready_staged != staged {
return Err(SandboxCoordinatorError::InvalidState(
"State Anchor readiness changed daemon ownership",
));
}
let readiness =
lazaret_state_anchor::VerifiedStateAnchorReadiness::from_parts(ready_staged, mount);
coordinator
.commit_daemon_ready(readiness, user_namespace, mount_namespace)
.and_then(|()| {
coordinator.commit_prepared(staged.generation_id(), prepared_artifact_id, mount)
})
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
Ok(mount)
}
/// Requests generation-local routing and commits exact staged and mount evidence.
///
/// # Errors
/// Refuses missing intent, event mismatch, changed ownership, or channel failure.
pub fn start_git_routing(
&mut self,
coordinator: &GitRoutingCoordinator<'_>,
) -> Result<MountIdentity, SandboxCoordinatorError> {
if !self.git_routing_required || self.git_routing_mount.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"Git routing is not awaiting start",
));
}
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
control.start_git_routing()?;
let staged = match control.receive()? {
SupervisorEvent::GitRoutingStaged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stage the Git router",
));
}
};
coordinator
.stage_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
let acknowledgment = coordinator
.acknowledge_daemon(staged)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
control.acknowledge_git_routing(acknowledgment)?;
let (ready_staged, mount) = match control.receive()? {
SupervisorEvent::GitRoutingReady { staged, mount } => (staged, mount),
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report Git-routing readiness",
));
}
};
if ready_staged != staged {
return Err(SandboxCoordinatorError::InvalidState(
"Git-routing readiness changed daemon ownership",
));
}
coordinator
.commit_ready(staged, mount)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
self.git_routing_mount = Some(mount);
Ok(mount)
}
/// Releases the RS-owned sidecar install path after exact State Anchor readiness.
///
/// # Errors
/// Refuses an unexpected RS response or channel failure.
pub fn start_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
control.start_sidecar()?;
if control.receive()? == SupervisorEvent::SidecarStarted {
Ok(())
} else {
Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not start the sidecar path",
))
}
}
/// Launches the RS through the HC broker and commits its staged ownership before execution.
///
/// # Errors
/// Refuses any pipe, channel, broker, staging, or acknowledgment failure.
pub fn launch(
coordinator: &SandboxCoordinator<'_>,
launch: &RuntimeSupervisorLaunch,
) -> Result<Self, SandboxCoordinatorError> {
if launch.network_sidecar_id().is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"networked Runtime Supervisor requires helper intent",
));
}
Self::launch_inner(coordinator, None, launch)
}
/// Launches the RS only after independent durable helper intent is recorded.
///
/// # Errors
/// Refuses a non-network launch or any sandbox/helper staging failure.
pub fn launch_networked(
coordinator: &SandboxCoordinator<'_>,
network: &NetworkCoordinator<'_>,
launch: &RuntimeSupervisorLaunch,
) -> Result<Self, SandboxCoordinatorError> {
if launch.network_sidecar_id().is_none() {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor launch has no network policy",
));
}
Self::launch_inner(coordinator, Some(network), launch)
}
fn launch_inner(
coordinator: &SandboxCoordinator<'_>,
network: Option<&NetworkCoordinator<'_>>,
launch: &RuntimeSupervisorLaunch,
) -> Result<Self, SandboxCoordinatorError> {
let parent = current_process_identity()?;
let (coordinator_control, supervisor_control) = private_seqpacket_pair()?;
let (supervisor_lifetime, coordinator_lifetime) =
pipe_with(PipeFlags::CLOEXEC).map_err(io::Error::from)?;
let process = child_broker().acquire_with_stdio(
parent,
launch.ownership(),
launch.target(),
launch.child_standard_io(supervisor_control, supervisor_lifetime)?,
|staged| {
coordinator
.stage_runtime_supervisor(staged)
.map_err(io::Error::other)?;
if let Some(network) = network {
network
.begin_helper_intent(
staged.generation_id(),
launch.network_sidecar_id().ok_or_else(|| {
io::Error::other("network sidecar identity is absent")
})?,
staged.identity(),
)
.map_err(io::Error::other)?;
}
Ok(())
},
|staged| {
coordinator
.acknowledge_runtime_supervisor(staged)
.map_err(io::Error::other)
},
)?;
Ok(Self {
supervisor: Some(process),
control: Some(CoordinatorChannel::new(coordinator_control)),
lifetime: Some(coordinator_lifetime),
staged: None,
helper_staged: None,
git_routing_required: launch.has_git_routing(),
git_routing_mount: None,
network_evidence: None,
runsc_command: launch.runsc_command().clone(),
})
}
/// Receives, persists, and acknowledges the exact helper child acquisition.
///
/// # Errors
/// Refuses unexpected events, changed ownership, or a missing network launch.
pub fn receive_helper_staged(
&mut self,
network: &NetworkCoordinator<'_>,
) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
if self.helper_staged.is_some() || self.network_evidence.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"helper stage was already received",
));
}
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
let staged = match control.receive()? {
SupervisorEvent::HelperStaged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stage the helper first",
));
}
};
let runtime_supervisor = self
.supervisor
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor handle is absent",
))?
.identity();
network.stage_helper(staged, runtime_supervisor)?;
let acknowledgment = network.acknowledge_helper(staged)?;
control.acknowledge_helper(acknowledgment)?;
self.helper_staged = Some(staged);
Ok(staged)
}
/// Commits helper readiness, independently observes the namespace, and validates attestation.
///
/// # Errors
/// Refuses event ordering, namespace mutation, or any attestation mismatch.
pub fn receive_network_readiness<BuildProbeRequest>(
&mut self,
network: &NetworkCoordinator<'_>,
policy: &CanonicalDerivedNetworkPolicy,
build_probe_request: BuildProbeRequest,
expected_reachability: &ReachabilityExpectation,
) -> Result<NetworkReadinessEvidence, SandboxCoordinatorError>
where
BuildProbeRequest: FnOnce(NamespaceIdentity) -> NetworkProbeRequest,
{
let staged = self
.helper_staged
.ok_or(SandboxCoordinatorError::InvalidState(
"helper ownership was not staged",
))?;
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
if control.receive()? != SupervisorEvent::HelperReady(staged) {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor helper readiness mismatched ownership",
));
}
network.commit_helper(staged)?;
let attestation = match control.receive()? {
SupervisorEvent::NetworkAttestation(attestation) => attestation,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not provide network attestation",
));
}
};
let namespace = network.begin_network_intent(
attestation.generation_id,
policy.digest(),
attestation.runtime_supervisor,
)?;
network.validate_attestation(policy, &attestation)?;
let probe_request = build_probe_request(namespace);
if probe_request.domain_id != attestation.domain_id
|| probe_request.generation_id != attestation.generation_id
|| probe_request.enforcement_namespace != attestation.enforcement_namespace
{
return Err(SandboxCoordinatorError::InvalidState(
"network probe request mismatches attestation",
));
}
control.acknowledge_network_attestation(probe_request)?;
let probe = match control.receive()? {
SupervisorEvent::NetworkProbe(probe) => probe,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not return the network probe",
));
}
};
let evidence =
network.commit_readiness(policy, attestation, probe, expected_reachability)?;
control.acknowledge_network_readiness()?;
self.network_evidence = Some(evidence.clone());
Ok(evidence)
}
/// Receives and durably stages the exact RS child without releasing its barrier.
///
/// # Errors
/// Refuses unexpected, duplicate, or mismatched messages and durable-state failures.
pub fn receive_staged(
&mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<StagedProcessOwnership, SandboxCoordinatorError> {
if self.staged.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc stage was already received",
));
}
if self.helper_staged.is_some() && self.network_evidence.is_none() {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc cannot start before network attestation",
));
}
if self.git_routing_required && self.git_routing_mount.is_none() {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc cannot start before Git-routing readiness",
));
}
let event = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?
.receive()?;
let staged = match event {
SupervisorEvent::Staged(staged) => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor sent readiness before staged ownership",
));
}
};
let acknowledgment = coordinator.stage_attached_runsc(staged)?;
self.staged = Some(acknowledgment.committed());
Ok(staged)
}
/// Releases the exact pending runsc barrier with its durable typed acknowledgment.
///
/// # Errors
/// Refuses a different or absent staged tuple or channel failure.
pub fn acknowledge(
mut self,
staged: StagedProcessOwnership,
handoff: &SandboxHandoffAuthority<'_>,
) -> Result<AwaitingSandboxReadiness, SandboxCoordinatorError> {
if self.staged != Some(staged) {
return Err(SandboxCoordinatorError::InvalidState(
"attached runsc acknowledgment does not match pending stage",
));
}
if staged.domain_id() != handoff.domain_id()
|| !handoff.matches_command(&self.runsc_command)
{
return Err(SandboxCoordinatorError::InvalidState(
"sandbox handoff authority does not match attached runsc",
));
}
handoff
.revalidate()
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
self.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?
.acknowledge_staged(ProcessOwnershipCommitAck::v1(staged))?;
Ok(AwaitingSandboxReadiness {
supervisor: self.supervisor.take(),
control: self.control.take(),
lifetime: self.lifetime.take(),
staged,
network_evidence: self.network_evidence.take(),
})
}
/// # Errors
/// Returns an error if the exact RS cannot be terminated and reaped.
pub fn abort(mut self) -> Result<(), SandboxCoordinatorError> {
self.lifetime.take();
cleanup_supervisor(&mut self.supervisor)?;
Ok(())
}
}
impl Drop for AwaitingAttachedRunsc {
fn drop(&mut self) {
self.lifetime.take();
let _ = cleanup_supervisor(&mut self.supervisor);
}
}
pub struct AwaitingSandboxReadiness {
supervisor: Option<RetainedProcess>,
control: Option<CoordinatorChannel>,
lifetime: Option<std::os::fd::OwnedFd>,
staged: StagedProcessOwnership,
network_evidence: Option<NetworkReadinessEvidence>,
}
impl AwaitingSandboxReadiness {
/// Commits only the exact ready attached runsc after RS capability removal.
///
/// # Errors
/// Refuses a mismatched readiness tuple, durable commit failure, or channel failure.
pub fn commit(
self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
if self.network_evidence.is_some() {
return Err(SandboxCoordinatorError::InvalidState(
"networked sandbox requires a successful sandbox-view probe",
));
}
self.commit_after_probe(coordinator, None)
}
/// Commits the Sandbox only after the enforcement-namespace probe is durably accepted.
///
/// # Errors
/// Refuses missing attestation, failed/mutated probe evidence, or any runtime mismatch.
pub fn commit_networked(
mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<(CommittedSandboxRuntime, NetworkReadinessEvidence), SandboxCoordinatorError> {
let evidence =
self.network_evidence
.take()
.ok_or(SandboxCoordinatorError::InvalidState(
"network readiness evidence is absent",
))?;
let runtime = self.commit_after_probe(coordinator, Some(evidence.clone()))?;
Ok((runtime, evidence))
}
fn commit_after_probe(
mut self,
coordinator: &SandboxCoordinator<'_>,
network_evidence: Option<NetworkReadinessEvidence>,
) -> Result<CommittedSandboxRuntime, SandboxCoordinatorError> {
let control = self
.control
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?;
let event = control.receive()?;
let staged = match event {
SupervisorEvent::Ready { staged, .. } => staged,
SupervisorEvent::Refused(detail) => {
return Err(SandboxCoordinatorError::Io(io::Error::other(detail)));
}
_ => {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report sandbox readiness",
));
}
};
if staged != self.staged {
return Err(SandboxCoordinatorError::InvalidState(
"sandbox readiness changed attached-runsc ownership",
));
}
coordinator.commit_ready(staged)?;
control.acknowledge_sandbox_committed()?;
Ok(CommittedSandboxRuntime {
generation_id: staged.generation_id(),
supervisor: self.supervisor.take(),
control: self
.control
.take()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor control channel is absent",
))?,
lifetime: self.lifetime.take(),
network_evidence,
})
}
}
impl Drop for AwaitingSandboxReadiness {
fn drop(&mut self) {
self.lifetime.take();
let _ = cleanup_supervisor(&mut self.supervisor);
}
}

309
crates/lazaret-host-coordinator/src/runtime/committed.rs Normal file
View file

@ -0,0 +1,309 @@
pub struct CommittedSandboxRuntime {
generation_id: GenerationId,
supervisor: Option<RetainedProcess>,
control: CoordinatorChannel,
lifetime: Option<std::os::fd::OwnedFd>,
network_evidence: Option<NetworkReadinessEvidence>,
}
impl CommittedSandboxRuntime {
fn receive_runtime_event(&mut self) -> Result<SupervisorEvent, SandboxCoordinatorError> {
loop {
let event = self.control.receive()?;
if event == SupervisorEvent::HelperExited {
self.network_evidence = None;
} else {
return Ok(event);
}
}
}
#[must_use]
pub const fn network_evidence(&self) -> Option<&NetworkReadinessEvidence> {
self.network_evidence.as_ref()
}
/// Withdraws live readiness after the RS reports exact helper-only loss.
///
/// # Errors
/// Refuses any event other than connectivity-helper loss.
pub fn wait_for_helper_exit(&mut self) -> Result<(), SandboxCoordinatorError> {
if self.control.receive()? != SupervisorEvent::HelperExited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report helper-only loss",
));
}
self.network_evidence = None;
Ok(())
}
#[must_use]
pub fn runtime_supervisor_identity(&self) -> Option<PidfsProcessIdentityV1> {
self.supervisor.as_ref().map(RetainedProcess::identity)
}
/// # Errors
/// Returns an error if the exact retained RS pidfd action fails.
pub fn signal_runtime_supervisor(&self, signal: Signal) -> Result<(), SandboxCoordinatorError> {
self.supervisor
.as_ref()
.ok_or(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor handle is absent",
))?
.send_signal(signal)?;
Ok(())
}
/// # Errors
/// Returns an error if the RS cannot apply the exact retained-runsc pidfd action.
pub fn signal_attached_runsc(
&mut self,
signal: AttachedRunscSignal,
) -> Result<(), SandboxCoordinatorError> {
self.control.signal_attached_runsc(signal)?;
if self.receive_runtime_event()? == SupervisorEvent::SignalApplied(signal) {
Ok(())
} else {
Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not confirm the exact runsc action",
))
}
}
/// Stops and durably cleans only the exact attached sandbox runtime.
///
/// # Errors
/// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
pub fn stop_sandbox(
&mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<(), SandboxCoordinatorError> {
coordinator.begin_release(self.generation_id)?;
self.control.stop_sandbox()?;
if self.receive_runtime_event()? != SupervisorEvent::SandboxStopped {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stop the sandbox",
));
}
coordinator.complete_clean_after_absence(self.generation_id)
}
/// Stops exact generation-local Git routing after sandbox absence.
///
/// # Errors
/// Refuses missing release intent, changed mount evidence, or an unexpected RS response.
pub fn stop_git_routing(
&mut self,
coordinator: &GitRoutingCoordinator<'_>,
) -> Result<(), SandboxCoordinatorError> {
let expected = coordinator
.begin_release(self.generation_id)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
self.control.stop_git_routing()?;
match self.receive_runtime_event()? {
SupervisorEvent::GitRoutingStopped { mount } if mount == expected => coordinator
.complete_clean(self.generation_id, mount)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error))),
SupervisorEvent::Refused(detail) => {
Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
}
_ => Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stop exact Git routing",
)),
}
}
/// Stops only the RS-owned network resources after sandbox absence.
///
/// # Errors
/// Refuses an unexpected RS response or channel failure.
pub fn stop_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
self.control.stop_sidecar()?;
if self.receive_runtime_event()? == SupervisorEvent::SidecarStopped {
self.network_evidence = None;
Ok(())
} else {
Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stop the sidecar",
))
}
}
/// Requests exact RS-owned State Anchor unmount after dependent runtime resources stop.
///
/// # Errors
/// Refuses an unexpected RS response or channel failure.
pub fn unmount_state_anchor(
&mut self,
expected_mount: MountIdentity,
) -> Result<(), SandboxCoordinatorError> {
self.control.unmount_state_anchor()?;
match self.receive_runtime_event()? {
SupervisorEvent::StateAnchorUnmounted { mount } if mount == expected_mount => Ok(()),
_ => Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not confirm the exact State Anchor unmount",
)),
}
}
/// Requests a fresh namespace-local observation of exact State Anchor absence.
///
/// # Errors
/// Refuses an RS refusal, echoed mount, mismatched authority, or channel failure.
pub fn observe_state_anchor_absence(
&mut self,
user_namespace: NamespaceIdentity,
mount_namespace: NamespaceIdentity,
mount: MountIdentity,
) -> Result<StateAnchorAbsenceObservation, SandboxCoordinatorError> {
self.control
.observe_state_anchor_absence(user_namespace, mount_namespace, mount)?;
match self.receive_runtime_event()? {
SupervisorEvent::StateAnchorAbsenceObserved {
user_namespace: observed_user_namespace,
mount_namespace: observed_mount_namespace,
mount: observed_mount,
observed,
} if observed_user_namespace == user_namespace
&& observed_mount_namespace == mount_namespace
&& observed_mount == mount
&& observed != mount =>
{
Ok(StateAnchorAbsenceObservation {
user_namespace: observed_user_namespace,
mount_namespace: observed_mount_namespace,
mount: observed_mount,
observed,
})
}
SupervisorEvent::Refused(detail) => {
Err(SandboxCoordinatorError::Io(io::Error::other(detail)))
}
_ => Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not provide exact State Anchor absence evidence",
)),
}
}
/// Gracefully exits and reaps the resource-empty exact Runtime Supervisor.
///
/// # Errors
/// Refuses an unexpected response or an RS that does not exit.
pub fn graceful_exit(mut self) -> Result<(), SandboxCoordinatorError> {
self.control.exit()?;
if self.receive_runtime_event()? != SupervisorEvent::Exited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not acknowledge graceful exit",
));
}
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)
}
/// Waits for an exact attached-runsc loss report and reaps the retained RS.
///
/// # Errors
/// Refuses any other event or an RS that does not then exit.
pub fn wait_for_attached_runsc_exit(mut self) -> Result<(), SandboxCoordinatorError> {
if self.receive_runtime_event()? != SupervisorEvent::AttachedRunscExited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report attached-runsc loss",
));
}
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)
}
/// Waits for an externally stopped exact RS handle to exit and reaps it.
///
/// # Errors
/// Refuses an RS that remains present beyond the bounded observation.
pub fn wait_for_runtime_supervisor_exit(mut self) -> Result<(), SandboxCoordinatorError> {
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)
}
/// Performs ordered normal stop and reaps the same retained RS handle.
///
/// # Errors
/// Refuses missing release intent, unexpected RS response, or exact cleanup failure.
pub fn stop(
mut self,
coordinator: &SandboxCoordinator<'_>,
) -> Result<(), SandboxCoordinatorError> {
coordinator.begin_release(self.generation_id)?;
self.control.stop()?;
if self.receive_runtime_event()? != SupervisorEvent::Stopped {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not complete sandbox stop",
));
}
self.lifetime.take();
wait_and_reap_supervisor(&mut self.supervisor)?;
coordinator.complete_clean_after_absence(self.generation_id)
}
}
impl Drop for CommittedSandboxRuntime {
fn drop(&mut self) {
self.lifetime.take();
let _ = cleanup_supervisor(&mut self.supervisor);
}
}
fn sandbox_participant(
generation_id: GenerationId,
state: ParticipantState,
ownership: SandboxOwnership,
) -> Result<Participant, SandboxCoordinatorError> {
Participant::snapshot(
generation_id,
ParticipantRole::Sandbox,
state,
Some(ParticipantOwnership::Sandbox(ownership)),
)
.map_err(|_| SandboxCoordinatorError::InvalidState("Sandbox participant shape is invalid"))
}
fn sandbox_role(participants: &[Participant], generation_id: GenerationId) -> Option<&Participant> {
participants.iter().find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::Sandbox
})
}
fn cleanup_supervisor(
supervisor: &mut Option<RetainedProcess>,
) -> Result<(), SandboxCoordinatorError> {
let Some(process) = supervisor.take() else {
return Ok(());
};
if !process.has_exited()? {
process.send_signal(Signal::KILL)?;
}
child_broker().reap(process)?;
Ok(())
}
fn wait_and_reap_supervisor(
supervisor: &mut Option<RetainedProcess>,
) -> Result<(), SandboxCoordinatorError> {
let Some(process) = supervisor.take() else {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor handle is absent",
));
};
let deadline = Instant::now() + SUPERVISOR_EXIT_TIMEOUT;
while !process.has_exited()? && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(5));
}
if !process.has_exited()? {
process.send_signal(Signal::KILL)?;
child_broker().reap(process)?;
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not exit after sandbox stop",
));
}
child_broker().reap(process)?;
Ok(())
}

1386
crates/lazaret-host-coordinator/src/terminal.rs
View file

File diff suppressed because it is too large Load diff

539
crates/lazaret-host-coordinator/src/terminal/engine.rs Normal file
View file

@ -0,0 +1,539 @@
impl SessionTerminalEngine {
#[must_use]
pub fn new(
store: Arc<HostCoordinatorDomainStore>,
factory: Arc<dyn SessionTerminalFactory>,
cancellation: Arc<dyn PromptRouteCancellation>,
) -> Self {
Self {
recorder: HostCoordinatorTerminalRecorder::new(store),
factory,
cancellation,
active: HashMap::new(),
}
}
/// # Errors
/// Refuses duplicate identity or any durable, PTY, causal acquisition, or guest failure.
pub fn acquire(&mut self, request: &TerminalAcquisition) -> Result<(), TerminalError> {
if self.active.contains_key(&request.session_id) {
return Err(TerminalError::DuplicateSession);
}
if self.recorder.require_authority(request)? != TerminalRoute::Sandbox {
return Err(TerminalError::AuthorityMismatch);
}
let intent = terminal_snapshot(
request,
SessionTerminalState::Intent,
PersistedHostProcessOwnership::unassigned(),
None,
PersistedHostProcessOwnership::unassigned(),
None,
)?;
self.recorder.put(intent)?;
let (record, process) = self.factory.acquire(request, &self.recorder)?;
self.active.insert(
request.session_id,
ActiveTerminal {
generation_id: request.generation_id,
request: request.clone(),
route: TerminalRoute::Sandbox,
record,
process,
releasing: false,
pending_input: VecDeque::new(),
},
);
Ok(())
}
/// # Errors
/// Refuses stale, unknown, detached, or releasing Session Terminals.
pub fn set_route(
&mut self,
session_id: SessionId,
generation_id: GenerationId,
route: TerminalRoute,
) -> Result<(), TerminalError> {
let authoritative = self.refresh_route(session_id, generation_id)?;
if authoritative != route {
return Err(TerminalError::InactiveRoute);
}
let terminal = self.active_mut(session_id, generation_id)?;
if terminal.releasing || route == TerminalRoute::Detached {
return Err(TerminalError::Releasing);
}
terminal.route = route;
Ok(())
}
/// # Errors
/// Refuses stale, unknown, inactive, releasing, oversized, or failed terminal events.
pub fn handle(
&mut self,
generation_id: GenerationId,
event: SessionTerminalEvent,
) -> Result<Vec<SessionTerminalMessage>, TerminalError> {
let session_id = match &event {
SessionTerminalEvent::Input(event) => event.session_id,
SessionTerminalEvent::Resize(event) => event.session_id,
SessionTerminalEvent::Signal(event) => event.session_id,
SessionTerminalEvent::Close(event) => event.session_id,
};
if matches!(event, SessionTerminalEvent::Close(_)) {
return self.close(
session_id,
generation_id,
None,
true,
GuestReleaseAuthority::Validate,
);
}
let authoritative = self.refresh_route(session_id, generation_id)?;
if authoritative != TerminalRoute::Sandbox {
return Err(TerminalError::InactiveRoute);
}
let terminal = self.active_mut(session_id, generation_id)?;
if terminal.releasing {
return Err(TerminalError::Releasing);
}
if terminal.route != TerminalRoute::Sandbox {
return Err(TerminalError::InactiveRoute);
}
match event {
SessionTerminalEvent::Input(input) => {
if input.bytes.len() > MAX_EVENT_BYTES {
return Err(TerminalError::OversizedInput);
}
if terminal.pending_input.len() + input.bytes.len() > MAX_INPUT_QUEUE_BYTES {
return Err(TerminalError::Backpressure);
}
terminal.pending_input.extend(input.bytes);
flush_input(terminal)?;
Ok(Vec::new())
}
SessionTerminalEvent::Resize(resize) => {
terminal.process.resize(resize)?;
terminal.process.signal_group("WINCH")?;
Ok(Vec::new())
}
SessionTerminalEvent::Signal(signal) => {
terminal.process.signal_group(match signal.signal {
TerminalSignalKind::Interrupt => "INT",
TerminalSignalKind::Quit => "QUIT",
TerminalSignalKind::Suspend => "TSTP",
})?;
Ok(Vec::new())
}
SessionTerminalEvent::Close(_) => unreachable!(),
}
}
/// # Errors
/// Refuses stale or unknown sessions and propagates terminal read or close failures.
pub fn poll(
&mut self,
session_id: SessionId,
generation_id: GenerationId,
) -> Result<Vec<SessionTerminalMessage>, TerminalError> {
let authoritative = self.refresh_route(session_id, generation_id)?;
let terminal = self.active_mut(session_id, generation_id)?;
if terminal.process.exec_exited()? {
return self.close(
session_id,
generation_id,
None,
true,
GuestReleaseAuthority::Validate,
);
}
if authoritative != TerminalRoute::Sandbox || terminal.releasing {
return Ok(Vec::new());
}
flush_input(terminal)?;
let mut bytes = [0_u8; MAX_OUTPUT_BYTES];
match terminal.process.read_output(&mut bytes) {
Ok(0) => self.close(
session_id,
generation_id,
Some(TerminalLoss::PtyLost),
true,
GuestReleaseAuthority::Validate,
),
Ok(length) => Ok(vec![SessionTerminalMessage::Output(TerminalOutput {
session_id,
bytes: bytes[..length].to_vec(),
})]),
Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(Vec::new()),
Err(_) => self.close(
session_id,
generation_id,
Some(TerminalLoss::PtyLost),
true,
GuestReleaseAuthority::Validate,
),
}
}
/// # Errors
/// Refuses stale or unknown sessions and propagates ordered release failures.
pub fn lose(
&mut self,
session_id: SessionId,
generation_id: GenerationId,
loss: TerminalLoss,
) -> Result<Vec<SessionTerminalMessage>, TerminalError> {
if loss == TerminalLoss::HostCoordinatorLost {
let mut terminal = self
.active
.remove(&session_id)
.ok_or(TerminalError::UnknownSession)?;
if terminal.generation_id != generation_id {
self.active.insert(session_id, terminal);
return Err(TerminalError::StaleGeneration);
}
terminal.process.close_pty();
return Ok(Vec::new());
}
let connected = !matches!(
loss,
TerminalLoss::ClientDisconnected | TerminalLoss::HostCoordinatorLost
);
self.close(
session_id,
generation_id,
Some(loss),
connected,
GuestReleaseAuthority::Validate,
)
}
#[must_use]
pub fn contains(&self, session_id: SessionId) -> bool {
self.active.contains_key(&session_id)
}
fn refresh_route(
&mut self,
session_id: SessionId,
generation_id: GenerationId,
) -> Result<TerminalRoute, TerminalError> {
let active_generation = self
.active
.get(&session_id)
.ok_or(TerminalError::UnknownSession)?
.generation_id;
if active_generation != generation_id {
let cleanup = self.close(
session_id,
active_generation,
Some(TerminalLoss::ExecClientLost),
false,
GuestReleaseAuthority::HostOnly,
);
return Err(with_cleanup(TerminalError::StaleGeneration, cleanup));
}
let request = {
let terminal = self.active_mut(session_id, generation_id)?;
terminal.request.clone()
};
match self.recorder.require_authority(&request) {
Ok(TerminalRoute::Detached) => {
let cleanup = self.close(
session_id,
generation_id,
Some(TerminalLoss::ExecClientLost),
false,
GuestReleaseAuthority::HostOnly,
);
Err(with_cleanup(TerminalError::AuthorityMismatch, cleanup))
}
Err(error) => {
let cleanup = self.close(
session_id,
generation_id,
Some(TerminalLoss::ExecClientLost),
false,
GuestReleaseAuthority::HostOnly,
);
Err(with_cleanup(error, cleanup))
}
Ok(route) => {
self.active_mut(session_id, generation_id)?.route = route;
Ok(route)
}
}
}
fn active_mut(
&mut self,
session_id: SessionId,
generation_id: GenerationId,
) -> Result<&mut ActiveTerminal, TerminalError> {
let terminal = self
.active
.get_mut(&session_id)
.ok_or(TerminalError::UnknownSession)?;
if terminal.generation_id != generation_id {
return Err(TerminalError::StaleGeneration);
}
Ok(terminal)
}
fn close(
&mut self,
session_id: SessionId,
generation_id: GenerationId,
loss: Option<TerminalLoss>,
connected: bool,
authority: GuestReleaseAuthority,
) -> Result<Vec<SessionTerminalMessage>, TerminalError> {
let active_generation = self
.active
.get(&session_id)
.ok_or(TerminalError::UnknownSession)?
.generation_id;
if active_generation != generation_id {
let cleanup = self.close(
session_id,
active_generation,
loss,
false,
GuestReleaseAuthority::HostOnly,
);
return Err(with_cleanup(TerminalError::StaleGeneration, cleanup));
}
let mut terminal = self
.active
.remove(&session_id)
.ok_or(TerminalError::UnknownSession)?;
let was_serviceable = matches!(authority, GuestReleaseAuthority::Validate)
&& (matches!(
self.recorder.require_authority(&terminal.request),
Ok(TerminalRoute::Sandbox | TerminalRoute::Prompt | TerminalRoute::HostProcess)
) || (terminal.record.state() == SessionTerminalState::ReleaseIntent
&& self
.recorder
.require_guest_signal_authority(&terminal.request, &terminal.record)
.is_ok()));
let result = (|| {
terminal.releasing = true;
terminal.route = TerminalRoute::Detached;
let release =
copy_terminal_state(&terminal.record, SessionTerminalState::ReleaseIntent)?;
self.recorder.begin_release(release.clone())?;
terminal.record = release;
self.cancellation.cancel_for_close(session_id)?;
let mut authorize_guest = || {
was_serviceable
&& self
.recorder
.require_guest_signal_authority(&terminal.request, &terminal.record)
.is_ok()
};
let (mut output, exit, guest_absent) = release_physical(
terminal.process.as_mut(),
session_id,
connected,
&mut authorize_guest,
)?;
if guest_absent {
let clean = clean_terminal(generation_id, session_id)?;
self.recorder.clean(clean)?;
}
if connected {
output.push(SessionTerminalMessage::Exit(exit_message(
session_id, loss, exit,
)?));
}
Ok(output)
})();
if result.is_err() {
self.active.insert(session_id, terminal);
}
result
}
}
fn release_physical(
process: &mut dyn SessionTerminalProcess,
session_id: SessionId,
connected: bool,
authorize_guest: &mut dyn FnMut() -> bool,
) -> Result<(Vec<SessionTerminalMessage>, ProcessExit, bool), TerminalError> {
let mut failure = None;
let mut guest_authorized = authorize_guest();
if guest_authorized {
for signal in ["HUP", "TERM", "KILL"] {
if !authorize_guest() {
guest_authorized = false;
break;
}
match process.group_absent() {
Ok(true) => break,
Ok(false) => {}
Err(error) => {
failure.get_or_insert(error);
break;
}
}
if !authorize_guest() {
guest_authorized = false;
break;
}
if let Err(error) = process.signal_group(signal) {
failure.get_or_insert(error);
break;
}
match wait_for_group_absence(process, authorize_guest) {
Ok(GuestWaitOutcome::Absent) => break,
Ok(GuestWaitOutcome::TimedOut) => {}
Ok(GuestWaitOutcome::Revoked) => {
guest_authorized = false;
break;
}
Err(error) => {
failure.get_or_insert(error);
guest_authorized = false;
break;
}
}
}
}
match process.exec_exited() {
Ok(false) => {
if let Err(error) = process.terminate_exec() {
failure.get_or_insert(error.into());
}
}
Ok(true) => {}
Err(error) => {
failure.get_or_insert(error.into());
}
}
let exit = match process.reap_exec() {
Ok(exit) => Some(exit),
Err(error) => {
failure.get_or_insert(error.into());
None
}
};
let output = drain_output(process, session_id, connected);
process.close_pty();
if guest_authorized && authorize_guest() {
match process.group_absent() {
Ok(true) if authorize_guest() => {}
Ok(true) => guest_authorized = false,
Ok(false) => {
failure.get_or_insert(TerminalError::GuestGroupStillPresent);
}
Err(error) => {
failure.get_or_insert(error);
}
}
} else {
guest_authorized = false;
}
if let Some(error) = failure {
return Err(error);
}
Ok((
output,
exit.ok_or_else(|| io::Error::other("exec client exit is unavailable"))?,
guest_authorized,
))
}
fn drain_output(
process: &mut dyn SessionTerminalProcess,
session_id: SessionId,
connected: bool,
) -> Vec<SessionTerminalMessage> {
let mut output = Vec::new();
if !connected {
return output;
}
let deadline = Instant::now() + DRAIN_DEADLINE;
let mut bytes = [0_u8; MAX_OUTPUT_BYTES];
let mut drained = 0;
while Instant::now() < deadline && drained < MAX_DRAIN_BYTES {
match process.read_output(&mut bytes) {
Ok(0) => break,
Ok(length) => {
let length = length.min(MAX_DRAIN_BYTES - drained);
drained += length;
output.push(SessionTerminalMessage::Output(TerminalOutput {
session_id,
bytes: bytes[..length].to_vec(),
}));
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
std::thread::sleep(POLL_INTERVAL);
}
Err(_) => break,
}
}
output
}
fn with_cleanup<T>(original: TerminalError, cleanup: Result<T, TerminalError>) -> TerminalError {
match cleanup {
Ok(_) => original,
Err(cleanup) => TerminalError::AcquisitionCleanup {
original: Box::new(original),
cleanup: Some(Box::new(cleanup)),
},
}
}
fn wait_for_group_absence(
process: &dyn SessionTerminalProcess,
authorize_guest: &mut dyn FnMut() -> bool,
) -> Result<GuestWaitOutcome, TerminalError> {
let deadline = Instant::now() + SIGNAL_GRACE;
while Instant::now() < deadline {
if !authorize_guest() {
return Ok(GuestWaitOutcome::Revoked);
}
if process.group_absent()? {
return Ok(GuestWaitOutcome::Absent);
}
std::thread::sleep(POLL_INTERVAL);
}
Ok(GuestWaitOutcome::TimedOut)
}
fn flush_input(terminal: &mut ActiveTerminal) -> Result<(), TerminalError> {
while !terminal.pending_input.is_empty() {
let (first, second) = terminal.pending_input.as_slices();
let bytes = if first.is_empty() { second } else { first };
match terminal.process.write_input(bytes) {
Ok(0) => {
return Err(io::Error::new(io::ErrorKind::WriteZero, "PTY input closed").into());
}
Ok(length) => {
terminal.pending_input.drain(..length);
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => return Ok(()),
Err(error) => return Err(error.into()),
}
}
Ok(())
}
fn exit_message(
session_id: SessionId,
loss: Option<TerminalLoss>,
exit: ProcessExit,
) -> Result<SessionExit, TerminalError> {
if loss.is_some() {
return Ok(SessionExit::runtime_lost(session_id));
}
match (exit.exit_code, exit.terminating_signal) {
(Some(code), None) => Ok(SessionExit::exited(session_id, code)),
(None, Some(_)) => Ok(SessionExit::terminated(session_id)),
_ => Err(TerminalError::InvalidExit),
}
}

Some files were not shown because too many files have changed in this diff Show more