ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Run non-interactive sessions through production coordination

Browse source
This commit is contained in:
ms2k 2026-07-16 16:28:18 +09:00
commit 71b0266cf2
13 changed files with 3770 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/lazaret-host-coordinator/Cargo.toml
View file

@ -30,6 +30,7 @@ rustls.workspace = true
serde.workspace = true
sha2.workspace = true
time.workspace = true
tokio.workspace = true
[dev-dependencies]
tempfile = "=3.27.0"

20
crates/lazaret-host-coordinator/src/bootstrap.rs
View file

@ -17,7 +17,8 @@ use lazaret_storage_catalog::Catalog;
use crate::control::ControlSessionAdmission;
use crate::control::{
ControlError, PublishedControlFace, SessionRegistry, StagedControlFace, TrustAnchor,
ControlError, NonInteractiveSessionExecutor, PublishedControlFace, SessionRegistry,
StagedControlFace, TrustAnchor,
};
use crate::domain_store::HostCoordinatorDomainStore;
use crate::lifecycle::{
@ -39,6 +40,12 @@ impl fmt::Display for ProductionLifecycleError {
impl std::error::Error for ProductionLifecycleError {}
impl ProductionLifecycleError {
pub(crate) fn service(error: impl fmt::Display) -> Self {
Self(error.to_string())
}
}
fn failure(error: impl fmt::Display) -> ProductionLifecycleError {
ProductionLifecycleError(error.to_string())
}
@ -59,6 +66,7 @@ pub struct ProductionBootstrapPlan<'a> {
pub runtime_root: PathBuf,
pub bundle_digest: BundleDigest,
pub invoking_uid: u32,
pub non_interactive_executor: Arc<dyn NonInteractiveSessionExecutor>,
}
pub struct ProductionLifecycleAdapter<'a> {
@ -332,7 +340,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.take()
.ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?;
self.staged_control = Some(
StagedControlFace::stage(
StagedControlFace::stage_with_executor(
path_from_host(
self.plan
.configuration
@ -343,6 +351,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
self.plan.invoking_uid,
anchor,
Arc::clone(&self.registry),
Arc::clone(&self.plan.non_interactive_executor),
)
.map_err(failure)?,
);
@ -507,10 +516,13 @@ pub fn serve_production_control(
.map_err(|error| ControlError::Lifecycle(error.to_string()))
},
|session_id| {
let mut lifecycle = lifecycle.lock().map_err(|_| ControlError::LockPoisoned)?;
lifecycle
.lock()
.map_err(|_| ControlError::LockPoisoned)?
.disconnect_session(session_id)
.map_err(|error| ControlError::Lifecycle(error.to_string()))?;
lifecycle
.teardown_if_idle()
.map(|_| ())
.map_err(|error| ControlError::Lifecycle(error.to_string()))
},
)

466
crates/lazaret-host-coordinator/src/client_bootstrap.rs Normal file
View file

@ -0,0 +1,466 @@
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs::File;
use std::io::Write;
use std::os::fd::AsFd;
use std::os::unix::fs::MetadataExt;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use lazaret_core::configuration::{
ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
};
use lazaret_core::identity::{
ClientId, ConfigurationFingerprint, DomainId, SessionAnchorId, SessionId,
};
use lazaret_core::records::{Domain, Session, SessionPhase};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use rustix::fs::{Mode, OFlags, openat};
use crate::bootstrap::{ProductionHostCoordinatorLifecycle, serve_production_control};
use crate::domain_store::HostCoordinatorDomainStore;
use crate::production::{BootstrapOwnerGuard, ProductionColdStart, ProductionColdStartError};
use crate::production_selector::{InstalledProductPaths, select_production_cold_start};
use crate::waiter::BootstrapWaiterFace;
pub const PRIVATE_CLIENT_BOOTSTRAP_ROLE: &str = "--lazaret-private-client-bootstrap-v1";
#[derive(Debug)]
pub struct PrivateClientBootstrapError(String);
impl fmt::Display for PrivateClientBootstrapError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
impl std::error::Error for PrivateClientBootstrapError {}
fn failure(error: impl fmt::Display) -> PrivateClientBootstrapError {
PrivateClientBootstrapError(error.to_string())
}
struct BootstrapArguments {
profile: String,
workspace: PathBuf,
configuration: Option<PathBuf>,
cli_id: ClientId,
session_id: SessionId,
session_anchor_id: SessionAnchorId,
expected_domain_id: DomainId,
expected_configuration_fingerprint: ConfigurationFingerprint,
}
/// Dispatches the exact private Client cold-start entry.
#[must_use]
pub fn dispatch_private_client_bootstrap(
arguments: &[OsString],
) -> Option<Result<(), PrivateClientBootstrapError>> {
if arguments
.first()
.is_none_or(|argument| argument != PRIVATE_CLIENT_BOOTSTRAP_ROLE)
{
return None;
}
Some(run_private_client_bootstrap(arguments))
}
fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateClientBootstrapError> {
let parsed = parse_arguments(arguments).inspect_err(|_| signal(b'E'))?;
let owner = BootstrapOwnerGuard::from_stdin()
.map_err(failure)
.inspect_err(|_| signal(b'E'))?;
let configuration = load_configuration(&parsed)
.map_err(failure)
.inspect_err(|_| signal(b'E'))?;
if !matches_client_expectation(
configuration.domain_id(),
configuration.configuration_fingerprint(),
&parsed,
) {
signal(b'E');
return Err(failure(
"independently resolved Domain identity differs from the Client expectation",
));
}
let persistent_domain = configuration
.persistent_domain_root()
.open_authority()
.map_err(failure)
.inspect_err(|_| signal(b'E'))?;
let lock_file = open_private_domain_file(&persistent_domain, "domain.lock")
.inspect_err(|_| signal(b'E'))?;
let domain_lock = match try_exclusive_flock(lock_file) {
Ok(lock) => lock,
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
signal(b'W');
return Ok(());
}
Err(error) => {
signal(b'E');
return Err(failure(error));
}
};
let domain = Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
};
let store = Arc::new(
HostCoordinatorDomainStore::open_file(
domain_lock,
open_private_domain_file(&persistent_domain, "domain.redb")
.inspect_err(|_| signal(b'E'))?,
&domain,
)
.map_err(failure)
.inspect_err(|_| signal(b'E'))?,
);
signal(b'L');
let executable = std::env::current_exe().map_err(failure)?;
let installed = executable
.parent()
.ok_or_else(|| failure("Host Coordinator executable has no installation directory"))?;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(failure)?;
let selection = runtime
.block_on(select_production_cold_start(
&configuration,
Arc::clone(&store),
InstalledProductPaths {
escalation_client: &installed.join("lazaret-escalation-client"),
sandbox_init: &executable,
},
))
.map_err(failure)?;
let inputs = selection
.cold_start_inputs(&configuration)
.map_err(failure)?;
let bootstrap_owner = Session {
generation_id: inputs.generation.generation_id,
session_id: parsed.session_id,
cli_id: parsed.cli_id,
session_anchor_id: parsed.session_anchor_id,
session_phase: SessionPhase::New,
};
ProductionColdStart::stage(inputs)
.map_err(failure)?
.run(bootstrap_owner, &owner, serve_published)
.map_err(failure)
}
fn serve_published(
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
waiters: &RefCell<BootstrapWaiterFace>,
bootstrap_owner: &BootstrapOwnerGuard,
bootstrap_owner_session_id: SessionId,
) -> Result<(), ProductionColdStartError> {
lifecycle
.lock()
.map_err(|_| ProductionColdStartError::service("production lifecycle lock is poisoned"))?
.adapter()
.published_control()
.ok_or_else(|| ProductionColdStartError::service("control face is absent"))?
.set_nonblocking(true)
.map_err(ProductionColdStartError::service)?;
loop {
{
let mut lifecycle = lifecycle.lock().map_err(|_| {
ProductionColdStartError::service("production lifecycle lock is poisoned")
})?;
lifecycle
.reconcile_published_bootstrap_owner(
bootstrap_owner_session_id,
bootstrap_owner
.is_live()
.map_err(ProductionColdStartError::service)?,
)
.map_err(ProductionColdStartError::service)?;
if lifecycle.adapter().published_control().is_none() {
return Ok(());
}
waiters
.borrow_mut()
.reconcile_published(&mut lifecycle)
.map_err(ProductionColdStartError::service)?;
}
match serve_production_control(lifecycle) {
Ok(()) => {}
Err(crate::control::ControlError::Io(error))
if error.kind() == std::io::ErrorKind::WouldBlock =>
{
std::thread::sleep(std::time::Duration::from_millis(10));
}
Err(error) => return Err(ProductionColdStartError::service(error)),
}
if lifecycle
.lock()
.map_err(|_| {
ProductionColdStartError::service("production lifecycle lock is poisoned")
})?
.adapter()
.published_control()
.is_none()
{
return Ok(());
}
}
}
fn matches_client_expectation(
domain_id: DomainId,
configuration_fingerprint: ConfigurationFingerprint,
arguments: &BootstrapArguments,
) -> bool {
domain_id == arguments.expected_domain_id
&& configuration_fingerprint == arguments.expected_configuration_fingerprint
}
fn parse_arguments(
arguments: &[OsString],
) -> Result<BootstrapArguments, PrivateClientBootstrapError> {
let mut arguments = arguments.iter().skip(1);
require_option(&mut arguments, "--profile")?;
let profile = required(&mut arguments, "profile")?
.to_str()
.ok_or_else(|| failure("profile is not UTF-8"))?
.to_owned();
require_option(&mut arguments, "--workspace")?;
let workspace = PathBuf::from(required(&mut arguments, "workspace")?);
let mut configuration = None;
let next = required(&mut arguments, "identity option")?;
let client_option = if next == OsStr::new("--config") {
configuration = Some(PathBuf::from(required(&mut arguments, "configuration")?));
required(&mut arguments, "Client ID option")?
} else {
next
};
if client_option != OsStr::new("--client-id") {
return Err(failure("expected --client-id"));
}
let cli_id = ClientId::new(decode_id(required(&mut arguments, "Client ID")?)?);
require_option(&mut arguments, "--session-id")?;
let session_id = SessionId::new(decode_id(required(&mut arguments, "Session ID")?)?);
require_option(&mut arguments, "--anchor-id")?;
let session_anchor_id =
SessionAnchorId::new(decode_id(required(&mut arguments, "Session Anchor ID")?)?);
require_option(&mut arguments, "--domain-id")?;
let expected_domain_id = DomainId::new(decode_id(required(&mut arguments, "Domain ID")?)?);
require_option(&mut arguments, "--configuration-fingerprint")?;
let expected_configuration_fingerprint = ConfigurationFingerprint::new(decode_id(required(
&mut arguments,
"Configuration Fingerprint",
)?)?);
if arguments.next().is_some() {
return Err(failure("unexpected private bootstrap argument"));
}
Ok(BootstrapArguments {
profile,
workspace,
configuration,
cli_id,
session_id,
session_anchor_id,
expected_domain_id,
expected_configuration_fingerprint,
})
}
fn open_private_domain_file(
root: &impl AsFd,
name: &str,
) -> Result<File, PrivateClientBootstrapError> {
let descriptor = openat(
root,
name,
OFlags::RDWR | OFlags::CREATE | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::RUSR | Mode::WUSR,
)
.map_err(failure)?;
let file = File::from(descriptor);
let metadata = file.metadata().map_err(failure)?;
if !metadata.file_type().is_file()
|| metadata.uid() != rustix::process::getuid().as_raw()
|| metadata.mode() & 0o7777 != 0o600
|| metadata.nlink() != 1
{
return Err(failure(
"private Domain file is not exact invoking-user mode-0600 authority",
));
}
Ok(file)
}
fn required<'a>(
arguments: &mut impl Iterator<Item = &'a OsString>,
label: &str,
) -> Result<&'a OsStr, PrivateClientBootstrapError> {
arguments
.next()
.map(OsString::as_os_str)
.ok_or_else(|| failure(format!("missing {label}")))
}
fn require_option<'a>(
arguments: &mut impl Iterator<Item = &'a OsString>,
expected: &str,
) -> Result<(), PrivateClientBootstrapError> {
if required(arguments, expected)? == OsStr::new(expected) {
Ok(())
} else {
Err(failure(format!("expected {expected}")))
}
}
fn decode_id(value: &OsStr) -> Result<[u8; 32], PrivateClientBootstrapError> {
let value = value
.to_str()
.ok_or_else(|| failure("identity is not UTF-8"))?;
if value.len() != 64 {
return Err(failure("identity is not 32-byte lowercase hexadecimal"));
}
let mut decoded = [0_u8; 32];
for (output, pair) in decoded.iter_mut().zip(value.as_bytes().chunks_exact(2)) {
*output = match (hex_nibble(pair[0]), hex_nibble(pair[1])) {
(Some(high), Some(low)) => high << 4 | low,
_ => return Err(failure("identity is not 32-byte lowercase hexadecimal")),
};
}
Ok(decoded)
}
const fn hex_nibble(byte: u8) -> Option<u8> {
match byte {
b'0'..=b'9' => Some(byte - b'0'),
b'a'..=b'f' => Some(byte - b'a' + 10),
_ => None,
}
}
fn load_configuration(
arguments: &BootstrapArguments,
) -> Result<ResolvedDomainConfiguration, Box<dyn std::error::Error>> {
let home = required_environment_path("HOME")?;
let runtime = required_environment_path("XDG_RUNTIME_DIR")?;
Ok(load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: arguments.configuration.clone(),
selected_profile: arguments.profile.clone(),
workspace: arguments.workspace.clone(),
home,
xdg_config_home: optional_environment_path("XDG_CONFIG_HOME"),
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
xdg_runtime_dir: runtime,
environment: environment(),
sandbox_layout: production_sandbox_layout()?,
})?)
}
fn required_environment_path(name: &str) -> Result<PathBuf, PrivateClientBootstrapError> {
std::env::var_os(name)
.filter(|value| !value.is_empty())
.map(PathBuf::from)
.ok_or_else(|| failure(format!("{name} is not set")))
}
fn optional_environment_path(name: &str) -> Option<PathBuf> {
std::env::var_os(name)
.filter(|value| !value.is_empty())
.map(PathBuf::from)
}
fn environment() -> BTreeMap<String, OsString> {
std::env::vars_os()
.filter_map(|(name, value)| name.into_string().ok().map(|name| (name, value)))
.collect()
}
fn signal(byte: u8) {
let _ = std::io::stdout().write_all(&[byte]);
let _ = std::io::stdout().flush();
}
#[cfg(test)]
mod tests {
use std::path::Path;
use super::*;
#[test]
fn private_bootstrap_arguments_are_exact_and_public() {
let arguments = [
PRIVATE_CLIENT_BOOTSTRAP_ROLE,
"--profile",
"agent",
"--workspace",
"/workspace/with spaces",
"--config",
"/host/config.toml",
"--client-id",
&"11".repeat(32),
"--session-id",
&"22".repeat(32),
"--anchor-id",
&"33".repeat(32),
"--domain-id",
&"44".repeat(32),
"--configuration-fingerprint",
&"55".repeat(32),
]
.map(OsString::from);
let parsed = parse_arguments(&arguments).unwrap();
assert_eq!(parsed.profile, "agent");
assert_eq!(parsed.workspace, Path::new("/workspace/with spaces"));
assert_eq!(
parsed.configuration.as_deref(),
Some(Path::new("/host/config.toml"))
);
assert_eq!(parsed.cli_id, ClientId::new([0x11; 32]));
assert_eq!(parsed.session_id, SessionId::new([0x22; 32]));
assert_eq!(parsed.session_anchor_id, SessionAnchorId::new([0x33; 32]));
assert_eq!(parsed.expected_domain_id, DomainId::new([0x44; 32]));
assert_eq!(
parsed.expected_configuration_fingerprint,
ConfigurationFingerprint::new([0x55; 32])
);
assert!(matches_client_expectation(
DomainId::new([0x44; 32]),
ConfigurationFingerprint::new([0x55; 32]),
&parsed
));
assert!(!matches_client_expectation(
DomainId::new([0x44; 32]),
ConfigurationFingerprint::new([0x56; 32]),
&parsed
));
}
#[test]
fn private_bootstrap_dispatch_and_identity_grammar_are_closed() {
assert!(dispatch_private_client_bootstrap(&[]).is_none());
let extra = [
PRIVATE_CLIENT_BOOTSTRAP_ROLE,
"--profile",
"agent",
"--workspace",
"/workspace",
"--client-id",
&"11".repeat(32),
"--session-id",
&"22".repeat(32),
"--anchor-id",
&"AA".repeat(32),
"--domain-id",
&"44".repeat(32),
"--configuration-fingerprint",
&"55".repeat(32),
]
.map(OsString::from);
assert!(parse_arguments(&extra).is_err());
}
}

473
crates/lazaret-host-coordinator/src/control.rs
View file

@ -1,16 +1,19 @@
use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::os::fd::AsFd;
use std::os::fd::{AsFd, AsRawFd};
use std::os::unix::ffi::OsStringExt;
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard};
use ed25519_dalek::SigningKey;
use lazaret_core::codec::{CodecError, decode, encode};
use lazaret_core::configuration::ConfinedPathPlan;
use lazaret_core::identity::{
CertificateDigest, ClientId, SessionAnchorId, SessionId, TrustManifestDigest,
CertificateDigest, ClientId, ContainerId, SessionAnchorId, SessionId, TrustManifestDigest,
};
use lazaret_core::protocol::{
ProtocolValidationError, issue_protocol_credential, issue_session_capability,
@ -18,10 +21,11 @@ use lazaret_core::protocol::{
};
use lazaret_core::records::{
ClientEnrollmentRequest, ClientEnrollmentResponse, ControlRequest, ControlResponse,
DurableDecision, GenerationIdentity, ProtocolCredential, ProtocolCredentialRole,
DurableDecision, GenerationIdentity, NonInteractiveSessionMessage,
NonInteractiveSessionRequest, ProtocolCredential, ProtocolCredentialRole,
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, SandboxCredentialBundle,
Session, SessionBinding, SessionPhase, SessionRegistrationRequest, SessionRegistrationResponse,
TlsCredential, TlsRole, TlsSubjectId, TrustManifest,
Session, SessionBinding, SessionExit, SessionPhase, SessionRegistrationRequest,
SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId, TrustManifest,
};
use rcgen::{
BasicConstraints, CertificateParams, CertificateSigningRequestParams, CertifiedIssuer,
@ -42,14 +46,16 @@ use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinator
use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, SessionAdmissionIdentity, SessionLifecycleAdapter,
};
use lazaret_linux::process::{ProcessExit, TargetCommand, child_broker};
use lazaret_runtime_supervisor::sandbox::RunscCommand;
pub const CONTROL_ALPN: &[u8] = b"lazaret-control/1";
pub const CONTROL_SERVER_NAME: &str = "coordinator.lazaret.invalid";
pub const MAX_ENROLLMENT_BYTES: usize = 8_192;
const MAX_CONTROL_BYTES: usize = 65_536;
const CONTROL_SOCKET_NAME: &str = "control.sock";
const CONTROL_SOCKET_NAME: &str = "c.sock";
const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard";
const STAGED_CONTROL_SOCKET_NAME: &str = ".control.sock.staged";
const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
#[derive(Debug)]
@ -824,6 +830,167 @@ pub struct PublishedControlFace {
anchor: TrustAnchor,
server_config: Arc<ServerConfig>,
registry: Arc<SessionRegistry>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
}
pub trait NonInteractiveSessionExecutor: Send + Sync {
/// # Errors
/// Returns a runtime, stream, or exact-session execution failure.
fn execute(
&self,
request: &NonInteractiveSessionRequest,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<SessionExit, ControlError>;
}
struct ExecutionUnavailable;
impl NonInteractiveSessionExecutor for ExecutionUnavailable {
fn execute(
&self,
_request: &NonInteractiveSessionRequest,
_emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<SessionExit, ControlError> {
Err(ControlError::InvalidRequest)
}
}
pub struct RunscNonInteractiveExecutor {
program: OsString,
global_arguments: Vec<OsString>,
container: OsString,
runner: Arc<dyn NonInteractiveCommandRunner>,
}
trait NonInteractiveCommandRunner: Send + Sync {
fn run(
&self,
target: &TargetCommand,
session_id: SessionId,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<ProcessExit, ControlError>;
}
struct ChildBrokerCommandRunner;
impl NonInteractiveCommandRunner for ChildBrokerCommandRunner {
fn run(
&self,
target: &TargetCommand,
session_id: SessionId,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<ProcessExit, ControlError> {
let mut child = child_broker().run_transient_captured(target)?;
let mut stdout_done = false;
let mut stderr_done = false;
while !(stdout_done && stderr_done && child.process().has_exited()?) {
let (stdout, stderr) = child.streams();
stdout_done |= emit_available(stdout, session_id, true, emit)?;
stderr_done |= emit_available(stderr, session_id, false, emit)?;
if !(stdout_done && stderr_done) {
std::thread::yield_now();
}
}
child_broker()
.reap_status(child.into_process())
.map_err(Into::into)
}
}
impl RunscNonInteractiveExecutor {
/// Derives the exact `runsc exec` prefix from the generation's attached `runsc run` command.
///
/// # Errors
/// Refuses a non-runsc command or a command without one exact run/container suffix.
pub fn from_run_command(
command: &RunscCommand,
container_id: ContainerId,
) -> Result<Self, ControlError> {
if command.program() != OsStr::new("runsc") {
return Err(ControlError::InvalidRequest);
}
let run = command
.arguments()
.iter()
.position(|argument| argument == OsStr::new("run"))
.ok_or(ControlError::InvalidRequest)?;
if run + 3 != command.arguments().len()
|| !command.arguments()[run + 1]
.as_encoded_bytes()
.starts_with(b"-bundle=")
{
return Err(ControlError::InvalidRequest);
}
let container = OsString::from(hex_lower(container_id.as_bytes()));
if command.arguments()[run + 2] != container {
return Err(ControlError::InvalidRequest);
}
Ok(Self {
program: command.program().to_owned(),
global_arguments: command.arguments()[..run].to_vec(),
container,
runner: Arc::new(ChildBrokerCommandRunner),
})
}
}
impl NonInteractiveSessionExecutor for RunscNonInteractiveExecutor {
fn execute(
&self,
request: &NonInteractiveSessionRequest,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<SessionExit, ControlError> {
let mut target = TargetCommand::new(self.program.clone());
for argument in &self.global_arguments {
target = target.argument(argument.clone());
}
target = target.argument("exec").argument(self.container.clone());
for argument in &request.argv {
target = target.argument(OsString::from_vec(argument.clone()));
}
let process_status = self.runner.run(&target, request.session_id, emit)?;
match (process_status.exit_code, process_status.terminating_signal) {
(Some(code), None) => Ok(SessionExit::exited(request.session_id, code)),
(None, Some(_)) => Ok(SessionExit::terminated(request.session_id)),
_ => Err(ControlError::InvalidRequest),
}
}
}
fn emit_available(
stream: &mut File,
session_id: SessionId,
stdout: bool,
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<bool, ControlError> {
let mut bytes = [0_u8; 16 * 1024];
loop {
match stream.read(&mut bytes) {
Ok(0) => return Ok(true),
Ok(length) => {
let output = lazaret_core::records::NonInteractiveOutput {
session_id,
bytes: bytes[..length].to_vec(),
};
emit(if stdout {
NonInteractiveSessionMessage::Stdout(output)
} else {
NonInteractiveSessionMessage::Stderr(output)
})?;
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => return Ok(false),
Err(error) => return Err(error.into()),
}
}
}
fn hex_lower(bytes: &[u8]) -> String {
use fmt::Write as _;
bytes.iter().fold(String::new(), |mut output, byte| {
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
output
})
}
pub struct StagedControlFace {
@ -838,6 +1005,7 @@ pub struct StagedControlFace {
anchor: Option<TrustAnchor>,
server_config: Arc<ServerConfig>,
registry: Arc<SessionRegistry>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
}
impl StagedControlFace {
@ -850,6 +1018,26 @@ impl StagedControlFace {
uid: u32,
anchor: TrustAnchor,
registry: Arc<SessionRegistry>,
) -> Result<Self, ControlError> {
Self::stage_with_executor(
runtime_domain_root,
uid,
anchor,
registry,
Arc::new(ExecutionUnavailable),
)
}
/// Stages a control face with its generation-bound non-interactive executor.
///
/// # Errors
/// Refuses the same invalid staging conditions as [`Self::stage`].
pub fn stage_with_executor(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
registry: Arc<SessionRegistry>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
) -> Result<Self, ControlError> {
if registry.generation_identity != anchor.generation_identity {
return Err(ControlError::StaleGeneration);
@ -902,6 +1090,7 @@ impl StagedControlFace {
anchor: Some(anchor),
server_config,
registry,
executor,
})
}
@ -959,6 +1148,7 @@ impl StagedControlFace {
))?,
server_config: Arc::clone(&self.server_config),
registry: Arc::clone(&self.registry),
executor: Arc::clone(&self.executor),
})
}
}
@ -971,6 +1161,128 @@ impl Drop for StagedControlFace {
}
impl PublishedControlFace {
/// Selects blocking or polling admission for the publication listener.
///
/// # Errors
/// Returns an operating-system failure changing the listener mode.
pub fn set_nonblocking(&self, nonblocking: bool) -> Result<(), ControlError> {
self.listener.set_nonblocking(nonblocking)?;
Ok(())
}
/// Discovers one publication through the retained runtime Domain authority.
///
/// # Errors
/// Refuses changed root authority or absent, replaced, non-private, oversized, or malformed
/// publication objects.
pub fn discover_confined(
runtime_domain_root: &ConfinedPathPlan,
uid: u32,
) -> Result<(File, PathBuf, ProtocolPublicationIdentity), ControlError> {
let root = File::from(
runtime_domain_root
.open_authority()
.map_err(|_| ControlError::InvalidPath("runtime Domain authority changed"))?,
);
let root_metadata = root.metadata()?;
if !root_metadata.is_dir()
|| root_metadata.uid() != uid
|| root_metadata.mode() & 0o7777 != 0o700
{
return Err(ControlError::InvalidPath(
"runtime Domain root is not invoking-user-owned mode 0700",
));
}
let descriptor = openat(
root.as_fd(),
PUBLICATION_IDENTITY_NAME,
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(io::Error::from)?;
let mut identity = File::from(descriptor);
let identity_metadata = identity.metadata()?;
if !identity_metadata.file_type().is_file()
|| identity_metadata.uid() != uid
|| identity_metadata.mode() & 0o7777 != 0o600
|| identity_metadata.nlink() != 1
|| identity_metadata.len() > MAX_CONTROL_BYTES as u64
{
return Err(ControlError::InvalidPublication(
"publication identity is not an exact private regular file",
));
}
let socket = File::from(
openat(
root.as_fd(),
CONTROL_SOCKET_NAME,
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(io::Error::from)?,
);
let socket_metadata = socket.metadata()?;
if !socket_metadata.file_type().is_socket()
|| socket_metadata.uid() != uid
|| socket_metadata.mode() & 0o7777 != 0o600
{
return Err(ControlError::InvalidPublication(
"control socket is not an invoking-user-private socket",
));
}
let mut bytes = Vec::new();
identity.read_to_end(&mut bytes)?;
let socket_path = PathBuf::from(format!(
"/proc/self/fd/{}/{}",
root.as_raw_fd(),
CONTROL_SOCKET_NAME
));
Ok((root, socket_path, decode(&bytes)?))
}
/// Discovers one exact current control publication beneath a validated runtime Domain root.
///
/// # Errors
/// Refuses absent, replaced, non-private, oversized, or malformed publication objects.
pub fn discover(
runtime_domain_root: &Path,
uid: u32,
) -> Result<(PathBuf, ProtocolPublicationIdentity), ControlError> {
let paths = ControlPaths::open(runtime_domain_root, uid)?;
let descriptor = openat(
paths.root_file.as_fd(),
PUBLICATION_IDENTITY_NAME,
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(io::Error::from)?;
let mut identity = File::from(descriptor);
let descriptor_metadata = identity.metadata()?;
let path_metadata = std::fs::symlink_metadata(&paths.identity)?;
let expected = FileIdentity::from_metadata(&descriptor_metadata);
if FileIdentity::from_metadata(&path_metadata) != expected
|| !descriptor_metadata.file_type().is_file()
|| descriptor_metadata.uid() != uid
|| descriptor_metadata.mode() & 0o7777 != 0o600
|| descriptor_metadata.nlink() != 1
|| descriptor_metadata.len() > MAX_CONTROL_BYTES as u64
{
return Err(ControlError::InvalidPublication(
"publication identity is not an exact private regular file",
));
}
let socket = std::fs::symlink_metadata(&paths.socket)?;
if !socket.file_type().is_socket() || socket.uid() != uid || socket.mode() & 0o7777 != 0o600
{
return Err(ControlError::InvalidPublication(
"control socket is not an invoking-user-private socket",
));
}
let mut bytes = Vec::new();
identity.read_to_end(&mut bytes)?;
Ok((paths.socket, decode(&bytes)?))
}
/// Publishes one generation control face in the fixed host-private runtime path.
///
/// # Errors
@ -981,12 +1293,39 @@ impl PublishedControlFace {
uid: u32,
anchor: TrustAnchor,
store: Arc<HostCoordinatorDomainStore>,
) -> Result<Self, ControlError> {
Self::publish_with_executor(
runtime_domain_root,
uid,
anchor,
store,
Arc::new(ExecutionUnavailable),
)
}
/// Publishes one generation control face with non-interactive session execution.
///
/// # Errors
/// Refuses the same invalid publication conditions as [`Self::publish`].
pub fn publish_with_executor(
runtime_domain_root: &Path,
uid: u32,
anchor: TrustAnchor,
store: Arc<HostCoordinatorDomainStore>,
executor: Arc<dyn NonInteractiveSessionExecutor>,
) -> Result<Self, ControlError> {
let registry = Arc::new(SessionRegistry::new(
store,
anchor.generation_identity.clone(),
));
StagedControlFace::stage(runtime_domain_root, uid, anchor, registry)?.expose()
StagedControlFace::stage_with_executor(
runtime_domain_root,
uid,
anchor,
registry,
executor,
)?
.expose()
}
#[must_use]
@ -1132,13 +1471,44 @@ impl PublishedControlFace {
session_capability: capability,
}),
)?;
let mut extra = [0_u8; 1];
match stream.read(&mut extra) {
Ok(0) => Ok(()),
Ok(_) => Err(ControlError::InvalidRequest),
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => Ok(()),
Err(error) => Err(error.into()),
let request: ControlRequest = match read_frame(stream, MAX_CONTROL_BYTES) {
Ok(request) => request,
Err(ControlError::Io(error))
if matches!(
error.kind(),
io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset
) =>
{
return Ok(());
}
Err(error) => return Err(error),
};
let ControlRequest::Execute(request) = request else {
return Err(ControlError::InvalidRequest);
};
if request.session_id != session_id
|| request.argv.is_empty()
|| request.argv[0].is_empty()
|| request.argv.iter().any(|argument| argument.contains(&0))
{
return Err(ControlError::InvalidRequest);
}
let mut send_message = |message: NonInteractiveSessionMessage| {
if matches!(message, NonInteractiveSessionMessage::Exit(_)) {
return Err(ControlError::InvalidRequest);
}
require_non_interactive_session(&message, request.session_id)?;
write_frame(stream, &message)
};
let outcome = self.executor.execute(&request, &mut send_message)?;
if outcome.session_id() != request.session_id {
return Err(ControlError::InvalidRequest);
}
require_non_interactive_session(
&NonInteractiveSessionMessage::Exit(outcome),
request.session_id,
)?;
write_frame(stream, &NonInteractiveSessionMessage::Exit(outcome))
})();
let disconnected = disconnect(session_id);
result.and(disconnected)
@ -1301,6 +1671,22 @@ fn remove_if_identity_matches(path: &Path, expected: FileIdentity) {
}
}
fn require_non_interactive_session(
message: &NonInteractiveSessionMessage,
expected: SessionId,
) -> Result<(), ControlError> {
let actual = match message {
NonInteractiveSessionMessage::Stdout(output)
| NonInteractiveSessionMessage::Stderr(output) => output.session_id,
NonInteractiveSessionMessage::Exit(exit) => exit.session_id(),
};
if actual == expected {
Ok(())
} else {
Err(ControlError::InvalidRequest)
}
}
fn leaf_parameters(
common_name: &str,
usage: ExtendedKeyUsagePurpose,
@ -1406,6 +1792,7 @@ fn write_frame<Record: serde::Serialize>(
#[cfg(test)]
mod tests {
use std::ffi::OsString;
use std::sync::Arc;
use std::thread;
@ -1483,6 +1870,64 @@ mod tests {
));
}
struct RecordingRunner {
observed: Mutex<Option<(OsString, Vec<OsString>)>>,
exit: ProcessExit,
}
impl NonInteractiveCommandRunner for RecordingRunner {
fn run(
&self,
target: &TargetCommand,
_session_id: SessionId,
_emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
) -> Result<ProcessExit, ControlError> {
*self.observed.lock().unwrap() =
Some((target.program().to_owned(), target.arguments().to_vec()));
Ok(self.exit)
}
}
#[test]
fn runsc_exec_uses_the_command_as_the_first_post_container_argument() {
let runner = Arc::new(RecordingRunner {
observed: Mutex::new(None),
exit: ProcessExit {
exit_code: Some(23),
terminating_signal: None,
},
});
let executor = RunscNonInteractiveExecutor {
program: "runsc".into(),
global_arguments: vec!["--root=/runtime".into()],
container: "container-id".into(),
runner: runner.clone(),
};
let session_id = SessionId::new([9; 32]);
let exit = executor
.execute(
&NonInteractiveSessionRequest {
protocol_version: ProtocolVersion::V1,
session_id,
argv: vec![b"printf".to_vec(), b"hello".to_vec()],
},
&mut |_| Ok(()),
)
.unwrap();
assert_eq!(exit.exit_code(), Some(23));
assert_eq!(
runner.observed.lock().unwrap().as_ref().unwrap(),
&(
OsString::from("runsc"),
["--root=/runtime", "exec", "container-id", "printf", "hello"]
.into_iter()
.map(OsString::from)
.collect()
)
);
}
#[test]
fn rustls_refuses_client_leaf_with_server_role() {
let anchor = TrustAnchor::create(generation()).unwrap();

27
crates/lazaret-host-coordinator/src/domain_store.rs
View file

@ -1,4 +1,5 @@
use std::fmt;
use std::fs::File;
use std::io;
use std::path::Path;
@ -159,6 +160,32 @@ impl HostCoordinatorDomainStore {
Ok(store)
}
/// Opens the Domain Store from a descriptor-relative file while retaining the Domain lock.
///
/// # Errors
/// Refuses I/O, database, metadata, row, or Domain identity failures.
pub fn open_file(
domain_lock: ExclusiveFileLock,
database_file: File,
expected_domain: &Domain,
) -> Result<Self, DomainStoreError> {
let is_new = database_file.metadata()?.len() == 0;
let database = redb::Database::builder().create_file(database_file)?;
let store = Self {
database,
_domain_lock: domain_lock,
};
if is_new {
store.initialize(expected_domain)?;
} else {
store.validate_metadata()?;
if store.snapshot()?.domain != *expected_domain {
return Err(DomainStoreError::DomainMismatch);
}
}
Ok(store)
}
/// Reads and validates every typed row as one consistent redb snapshot.
///
/// # Errors

16
crates/lazaret-host-coordinator/src/domain_store/tests.rs
View file

@ -134,6 +134,22 @@ fn open_store(directory: &TestDirectory) -> HostCoordinatorDomainStore {
HostCoordinatorDomainStore::open(lock, &directory.path("domain.redb"), &domain()).unwrap()
}
#[test]
fn descriptor_store_open_cannot_be_redirected_by_path_replacement() {
let directory = TestDirectory::new("descriptor-open");
let database_path = directory.path("domain.redb");
let database_file = lock_file(&database_path);
let retained_path = directory.path("retained.redb");
std::fs::rename(&database_path, &retained_path).unwrap();
drop(lock_file(&database_path));
let lock = try_exclusive_flock(lock_file(&directory.path("domain.lock"))).unwrap();
drop(HostCoordinatorDomainStore::open_file(lock, database_file, &domain()).unwrap());
assert!(Database::open(retained_path).is_ok());
assert_eq!(std::fs::metadata(database_path).unwrap().len(), 0);
}
#[test]
fn transaction_reopen_and_interrupted_write_preserve_complete_documents() {
let directory = TestDirectory::new("transaction");

2
crates/lazaret-host-coordinator/src/lib.rs
View file

@ -1,6 +1,7 @@
#![doc = "Host-side coordination authority for one Lazaret Domain."]
pub mod bootstrap;
pub mod client_bootstrap;
pub mod control;
pub mod domain_store;
pub mod escalation;
@ -11,3 +12,4 @@ pub mod production_selector;
pub mod runtime;
pub mod sandbox_init;
pub mod state_anchor;
pub mod waiter;

69
crates/lazaret-host-coordinator/src/lifecycle.rs
View file

@ -293,15 +293,37 @@ where
/// # Errors
/// Returns the first invalid transition, durable update, or production adapter failure.
pub fn bootstrap_new_domain_while_owner_live(
&mut self,
generation: Generation,
session: Session,
owner_live: impl FnMut() -> bool,
) -> Result<bool, LifecycleError<A::Error>> {
self.bootstrap_new_domain_while_owner_live_and_service(
generation,
session,
owner_live,
|_| Ok(()),
|_| Ok(()),
)
}
/// Drives cold start while servicing bounded pre-publication waiters between actions.
///
/// # Errors
/// Returns the first invalid transition, durable update, adapter, or waiter-service failure.
pub fn bootstrap_new_domain_while_owner_live_and_service(
&mut self,
generation: Generation,
session: Session,
mut owner_live: impl FnMut() -> bool,
mut service_waiters: impl FnMut(&mut Self) -> Result<(), A::Error>,
mut withdraw_waiter_admission: impl FnMut(&mut Self) -> Result<(), A::Error>,
) -> Result<bool, LifecycleError<A::Error>> {
let session_id = session.session_id;
self.start_new_domain(generation, session)?;
macro_rules! checkpoint {
() => {
service_waiters(self).map_err(LifecycleError::Adapter)?;
if !owner_live() {
self.disconnect_session(session_id)?;
return Ok(false);
@ -327,6 +349,11 @@ where
checkpoint!();
self.start_sandbox()?;
checkpoint!();
withdraw_waiter_admission(self).map_err(LifecycleError::Adapter)?;
if !owner_live() {
self.disconnect_session(session_id)?;
return Ok(false);
}
self.publish_domain()?;
Ok(true)
}
@ -750,6 +777,48 @@ where
})
}
/// Performs the complete ordered normal teardown when the final session has disconnected.
///
/// # Errors
/// Returns the first physical or durable teardown failure.
pub fn teardown_if_idle(&mut self) -> Result<bool, LifecycleError<A::Error>> {
if !self.is_teardown_ready() {
return Ok(false);
}
self.begin_teardown()?;
self.stop_sandbox()?;
self.stop_sidecar()?;
self.unmount_state_anchor()?;
self.release_prepared_lower_lease()?;
self.graceful_supervisor_exit()?;
self.finish_teardown()?;
Ok(true)
}
/// Transfers a published cold-start owner's lifetime edge to exact control admission.
///
/// # Errors
/// Refuses an unpublished Domain, unknown owner, or durable disconnect/teardown failure.
pub fn reconcile_published_bootstrap_owner(
&mut self,
session_id: SessionId,
bootstrap_owner_live: bool,
) -> Result<(), LifecycleError<A::Error>> {
Self::require(
LifecycleAction::DisconnectSession,
self.phase == DomainPhase::Ready && self.published,
"Domain is not published",
)?;
let phase = self
.session(session_id, LifecycleAction::DisconnectSession)?
.session_phase;
if phase == SessionPhase::Waiting && !bootstrap_owner_live {
self.disconnect_session(session_id)?;
self.teardown_if_idle()?;
}
Ok(())
}
/// # Errors
/// Refuses while any session retains Domain interest.
pub fn begin_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {

9
crates/lazaret-host-coordinator/src/main.rs
View file

@ -1,5 +1,14 @@
fn main() {
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
if let Some(result) =
lazaret_host_coordinator::client_bootstrap::dispatch_private_client_bootstrap(&arguments)
{
if let Err(error) = result {
eprintln!("Host Coordinator bootstrap failed: {error}");
std::process::exit(1);
}
return;
}
if lazaret_host_coordinator::sandbox_init::dispatch_private_sandbox_init(&arguments) {
return;
}

106
crates/lazaret-host-coordinator/src/production.rs
View file

@ -1,3 +1,4 @@
use std::cell::RefCell;
use std::ffi::OsString;
use std::fmt;
use std::fs::File;
@ -28,13 +29,15 @@ use rustix::fs::{
AtFlags, FileType, Mode, OFlags, RawDir, ResolveFlags, fchmod, fstat, mkdirat, openat2, statat,
unlinkat,
};
use rustix::io::{FdFlags, fcntl_setfd};
use rustix::io::{FdFlags, dup, fcntl_setfd};
use sha2::{Digest, Sha256};
use crate::bootstrap::{
ProductionBootstrapPlan, ProductionHostCoordinatorLifecycle, production_lifecycle,
};
use crate::control::RunscNonInteractiveExecutor;
use crate::domain_store::HostCoordinatorDomainStore;
use crate::waiter::{BootstrapWaitOutcome, BootstrapWaiterFace};
const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
@ -49,6 +52,12 @@ impl fmt::Display for ProductionColdStartError {
impl std::error::Error for ProductionColdStartError {}
impl ProductionColdStartError {
pub(crate) fn service(error: impl fmt::Display) -> Self {
Self(error.to_string())
}
}
fn failure(error: impl fmt::Display) -> ProductionColdStartError {
ProductionColdStartError(error.to_string())
}
@ -74,6 +83,15 @@ pub struct BootstrapOwnerGuard {
}
impl BootstrapOwnerGuard {
/// Adopts the private bootstrap entry's pipe standard input.
///
/// # Errors
/// Refuses a standard input that cannot be duplicated or is not a pipe.
pub fn from_stdin() -> Result<Self, ProductionColdStartError> {
let stdin = std::io::stdin();
Self::from_inherited(dup(&stdin).map_err(failure)?)
}
/// Adopts an inherited bootstrap-owner lifetime pipe without PID sampling.
///
/// # Errors
@ -89,7 +107,11 @@ impl BootstrapOwnerGuard {
Ok(Self { descriptor })
}
fn is_live(&self) -> std::io::Result<bool> {
/// Observes whether the exact retained bootstrap-owner pipe still has a writer.
///
/// # Errors
/// Returns a polling failure for the retained descriptor.
pub fn is_live(&self) -> std::io::Result<bool> {
let mut descriptors = [PollFd::new(&self.descriptor, PollFlags::IN)];
poll(&mut descriptors, Some(&Timespec::default()))?;
Ok(!descriptors[0]
@ -222,23 +244,76 @@ impl<'a> ProductionColdStart<'a> {
where
Service: FnOnce(
&Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
&RefCell<BootstrapWaiterFace>,
&BootstrapOwnerGuard,
lazaret_core::identity::SessionId,
) -> Result<(), ProductionColdStartError>,
{
let bootstrap_owner_session_id = bootstrap_owner.session_id;
let waiters = RefCell::new(
BootstrapWaiterFace::bind(
self.configuration,
self.generation.generation_id,
self.invoking_uid,
)
.map_err(failure)?,
);
let store = Arc::clone(&self.store);
let generation = self.generation.clone();
let plan = self.take_plan()?;
let mut lifecycle = production_lifecycle(&store, plan).map_err(failure)?;
let published = lifecycle
.bootstrap_new_domain_while_owner_live(generation, bootstrap_owner, || {
owner_guard.is_live().unwrap_or(false)
})
.map_err(failure)?;
if !published {
return Err(ProductionColdStartError(
"bootstrap owner disconnected before publication".into(),
));
let bootstrap = lifecycle.bootstrap_new_domain_while_owner_live_and_service(
generation,
bootstrap_owner,
|| owner_guard.is_live().unwrap_or(false),
|lifecycle| {
waiters
.borrow_mut()
.admit_pending(lifecycle)
.map_err(crate::bootstrap::ProductionLifecycleError::service)
},
|lifecycle| {
let mut waiters = waiters.borrow_mut();
waiters
.admit_pending(lifecycle)
.map_err(crate::bootstrap::ProductionLifecycleError::service)?;
waiters
.withdraw()
.map_err(crate::bootstrap::ProductionLifecycleError::service)
},
);
match bootstrap {
Ok(true) => {
for session_id in waiters.borrow_mut().notify(BootstrapWaitOutcome::Published) {
lifecycle.disconnect_session(session_id).map_err(failure)?;
}
}
Ok(false) => {
let mut waiters = waiters.borrow_mut();
let withdrawal = waiters.withdraw();
let disconnected = waiters.notify(BootstrapWaitOutcome::Failed);
withdrawal.map_err(failure)?;
drop(waiters);
for session_id in disconnected {
lifecycle.disconnect_session(session_id).map_err(failure)?;
}
return Err(ProductionColdStartError(
"bootstrap owner disconnected before publication".into(),
));
}
Err(error) => {
let mut waiters = waiters.borrow_mut();
let _ = waiters.withdraw();
let _ = waiters.notify(BootstrapWaitOutcome::Failed);
return Err(failure(error));
}
}
service(&Mutex::new(lifecycle))
service(
&Mutex::new(lifecycle),
&waiters,
owner_guard,
bootstrap_owner_session_id,
)
}
fn take_plan(&mut self) -> Result<ProductionBootstrapPlan<'_>, ProductionColdStartError> {
@ -260,6 +335,13 @@ impl<'a> ProductionColdStart<'a> {
runtime_root: self.runtime_root.clone(),
bundle_digest: self.bundle_digest,
invoking_uid: self.invoking_uid,
non_interactive_executor: Arc::new(
RunscNonInteractiveExecutor::from_run_command(
self.sandbox_bundle.command(),
self.container_id,
)
.map_err(failure)?,
),
})
}
}

786
crates/lazaret-host-coordinator/src/waiter.rs Normal file
View file

@ -0,0 +1,786 @@
use std::fmt;
use std::fs::File;
use std::io::{self, Read, Write};
use std::os::fd::AsFd;
use std::os::unix::ffi::OsStringExt;
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
use std::os::unix::net::{UnixListener, UnixStream};
use std::path::PathBuf;
use lazaret_core::codec;
use lazaret_core::configuration::ResolvedDomainConfiguration;
use lazaret_core::identity::{
ClientId, ConfigurationFingerprint, DomainId, GenerationId, SessionAnchorId, SessionId,
};
use lazaret_core::records::{ProtocolVersion, Session, SessionPhase};
use rustix::net::sockopt::socket_peercred;
use serde::{Deserialize, Serialize};
use crate::lifecycle::{HostCoordinatorLifecycle, LifecycleAdapter, SessionLifecycleAdapter};
pub const BOOTSTRAP_WAITER_SOCKET_NAME: &str = "wait.sock";
const MAX_FRAME_BYTES: usize = 4_096;
const MAX_WAITERS: usize = 64;
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct BootstrapWaitRequest {
pub protocol_version: ProtocolVersion,
pub domain_id: DomainId,
pub configuration_fingerprint: ConfigurationFingerprint,
pub cli_id: ClientId,
pub session_id: SessionId,
pub session_anchor_id: SessionAnchorId,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum BootstrapWaitOutcome {
Waiting,
Published,
Failed,
Refused,
}
#[derive(Debug)]
pub struct BootstrapWaiterError(String);
impl fmt::Display for BootstrapWaiterError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
impl std::error::Error for BootstrapWaiterError {}
fn failure(error: impl fmt::Display) -> BootstrapWaiterError {
BootstrapWaiterError(error.to_string())
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct FileIdentity {
device: u64,
inode: u64,
}
impl FileIdentity {
fn from_metadata(metadata: &std::fs::Metadata) -> Self {
Self {
device: metadata.dev(),
inode: metadata.ino(),
}
}
}
struct PendingWaiter {
stream: UnixStream,
bytes: Vec<u8>,
}
struct LiveWaiter {
stream: UnixStream,
session_id: SessionId,
}
pub struct BootstrapWaiterFace {
root: File,
root_path: PathBuf,
root_identity: FileIdentity,
socket_path: PathBuf,
socket_identity: FileIdentity,
listener: UnixListener,
pending: Vec<PendingWaiter>,
waiters: Vec<LiveWaiter>,
domain_id: DomainId,
configuration_fingerprint: ConfigurationFingerprint,
generation_id: GenerationId,
uid: u32,
withdrawn: bool,
}
impl BootstrapWaiterFace {
/// Publishes one bounded invoking-user-only waiter admission socket.
///
/// # Errors
/// Refuses changed Runtime Domain authority or a preexisting, replaced, or non-private path.
pub fn bind(
configuration: &ResolvedDomainConfiguration,
generation_id: GenerationId,
uid: u32,
) -> Result<Self, BootstrapWaiterError> {
let root = File::from(
configuration
.runtime_domain_root()
.open_authority()
.map_err(failure)?,
);
let root_path = PathBuf::from(std::ffi::OsString::from_vec(
configuration
.runtime_domain_root()
.canonical_label()
.as_bytes()
.to_vec(),
));
let root_metadata = root.metadata().map_err(failure)?;
let path_metadata = std::fs::symlink_metadata(&root_path).map_err(failure)?;
if FileIdentity::from_metadata(&root_metadata)
!= FileIdentity::from_metadata(&path_metadata)
|| !root_metadata.is_dir()
|| root_metadata.uid() != uid
|| root_metadata.mode() & 0o7777 != 0o700
{
return Err(failure(
"Runtime Domain root is not exact invoking-user mode 0700",
));
}
let socket_path = root_path.join(BOOTSTRAP_WAITER_SOCKET_NAME);
match std::fs::symlink_metadata(&socket_path) {
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Ok(_) => return Err(failure("bootstrap waiter path already exists")),
Err(error) => return Err(failure(error)),
}
let listener = UnixListener::bind(&socket_path).map_err(failure)?;
std::fs::set_permissions(&socket_path, std::fs::Permissions::from_mode(0o600))
.map_err(failure)?;
listener.set_nonblocking(true).map_err(failure)?;
let socket_metadata = std::fs::symlink_metadata(&socket_path).map_err(failure)?;
if !socket_metadata.file_type().is_socket()
|| socket_metadata.uid() != uid
|| socket_metadata.mode() & 0o7777 != 0o600
{
return Err(failure(
"bootstrap waiter socket is not invoking-user mode 0600",
));
}
Ok(Self {
root,
root_path,
root_identity: FileIdentity::from_metadata(&root_metadata),
socket_path,
socket_identity: FileIdentity::from_metadata(&socket_metadata),
listener,
pending: Vec::new(),
waiters: Vec::new(),
domain_id: configuration.domain_id(),
configuration_fingerprint: configuration.configuration_fingerprint(),
generation_id,
uid,
withdrawn: false,
})
}
/// Admits every complete valid request currently available without blocking.
///
/// # Errors
/// Refuses changed socket authority, I/O failures, or durable lifecycle failures.
pub fn admit_pending<A, S>(
&mut self,
lifecycle: &mut HostCoordinatorLifecycle<'_, A, S>,
) -> Result<(), BootstrapWaiterError>
where
A: LifecycleAdapter,
A::Error: fmt::Display,
S: SessionLifecycleAdapter<Error = A::Error>,
{
self.validate_paths()?;
while self.pending.len() + self.waiters.len() < MAX_WAITERS {
match self.listener.accept() {
Ok((stream, _)) => {
let credentials = socket_peercred(stream.as_fd()).map_err(failure)?;
if credentials.uid.as_raw() != self.uid {
continue;
}
stream.set_nonblocking(true).map_err(failure)?;
self.pending.push(PendingWaiter {
stream,
bytes: Vec::new(),
});
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => break,
Err(error) => return Err(failure(error)),
}
}
let mut index = 0;
while index < self.pending.len() {
match read_request(&mut self.pending[index]) {
Ok(Some(request)) => {
let mut pending = self.pending.swap_remove(index);
if request.protocol_version != ProtocolVersion::V1
|| request.domain_id != self.domain_id
|| request.configuration_fingerprint != self.configuration_fingerprint
{
let _ = send_outcome(&mut pending.stream, BootstrapWaitOutcome::Refused);
continue;
}
let session = Session {
generation_id: self.generation_id,
session_id: request.session_id,
cli_id: request.cli_id,
session_anchor_id: request.session_anchor_id,
session_phase: SessionPhase::New,
};
if lifecycle.wait_for_domain(session).is_err() {
let _ = send_outcome(&mut pending.stream, BootstrapWaitOutcome::Refused);
continue;
}
if send_outcome(&mut pending.stream, BootstrapWaitOutcome::Waiting).is_err() {
lifecycle
.disconnect_session(request.session_id)
.map_err(failure)?;
continue;
}
self.waiters.push(LiveWaiter {
stream: pending.stream,
session_id: request.session_id,
});
}
Ok(None) => index += 1,
Err(_) => {
self.pending.swap_remove(index);
}
}
}
Ok(())
}
/// Withdraws the exact waiter admission path.
///
/// # Errors
/// Refuses changed authority or failure to remove the exact socket.
pub fn withdraw(&mut self) -> Result<(), BootstrapWaiterError> {
if self.withdrawn {
return Ok(());
}
self.validate_paths()?;
std::fs::remove_file(&self.socket_path).map_err(failure)?;
self.withdrawn = true;
Ok(())
}
#[must_use]
pub fn notify(&mut self, outcome: BootstrapWaitOutcome) -> Vec<SessionId> {
if matches!(
outcome,
BootstrapWaitOutcome::Waiting | BootstrapWaitOutcome::Refused
) {
return self
.waiters
.iter()
.map(|waiter| waiter.session_id)
.collect();
}
let mut failed = Vec::new();
self.waiters.retain_mut(|waiter| {
let retained = send_outcome(&mut waiter.stream, outcome).is_ok()
&& outcome == BootstrapWaitOutcome::Published
&& waiter.stream.set_nonblocking(true).is_ok();
if !retained && outcome == BootstrapWaitOutcome::Published {
failed.push(waiter.session_id);
}
retained
});
failed
}
/// Reconciles published waiter streams with their durable control-registration phase.
///
/// # Errors
/// Returns a durable lifecycle error when a vanished Waiting session cannot be disconnected.
pub fn reconcile_published<A, S>(
&mut self,
lifecycle: &mut HostCoordinatorLifecycle<'_, A, S>,
) -> Result<(), BootstrapWaiterError>
where
A: LifecycleAdapter,
A::Error: fmt::Display,
S: SessionLifecycleAdapter<Error = A::Error>,
{
let mut index = 0;
while index < self.waiters.len() {
let closed = !matches!(
self.waiters[index].stream.read(&mut [0_u8; 1]),
Err(error) if error.kind() == io::ErrorKind::WouldBlock
);
if !closed {
index += 1;
continue;
}
let waiter = self.waiters.swap_remove(index);
if lifecycle.session_phase(waiter.session_id) == Some(SessionPhase::Waiting) {
lifecycle
.disconnect_session(waiter.session_id)
.map_err(failure)?;
lifecycle.teardown_if_idle().map_err(failure)?;
}
}
Ok(())
}
fn validate_paths(&self) -> Result<(), BootstrapWaiterError> {
let root_metadata = self.root.metadata().map_err(failure)?;
let path_metadata = std::fs::symlink_metadata(&self.root_path).map_err(failure)?;
let socket_metadata = std::fs::symlink_metadata(&self.socket_path).map_err(failure)?;
if FileIdentity::from_metadata(&root_metadata) != self.root_identity
|| FileIdentity::from_metadata(&path_metadata) != self.root_identity
|| !root_metadata.is_dir()
|| root_metadata.uid() != self.uid
|| root_metadata.mode() & 0o7777 != 0o700
|| FileIdentity::from_metadata(&socket_metadata) != self.socket_identity
|| !socket_metadata.file_type().is_socket()
|| socket_metadata.uid() != self.uid
|| socket_metadata.mode() & 0o7777 != 0o600
{
return Err(failure("bootstrap waiter authority changed"));
}
Ok(())
}
}
impl Drop for BootstrapWaiterFace {
fn drop(&mut self) {
if !self.withdrawn
&& std::fs::symlink_metadata(&self.socket_path)
.ok()
.is_some_and(|metadata| {
FileIdentity::from_metadata(&metadata) == self.socket_identity
})
{
let _ = std::fs::remove_file(&self.socket_path);
}
}
}
fn read_request(
pending: &mut PendingWaiter,
) -> Result<Option<BootstrapWaitRequest>, BootstrapWaiterError> {
let mut buffer = [0_u8; 1_024];
loop {
match pending.stream.read(&mut buffer) {
Ok(0) => return Err(failure("bootstrap waiter closed before one request")),
Ok(length) => {
pending.bytes.extend_from_slice(&buffer[..length]);
if pending.bytes.len() > MAX_FRAME_BYTES + 4 {
return Err(failure("bootstrap waiter request is oversized"));
}
}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => break,
Err(error) => return Err(failure(error)),
}
}
if pending.bytes.len() < 4 {
return Ok(None);
}
let length = u32::from_be_bytes(pending.bytes[..4].try_into().expect("four bytes")) as usize;
if length > MAX_FRAME_BYTES {
return Err(failure("bootstrap waiter request is oversized"));
}
if pending.bytes.len() < length + 4 {
return Ok(None);
}
if pending.bytes.len() != length + 4 {
return Err(failure("bootstrap waiter request has trailing bytes"));
}
codec::decode::<BootstrapWaitRequest>(&pending.bytes[4..])
.map(Some)
.map_err(failure)
}
fn send_outcome(
stream: &mut UnixStream,
outcome: BootstrapWaitOutcome,
) -> Result<(), BootstrapWaiterError> {
stream.set_nonblocking(false).map_err(failure)?;
write_frame(stream, &outcome)
}
/// # Errors
/// Refuses oversized or unencodable records and stream write failures.
pub fn write_frame<T: Serialize>(
stream: &mut UnixStream,
record: &T,
) -> Result<(), BootstrapWaiterError> {
let encoded = codec::encode(record).map_err(failure)?;
if encoded.len() > MAX_FRAME_BYTES {
return Err(failure("bootstrap waiter frame is oversized"));
}
let length = u32::try_from(encoded.len()).map_err(failure)?;
stream.write_all(&length.to_be_bytes()).map_err(failure)?;
stream.write_all(&encoded).map_err(failure)?;
stream.flush().map_err(failure)
}
/// # Errors
/// Refuses oversized, malformed, or incomplete frames and stream read failures.
pub fn read_frame<T>(stream: &mut UnixStream) -> Result<T, BootstrapWaiterError>
where
T: serde::de::DeserializeOwned + Serialize,
{
let mut length = [0_u8; 4];
stream.read_exact(&mut length).map_err(failure)?;
let length = u32::from_be_bytes(length) as usize;
if length > MAX_FRAME_BYTES {
return Err(failure("bootstrap waiter frame is oversized"));
}
let mut encoded = vec![0_u8; length];
stream.read_exact(&mut encoded).map_err(failure)?;
codec::decode(&encoded).map_err(failure)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::fs::{self, OpenOptions};
use std::sync::Arc;
use std::thread;
use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration};
use lazaret_core::identity::{CoordinatorId, NetworkSidecarId, SandboxId};
use lazaret_core::records::{Domain, DurableDecision, Generation};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::lifecycle::{LifecycleAction, SessionAdmissionIdentity};
use super::*;
struct Adapter;
impl LifecycleAdapter for Adapter {
type Error = &'static str;
fn execute(
&mut self,
_action: LifecycleAction,
_generation: &Generation,
) -> Result<(), Self::Error> {
Ok(())
}
}
#[derive(Clone)]
struct Admission(Session, ConfigurationFingerprint);
struct Sessions;
impl SessionLifecycleAdapter for Sessions {
type Error = &'static str;
type Admission = Admission;
fn admission_identity(&self, admission: &Admission) -> SessionAdmissionIdentity {
SessionAdmissionIdentity {
generation_id: admission.0.generation_id,
session_id: admission.0.session_id,
configuration_fingerprint: admission.1,
}
}
fn register_session(
&self,
store: &HostCoordinatorDomainStore,
_entry_action: LifecycleAction,
admission: Admission,
) -> Result<Session, Self::Error> {
let registering = Session {
session_phase: SessionPhase::Registering,
..admission.0
};
let attached = Session {
session_phase: SessionPhase::Attached,
..registering.clone()
};
store
.apply(&[
DomainStoreMutation::PutSession(registering),
DomainStoreMutation::PutSession(attached.clone()),
])
.map_err(|_| "register")?;
Ok(attached)
}
fn disconnect_session(
&self,
store: &HostCoordinatorDomainStore,
session: &Session,
) -> Result<(), Self::Error> {
store
.apply(&[DomainStoreMutation::PutSession(Session {
session_phase: SessionPhase::Closed,
..session.clone()
})])
.map_err(|_| "disconnect")
}
}
#[test]
fn waiter_is_durable_before_notification_and_advances_after_publish() {
let tree = tempfile::Builder::new()
.prefix("lz-")
.tempdir_in("/tmp")
.unwrap();
let configuration = configuration(tree.path());
let store = store(&configuration);
let mut lifecycle = HostCoordinatorLifecycle::new(&store, Adapter, Sessions);
lifecycle
.start_new_domain(generation(&configuration), session(1))
.unwrap();
let mut face = BootstrapWaiterFace::bind(
&configuration,
GenerationId::new(1),
rustix::process::getuid().as_raw(),
)
.unwrap();
let socket = face.socket_path.clone();
let request = BootstrapWaitRequest {
protocol_version: ProtocolVersion::V1,
domain_id: configuration.domain_id(),
configuration_fingerprint: configuration.configuration_fingerprint(),
cli_id: ClientId::new([34; 32]),
session_id: SessionId::new([2; 32]),
session_anchor_id: SessionAnchorId::new([66; 32]),
};
let client_request = request.clone();
let client = thread::spawn(move || {
let mut stream = UnixStream::connect(socket).unwrap();
write_frame(&mut stream, &client_request).unwrap();
assert_eq!(
read_frame::<BootstrapWaitOutcome>(&mut stream).unwrap(),
BootstrapWaitOutcome::Waiting
);
assert_eq!(
read_frame::<BootstrapWaitOutcome>(&mut stream).unwrap(),
BootstrapWaitOutcome::Published
);
});
for _ in 0..10_000 {
face.admit_pending(&mut lifecycle).unwrap();
if lifecycle.session_phase(request.session_id).is_some() {
break;
}
thread::yield_now();
}
assert!(lifecycle.session_phase(request.session_id).is_some());
assert_eq!(
store
.snapshot()
.unwrap()
.sessions
.iter()
.find(|session| session.session_id == request.session_id)
.unwrap()
.session_phase,
SessionPhase::Waiting
);
complete_startup(&mut lifecycle);
face.withdraw().unwrap();
assert!(face.notify(BootstrapWaitOutcome::Published).is_empty());
lifecycle
.admit_session(Admission(
Session {
generation_id: GenerationId::new(1),
session_id: request.session_id,
cli_id: request.cli_id,
session_anchor_id: request.session_anchor_id,
session_phase: SessionPhase::Waiting,
},
configuration.configuration_fingerprint(),
))
.unwrap();
assert_eq!(
lifecycle.session_phase(request.session_id),
Some(SessionPhase::Attached)
);
client.join().unwrap();
face.reconcile_published(&mut lifecycle).unwrap();
assert_eq!(
lifecycle.session_phase(request.session_id),
Some(SessionPhase::Attached)
);
}
#[test]
fn vanished_published_waiter_releases_its_durable_interest() {
let tree = tempfile::Builder::new()
.prefix("lz-")
.tempdir_in("/tmp")
.unwrap();
let configuration = configuration(tree.path());
let store = store(&configuration);
let mut lifecycle = HostCoordinatorLifecycle::new(&store, Adapter, Sessions);
lifecycle
.start_new_domain(generation(&configuration), session(1))
.unwrap();
let mut face = BootstrapWaiterFace::bind(
&configuration,
GenerationId::new(1),
rustix::process::getuid().as_raw(),
)
.unwrap();
let request = BootstrapWaitRequest {
protocol_version: ProtocolVersion::V1,
domain_id: configuration.domain_id(),
configuration_fingerprint: configuration.configuration_fingerprint(),
cli_id: ClientId::new([35; 32]),
session_id: SessionId::new([3; 32]),
session_anchor_id: SessionAnchorId::new([67; 32]),
};
let socket = face.socket_path.clone();
let client_request = request.clone();
let client = thread::spawn(move || {
let mut stream = UnixStream::connect(socket).unwrap();
write_frame(&mut stream, &client_request).unwrap();
assert_eq!(
read_frame::<BootstrapWaitOutcome>(&mut stream).unwrap(),
BootstrapWaitOutcome::Waiting
);
assert_eq!(
read_frame::<BootstrapWaitOutcome>(&mut stream).unwrap(),
BootstrapWaitOutcome::Published
);
});
for _ in 0..10_000 {
face.admit_pending(&mut lifecycle).unwrap();
if lifecycle.session_phase(request.session_id) == Some(SessionPhase::Waiting) {
break;
}
thread::yield_now();
}
complete_startup(&mut lifecycle);
face.withdraw().unwrap();
assert!(face.notify(BootstrapWaitOutcome::Published).is_empty());
client.join().unwrap();
for _ in 0..10_000 {
face.reconcile_published(&mut lifecycle).unwrap();
if lifecycle.session_phase(request.session_id) == Some(SessionPhase::Closed) {
break;
}
thread::yield_now();
}
assert_eq!(
lifecycle.session_phase(request.session_id),
Some(SessionPhase::Closed)
);
}
#[test]
fn replacement_cannot_receive_or_be_removed_as_waiter_authority() {
let tree = tempfile::Builder::new()
.prefix("lz-")
.tempdir_in("/tmp")
.unwrap();
let configuration = configuration(tree.path());
let store = store(&configuration);
let mut lifecycle = HostCoordinatorLifecycle::new(&store, Adapter, Sessions);
lifecycle
.start_new_domain(generation(&configuration), session(1))
.unwrap();
let mut face = BootstrapWaiterFace::bind(
&configuration,
GenerationId::new(1),
rustix::process::getuid().as_raw(),
)
.unwrap();
let replacement = face.socket_path.with_extension("old");
fs::rename(&face.socket_path, replacement).unwrap();
let replacement_listener = UnixListener::bind(&face.socket_path).unwrap();
fs::set_permissions(&face.socket_path, fs::Permissions::from_mode(0o600)).unwrap();
assert!(face.admit_pending(&mut lifecycle).is_err());
let path = face.socket_path.clone();
drop(face);
assert!(path.exists());
drop(replacement_listener);
}
fn complete_startup(lifecycle: &mut HostCoordinatorLifecycle<'_, Adapter, Sessions>) {
lifecycle.create_trust_anchor().unwrap();
lifecycle.start_runtime_supervisor().unwrap();
lifecycle.lease_prepared_lower().unwrap();
lifecycle.mount_state_anchor().unwrap();
lifecycle.start_sidecar().unwrap();
lifecycle.start_connectivity_helper().unwrap();
lifecycle.mark_network_ready().unwrap();
lifecycle.arm_runtime_causal_edge().unwrap();
lifecycle.start_sandbox().unwrap();
lifecycle.publish_domain().unwrap();
}
fn store(configuration: &ResolvedDomainConfiguration) -> Arc<HostCoordinatorDomainStore> {
let root = PathBuf::from(std::ffi::OsString::from_vec(
configuration
.persistent_domain_root()
.canonical_label()
.as_bytes()
.to_vec(),
));
let lock = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(root.join("domain.lock"))
.unwrap();
Arc::new(
HostCoordinatorDomainStore::open(
try_exclusive_flock(lock).unwrap(),
&root.join("domain.redb"),
&Domain {
domain_id: configuration.domain_id(),
profile_name: "agent".into(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
},
)
.unwrap(),
)
}
fn configuration(root: &std::path::Path) -> ResolvedDomainConfiguration {
let configuration_directory = root.join("configuration");
let workspace = root.join("workspace");
let runtime = root.join("r");
fs::create_dir(&configuration_directory).unwrap();
fs::create_dir(&workspace).unwrap();
fs::create_dir(&runtime).unwrap();
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.agent]\nimage = \"example.invalid/image:v1\"\n[profiles.agent.readiness]\ndenied = [{{ address = \"10.0.0.1\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
root.join("data").display()
),
)
.unwrap();
load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "agent".into(),
workspace,
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: production_sandbox_layout().unwrap(),
})
.unwrap()
}
fn generation(configuration: &ResolvedDomainConfiguration) -> Generation {
Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(1),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([3; 32]),
network_sidecar_id: NetworkSidecarId::new([4; 32]),
sandbox_id: SandboxId::new([5; 32]),
durable_decision: DurableDecision::None,
}
}
fn session(seed: u8) -> Session {
Session {
generation_id: GenerationId::new(1),
session_id: SessionId::new([seed; 32]),
cli_id: ClientId::new([seed + 32; 32]),
session_anchor_id: SessionAnchorId::new([seed + 64; 32]),
session_phase: SessionPhase::New,
}
}
}

6
crates/lazaret-host-coordinator/tests/control.rs
View file

@ -421,7 +421,7 @@ fn real_uds_tls_enrollment_and_registration_preserve_exact_profiles_and_session_
assert_eq!(socket_metadata.mode() & 0o7777, 0o600);
assert_eq!(identity_metadata.uid(), getuid().as_raw());
assert_eq!(identity_metadata.mode() & 0o7777, 0o600);
assert_eq!(face.socket_path().file_name().unwrap(), "control.sock");
assert_eq!(face.socket_path().file_name().unwrap(), "c.sock");
assert_eq!(
face.identity_path().file_name().unwrap(),
"protocol-identity.postcard"
@ -740,7 +740,7 @@ fn publication_refuses_preexisting_fixed_control_path() {
let directory = TestDirectory::new("existing");
let identity = generation_identity(7);
let store = open_store(&directory, &identity);
File::create(directory.path("control.sock")).unwrap();
File::create(directory.path("c.sock")).unwrap();
assert!(matches!(
PublishedControlFace::publish(
&directory.0,
@ -765,7 +765,7 @@ fn prepared_control_face_admits_only_after_generation_publication() {
registry,
)
.unwrap();
assert!(std::os::unix::net::UnixStream::connect(directory.path("control.sock")).is_err());
assert!(std::os::unix::net::UnixStream::connect(directory.path("c.sock")).is_err());
store
.apply(&[DomainStoreMutation::PutGeneration(generation(

66
crates/lazaret-host-coordinator/tests/lifecycle.rs
View file

@ -12,7 +12,9 @@ use lazaret_host_coordinator::lifecycle::{
DomainPhase, HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, LifecycleError,
SessionAdmissionIdentity, SessionLifecycleAdapter,
};
use lazaret_host_coordinator::production::BootstrapOwnerGuard;
use lazaret_linux::filesystem::try_exclusive_flock;
use rustix::pipe::{PipeFlags, pipe_with};
#[derive(Clone, Default)]
struct RecordingAdapter {
@ -256,6 +258,61 @@ fn bootstrap_owner_guard_failure_uses_exact_modeled_disconnect_transition() {
);
}
#[test]
fn published_bootstrap_owner_eof_disconnects_waiting_session_and_tears_down() {
let directory = tempfile::tempdir().unwrap();
let store = open_store(&directory);
let (domain_adapter, session_adapter) = adapters();
let mut lifecycle = HostCoordinatorLifecycle::new(&store, domain_adapter, session_adapter);
let owner = session(10, 1);
lifecycle
.start_new_domain(generation(1), owner.clone())
.unwrap();
complete_startup(&mut lifecycle);
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).unwrap();
let owner_guard = BootstrapOwnerGuard::from_inherited(reader).unwrap();
drop(writer);
lifecycle
.reconcile_published_bootstrap_owner(owner.session_id, owner_guard.is_live().unwrap())
.unwrap();
assert_eq!(lifecycle.phase(), DomainPhase::Absent);
assert!(!lifecycle.is_published());
assert_eq!(
lifecycle.session_phase(owner.session_id),
Some(SessionPhase::Closed)
);
}
#[test]
fn admitted_bootstrap_owner_ignores_later_bootstrap_pipe_eof() {
let directory = tempfile::tempdir().unwrap();
let store = open_store(&directory);
let (domain_adapter, session_adapter) = adapters();
let mut lifecycle = HostCoordinatorLifecycle::new(&store, domain_adapter, session_adapter);
let owner = session(10, 1);
lifecycle
.start_new_domain(generation(1), owner.clone())
.unwrap();
complete_startup(&mut lifecycle);
lifecycle.admit_session(admission(owner.clone())).unwrap();
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).unwrap();
let owner_guard = BootstrapOwnerGuard::from_inherited(reader).unwrap();
drop(writer);
lifecycle
.reconcile_published_bootstrap_owner(owner.session_id, owner_guard.is_live().unwrap())
.unwrap();
assert_eq!(lifecycle.phase(), DomainPhase::Ready);
assert!(lifecycle.is_published());
assert_eq!(
lifecycle.session_phase(owner.session_id),
Some(SessionPhase::Attached)
);
}
#[test]
fn exact_start_join_disconnect_and_teardown_actions_are_ordered() {
let directory = tempfile::tempdir().unwrap();
@ -274,15 +331,10 @@ fn exact_start_join_disconnect_and_teardown_actions_are_ordered() {
lifecycle.admit_session(admission(waiter.clone())).unwrap();
lifecycle.disconnect_session(owner.session_id).unwrap();
assert!(!lifecycle.is_teardown_ready());
assert!(!lifecycle.teardown_if_idle().unwrap());
lifecycle.disconnect_session(waiter.session_id).unwrap();
assert!(lifecycle.is_teardown_ready());
lifecycle.begin_teardown().unwrap();
lifecycle.stop_sandbox().unwrap();
lifecycle.stop_sidecar().unwrap();
lifecycle.unmount_state_anchor().unwrap();
lifecycle.release_prepared_lower_lease().unwrap();
lifecycle.graceful_supervisor_exit().unwrap();
lifecycle.finish_teardown().unwrap();
assert!(lifecycle.teardown_if_idle().unwrap());
assert_eq!(lifecycle.phase(), DomainPhase::Absent);
assert!(!lifecycle.is_published());