Run non-interactive sessions through production coordination
This commit is contained in:
parent
308c5a1b56
commit
71b0266cf2
24 changed files with 3770 additions and 44 deletions
|
|
@ -10,7 +10,18 @@ name = "lazaret"
|
|||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
ed25519-dalek.workspace = true
|
||||
getrandom.workspace = true
|
||||
lazaret-core = { version = "=0.0.0", path = "../lazaret-core" }
|
||||
lazaret-host-coordinator = { version = "=0.0.0", path = "../lazaret-host-coordinator" }
|
||||
lazaret-linux = { version = "=0.0.0", path = "../lazaret-linux" }
|
||||
lazaret-runtime-supervisor = { version = "=0.0.0", path = "../lazaret-runtime-supervisor" }
|
||||
rustix.workspace = true
|
||||
rustls.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "=3.27.0"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
453
crates/lazaret-client/src/lib.rs
Normal file
453
crates/lazaret-client/src/lib.rs
Normal file
|
|
@ -0,0 +1,453 @@
|
|||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Write};
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use ed25519_dalek::SigningKey;
|
||||
use lazaret_core::identity::{ClientId, SessionAnchorId, SessionId};
|
||||
use lazaret_core::records::{
|
||||
ClientEnrollmentResponse, ControlRequest, ControlResponse, NonInteractiveSessionMessage,
|
||||
NonInteractiveSessionRequest, ProtocolPublicationIdentity, ProtocolVersion, SessionBinding,
|
||||
SessionExitOutcome, SessionRegistrationRequest, TlsSubjectId,
|
||||
};
|
||||
use lazaret_host_coordinator::control::{
|
||||
ControlError, PendingClientEnrollment, PublishedControlFace,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub mod production;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ClientCommand {
|
||||
pub profile: String,
|
||||
pub workspace: PathBuf,
|
||||
pub configuration: Option<PathBuf>,
|
||||
pub argv: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ArgumentError(&'static str);
|
||||
|
||||
impl fmt::Display for ArgumentError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ArgumentError {}
|
||||
|
||||
/// Parses `--profile`, optional host paths, and the exact command following `--`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a concise usage error for missing values, unknown options, or a missing command.
|
||||
pub fn parse_command_line(
|
||||
arguments: impl IntoIterator<Item = OsString>,
|
||||
current_workspace: PathBuf,
|
||||
) -> Result<ClientCommand, ArgumentError> {
|
||||
let mut arguments = arguments.into_iter();
|
||||
let mut profile = None;
|
||||
let mut workspace = None;
|
||||
let mut configuration = None;
|
||||
let mut argv = Vec::new();
|
||||
while let Some(argument) = arguments.next() {
|
||||
match argument.to_str() {
|
||||
Some("--profile") => profile = Some(required_utf8(&mut arguments, "--profile")?),
|
||||
Some("--workspace") => {
|
||||
workspace = Some(PathBuf::from(required_value(
|
||||
&mut arguments,
|
||||
"--workspace",
|
||||
)?));
|
||||
}
|
||||
Some("--config") => {
|
||||
configuration = Some(PathBuf::from(required_value(&mut arguments, "--config")?));
|
||||
}
|
||||
Some("--") => {
|
||||
argv.extend(arguments.map(OsString::into_vec));
|
||||
break;
|
||||
}
|
||||
Some(_) | None => return Err(ArgumentError("unknown option before `--`")),
|
||||
}
|
||||
}
|
||||
let profile = profile.ok_or(ArgumentError("`--profile <name>` is required"))?;
|
||||
if argv.is_empty() {
|
||||
return Err(ArgumentError("a command is required after `--`"));
|
||||
}
|
||||
Ok(ClientCommand {
|
||||
profile,
|
||||
workspace: workspace.unwrap_or(current_workspace),
|
||||
configuration,
|
||||
argv,
|
||||
})
|
||||
}
|
||||
|
||||
fn required_value(
|
||||
arguments: &mut impl Iterator<Item = OsString>,
|
||||
option: &'static str,
|
||||
) -> Result<OsString, ArgumentError> {
|
||||
arguments.next().ok_or(match option {
|
||||
"--profile" => ArgumentError("`--profile` requires a value"),
|
||||
"--workspace" => ArgumentError("`--workspace` requires a path"),
|
||||
"--config" => ArgumentError("`--config` requires a path"),
|
||||
_ => ArgumentError("option requires a value"),
|
||||
})
|
||||
}
|
||||
|
||||
fn required_utf8(
|
||||
arguments: &mut impl Iterator<Item = OsString>,
|
||||
option: &'static str,
|
||||
) -> Result<String, ArgumentError> {
|
||||
required_value(arguments, option)?
|
||||
.into_string()
|
||||
.map_err(|_| ArgumentError("profile name must be UTF-8"))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum DomainStatus<E> {
|
||||
Absent,
|
||||
Starting,
|
||||
Ready(E),
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum BootstrapAcquisition {
|
||||
Leader,
|
||||
Waiter,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ClientError<E> {
|
||||
Configuration(String),
|
||||
UnsupportedHost(String),
|
||||
StaleCoordinator,
|
||||
LostConnection,
|
||||
BootstrapOwnerDisconnected,
|
||||
WaitingDomainFailed,
|
||||
EmptyCommand,
|
||||
Backend(E),
|
||||
Io(io::Error),
|
||||
Control(ControlError),
|
||||
Protocol(&'static str),
|
||||
}
|
||||
|
||||
impl<E: fmt::Display> fmt::Display for ClientError<E> {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Configuration(message) => write!(formatter, "configuration is invalid: {message}"),
|
||||
Self::UnsupportedHost(message) => write!(formatter, "host is unsupported: {message}"),
|
||||
Self::StaleCoordinator => formatter.write_str(
|
||||
"the Domain coordinator is stale; remove no files manually and run Lazaret recovery",
|
||||
),
|
||||
Self::LostConnection => formatter.write_str(
|
||||
"the Domain coordinator connection was lost before the command completed",
|
||||
),
|
||||
Self::BootstrapOwnerDisconnected => formatter.write_str(
|
||||
"the Domain bootstrap owner disconnected before publication",
|
||||
),
|
||||
Self::WaitingDomainFailed => formatter.write_str(
|
||||
"the Domain failed while this Client was waiting; recovery is required before retry",
|
||||
),
|
||||
Self::EmptyCommand => formatter.write_str("a command is required after `--`"),
|
||||
Self::Backend(error) => error.fmt(formatter),
|
||||
Self::Io(error) => write!(formatter, "standard stream forwarding failed: {error}"),
|
||||
Self::Control(error) => error.fmt(formatter),
|
||||
Self::Protocol(message) => write!(formatter, "control protocol failed: {message}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: fmt::Debug + fmt::Display> std::error::Error for ClientError<E> {}
|
||||
|
||||
pub trait ClientBackend {
|
||||
type Endpoint;
|
||||
type Error;
|
||||
|
||||
/// # Errors
|
||||
/// Returns actionable host or configuration validation failures.
|
||||
fn validate(&mut self, command: &ClientCommand) -> Result<(), ClientError<Self::Error>>;
|
||||
/// # Errors
|
||||
/// Returns failures while inspecting current Domain coordination state.
|
||||
fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error>;
|
||||
/// # Errors
|
||||
/// Returns failures while attempting exclusive bootstrap leadership.
|
||||
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error>;
|
||||
/// # Errors
|
||||
/// Returns startup failure or loss of the bootstrap owner.
|
||||
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>>;
|
||||
/// # Errors
|
||||
/// Returns wait failure, stale publication, or bootstrap-owner loss.
|
||||
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>>;
|
||||
/// # Errors
|
||||
/// Returns session registration, stream forwarding, runtime, or connection failure.
|
||||
fn execute(
|
||||
&mut self,
|
||||
endpoint: Self::Endpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<Self::Error>>;
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns validation, coordination, bootstrap, connection, or command execution failures.
|
||||
pub fn run<B: ClientBackend>(
|
||||
backend: &mut B,
|
||||
command: &ClientCommand,
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<B::Error>> {
|
||||
if command.argv.is_empty() {
|
||||
return Err(ClientError::EmptyCommand);
|
||||
}
|
||||
backend.validate(command)?;
|
||||
let endpoint = match backend.observe().map_err(ClientError::Backend)? {
|
||||
DomainStatus::Ready(endpoint) => endpoint,
|
||||
DomainStatus::Starting => backend.wait_until_ready()?,
|
||||
DomainStatus::Stale => return Err(ClientError::StaleCoordinator),
|
||||
DomainStatus::Absent => match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
|
||||
BootstrapAcquisition::Leader => match backend.bootstrap() {
|
||||
Ok(endpoint) => endpoint,
|
||||
Err(ClientError::BootstrapOwnerDisconnected) => {
|
||||
return Err(ClientError::BootstrapOwnerDisconnected);
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
},
|
||||
BootstrapAcquisition::Waiter => backend.wait_until_ready()?,
|
||||
},
|
||||
};
|
||||
backend.execute(endpoint, &command.argv, stdout, stderr)
|
||||
}
|
||||
|
||||
pub struct ControlEndpoint {
|
||||
pub socket_path: PathBuf,
|
||||
pub publication: ProtocolPublicationIdentity,
|
||||
socket_authority: Option<File>,
|
||||
waiter_transfer: Mutex<Option<std::os::unix::net::UnixStream>>,
|
||||
}
|
||||
|
||||
impl ControlEndpoint {
|
||||
#[must_use]
|
||||
pub fn new(socket_path: PathBuf, publication: ProtocolPublicationIdentity) -> Self {
|
||||
Self {
|
||||
socket_path,
|
||||
publication,
|
||||
socket_authority: None,
|
||||
waiter_transfer: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn confined(
|
||||
socket_path: PathBuf,
|
||||
publication: ProtocolPublicationIdentity,
|
||||
authority: File,
|
||||
) -> Self {
|
||||
Self {
|
||||
socket_path,
|
||||
publication,
|
||||
socket_authority: Some(authority),
|
||||
waiter_transfer: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn retain_waiter_transfer(mut self, waiter: std::os::unix::net::UnixStream) -> Self {
|
||||
self.waiter_transfer = Mutex::new(Some(waiter));
|
||||
self
|
||||
}
|
||||
|
||||
fn acknowledge_waiter_transfer<E>(&self) -> Result<(), ClientError<E>> {
|
||||
self.waiter_transfer
|
||||
.lock()
|
||||
.map_err(|_| ClientError::Protocol("waiter transfer lock is poisoned"))?
|
||||
.take();
|
||||
let _ = &self.socket_authority;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ClientSessionIdentity {
|
||||
cli_id: ClientId,
|
||||
session_id: SessionId,
|
||||
anchor: SigningKey,
|
||||
}
|
||||
|
||||
impl ClientSessionIdentity {
|
||||
/// Creates one fresh CLI, Session, and host-private Session Anchor identity.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when operating-system randomness is unavailable.
|
||||
pub fn generate<E>() -> Result<Self, ClientError<E>> {
|
||||
Ok(Self {
|
||||
cli_id: ClientId::new(random_bytes()?),
|
||||
session_id: SessionId::new(random_bytes()?),
|
||||
anchor: SigningKey::from_bytes(&random_bytes()?),
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cli_id(&self) -> ClientId {
|
||||
self.cli_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn session_id(&self) -> SessionId {
|
||||
self.session_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn session_anchor_id(&self) -> SessionAnchorId {
|
||||
SessionAnchorId::new(Sha256::digest(self.anchor.verifying_key().as_bytes()).into())
|
||||
}
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns enrollment, registration, framing, stream forwarding, or runtime-loss failures.
|
||||
pub fn execute_over_control_face(
|
||||
endpoint: &ControlEndpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<std::convert::Infallible>> {
|
||||
let identity = ClientSessionIdentity::generate()?;
|
||||
execute_over_control_face_as(endpoint, argv, stdout, stderr, &identity)
|
||||
}
|
||||
|
||||
/// Executes through the exact identity that owns cold-start leadership.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns enrollment, registration, framing, stream forwarding, or runtime-loss failures.
|
||||
pub fn execute_over_control_face_as<E>(
|
||||
endpoint: &ControlEndpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
identity: &ClientSessionIdentity,
|
||||
) -> Result<i32, ClientError<E>> {
|
||||
if argv.is_empty() {
|
||||
return Err(ClientError::EmptyCommand);
|
||||
}
|
||||
let generation = endpoint.publication.generation_identity.clone();
|
||||
let cli_id = identity.cli_id;
|
||||
let session_id = identity.session_id;
|
||||
let pending = PendingClientEnrollment::create(generation.clone(), cli_id)?;
|
||||
let anonymous = PublishedControlFace::client_config(&endpoint.publication, None)?;
|
||||
let mut enrollment =
|
||||
PublishedControlFace::connect(&endpoint.socket_path, anonymous, &endpoint.publication)?;
|
||||
PublishedControlFace::send(
|
||||
&mut enrollment,
|
||||
&ControlRequest::Enroll(pending.request().clone()),
|
||||
)?;
|
||||
let ControlResponse::Enrolled(ClientEnrollmentResponse { credential, .. }) =
|
||||
PublishedControlFace::receive(&mut enrollment)?
|
||||
else {
|
||||
return Err(ClientError::Protocol(
|
||||
"enrollment response was not Enrolled",
|
||||
));
|
||||
};
|
||||
if credential.generation_identity != generation
|
||||
|| credential.subject_id() != TlsSubjectId::ClientCli(cli_id)
|
||||
{
|
||||
return Err(ClientError::Protocol(
|
||||
"enrollment credential identity differs",
|
||||
));
|
||||
}
|
||||
drop(enrollment);
|
||||
|
||||
let binding = SessionBinding {
|
||||
generation_identity: generation,
|
||||
cli_id,
|
||||
session_id,
|
||||
session_anchor_public_key: identity.anchor.verifying_key().to_bytes().to_vec(),
|
||||
};
|
||||
let authenticated = PublishedControlFace::client_config(
|
||||
&endpoint.publication,
|
||||
Some((&credential, pending.private_key())),
|
||||
)?;
|
||||
let mut stream =
|
||||
PublishedControlFace::connect(&endpoint.socket_path, authenticated, &endpoint.publication)?;
|
||||
PublishedControlFace::send(
|
||||
&mut stream,
|
||||
&ControlRequest::Register(SessionRegistrationRequest {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
session_binding: binding.clone(),
|
||||
}),
|
||||
)?;
|
||||
let ControlResponse::Registered(registered) = PublishedControlFace::receive(&mut stream)?
|
||||
else {
|
||||
return Err(ClientError::Protocol(
|
||||
"registration response was not Registered",
|
||||
));
|
||||
};
|
||||
if registered.session_capability.session_binding != binding {
|
||||
return Err(ClientError::Protocol(
|
||||
"registration capability identity differs",
|
||||
));
|
||||
}
|
||||
endpoint.acknowledge_waiter_transfer()?;
|
||||
PublishedControlFace::send(
|
||||
&mut stream,
|
||||
&ControlRequest::Execute(NonInteractiveSessionRequest {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
session_id,
|
||||
argv: argv.to_vec(),
|
||||
}),
|
||||
)?;
|
||||
loop {
|
||||
let message = PublishedControlFace::receive(&mut stream).map_err(|error| match error {
|
||||
ControlError::Io(_) => ClientError::LostConnection,
|
||||
other => ClientError::Control(other),
|
||||
})?;
|
||||
match message {
|
||||
NonInteractiveSessionMessage::Stdout(output) => {
|
||||
require_session(output.session_id, session_id)?;
|
||||
stdout.write_all(&output.bytes)?;
|
||||
stdout.flush()?;
|
||||
}
|
||||
NonInteractiveSessionMessage::Stderr(output) => {
|
||||
require_session(output.session_id, session_id)?;
|
||||
stderr.write_all(&output.bytes)?;
|
||||
stderr.flush()?;
|
||||
}
|
||||
NonInteractiveSessionMessage::Exit(exit) => {
|
||||
require_session(exit.session_id(), session_id)?;
|
||||
return match exit.outcome() {
|
||||
SessionExitOutcome::Exited => exit
|
||||
.exit_code()
|
||||
.ok_or(ClientError::Protocol("Exited omitted its exit code")),
|
||||
SessionExitOutcome::Terminated | SessionExitOutcome::RuntimeLost => {
|
||||
Err(ClientError::LostConnection)
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn random_bytes<E>() -> Result<[u8; 32], ClientError<E>> {
|
||||
let mut bytes = [0_u8; 32];
|
||||
getrandom::fill(&mut bytes)
|
||||
.map_err(|_| ClientError::Protocol("operating-system randomness is unavailable"))?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn require_session<E>(actual: SessionId, expected: SessionId) -> Result<(), ClientError<E>> {
|
||||
if actual == expected {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ClientError::Protocol("response Session ID differs"))
|
||||
}
|
||||
}
|
||||
|
||||
impl<E> From<io::Error> for ClientError<E> {
|
||||
fn from(error: io::Error) -> Self {
|
||||
Self::Io(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl<E> From<ControlError> for ClientError<E> {
|
||||
fn from(error: ControlError) -> Self {
|
||||
Self::Control(error)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,36 @@
|
|||
fn main() {
|
||||
let _ = lazaret_core::PROTOCOL_NAME;
|
||||
let current_workspace = match std::env::current_dir() {
|
||||
Ok(path) => path,
|
||||
Err(error) => {
|
||||
eprintln!("cannot determine current workspace: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let command =
|
||||
match lazaret_client::parse_command_line(std::env::args_os().skip(1), current_workspace) {
|
||||
Ok(command) => command,
|
||||
Err(error) => {
|
||||
eprintln!("invalid command line: {error}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
let mut backend = match lazaret_client::production::ProductionClientBackend::new() {
|
||||
Ok(backend) => backend,
|
||||
Err(error) => {
|
||||
eprintln!("Lazaret failed: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
match lazaret_client::run(
|
||||
&mut backend,
|
||||
&command,
|
||||
&mut std::io::stdout(),
|
||||
&mut std::io::stderr(),
|
||||
) {
|
||||
Ok(status) => std::process::exit(status),
|
||||
Err(error) => {
|
||||
eprintln!("Lazaret failed: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
650
crates/lazaret-client/src/production.rs
Normal file
650
crates/lazaret-client/src/production.rs
Normal file
|
|
@ -0,0 +1,650 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Write};
|
||||
use std::os::fd::{AsFd, AsRawFd};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::PathBuf;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use lazaret_core::configuration::{
|
||||
ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
|
||||
};
|
||||
use lazaret_host_coordinator::client_bootstrap::PRIVATE_CLIENT_BOOTSTRAP_ROLE;
|
||||
use lazaret_host_coordinator::control::{ControlError, PublishedControlFace};
|
||||
use lazaret_host_coordinator::waiter::{
|
||||
BOOTSTRAP_WAITER_SOCKET_NAME, BootstrapWaitOutcome, BootstrapWaitRequest, read_frame,
|
||||
write_frame,
|
||||
};
|
||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||
use lazaret_linux::platform::{PlatformObservation, check_platform};
|
||||
use lazaret_linux::process::{BootstrapChild, TargetCommand, child_broker};
|
||||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
use rustix::pipe::{PipeFlags, pipe_with};
|
||||
|
||||
use crate::{
|
||||
BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, ClientSessionIdentity,
|
||||
ControlEndpoint, DomainStatus, execute_over_control_face_as,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ProductionClientError(String);
|
||||
|
||||
impl fmt::Display for ProductionClientError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ProductionClientError {}
|
||||
|
||||
fn failure(error: impl fmt::Display) -> ProductionClientError {
|
||||
ProductionClientError(error.to_string())
|
||||
}
|
||||
|
||||
pub struct ProductionClientBackend {
|
||||
configuration: Option<ResolvedDomainConfiguration>,
|
||||
command: Option<ClientCommand>,
|
||||
identity: ClientSessionIdentity,
|
||||
coordinator: Option<BootstrapChild>,
|
||||
bootstrap_owner: Option<rustix::fd::OwnedFd>,
|
||||
}
|
||||
|
||||
impl ProductionClientBackend {
|
||||
/// Creates one production client invocation and its host-private session identity.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when operating-system randomness is unavailable.
|
||||
pub fn new() -> Result<Self, ClientError<ProductionClientError>> {
|
||||
Ok(Self {
|
||||
configuration: None,
|
||||
command: None,
|
||||
identity: ClientSessionIdentity::generate()?,
|
||||
coordinator: None,
|
||||
bootstrap_owner: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn configuration(&self) -> Result<&ResolvedDomainConfiguration, ProductionClientError> {
|
||||
self.configuration
|
||||
.as_ref()
|
||||
.ok_or_else(|| failure("client configuration was not validated"))
|
||||
}
|
||||
|
||||
fn discover(&self) -> Result<Discovery, ProductionClientError> {
|
||||
let configuration = self.configuration()?;
|
||||
match PublishedControlFace::discover_confined(
|
||||
configuration.runtime_domain_root(),
|
||||
rustix::process::getuid().as_raw(),
|
||||
) {
|
||||
Ok((authority, socket_path, publication))
|
||||
if publication_matches(configuration, &publication) =>
|
||||
{
|
||||
Ok(Discovery::Ready(Box::new(ControlEndpoint::confined(
|
||||
socket_path,
|
||||
publication,
|
||||
authority,
|
||||
))))
|
||||
}
|
||||
Err(ControlError::Io(error)) if error.kind() == io::ErrorKind::NotFound => {
|
||||
Ok(Discovery::Absent)
|
||||
}
|
||||
Ok(_) | Err(_) => Ok(Discovery::Stale),
|
||||
}
|
||||
}
|
||||
|
||||
fn domain_lock_is_available(&self) -> Result<bool, ProductionClientError> {
|
||||
let configuration = self.configuration()?;
|
||||
let root = configuration
|
||||
.persistent_domain_root()
|
||||
.open_authority()
|
||||
.map_err(failure)?;
|
||||
let file = open_private_domain_lock(&root)?;
|
||||
match try_exclusive_flock(file) {
|
||||
Ok(lock) => {
|
||||
drop(lock);
|
||||
Ok(true)
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(false),
|
||||
Err(error) => Err(failure(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn launch_coordinator(
|
||||
&mut self,
|
||||
command: &ClientCommand,
|
||||
) -> Result<BootstrapAcquisition, ProductionClientError> {
|
||||
let executable = std::env::current_exe().map_err(failure)?;
|
||||
let coordinator = executable
|
||||
.parent()
|
||||
.ok_or_else(|| failure("Lazaret executable has no installation directory"))?
|
||||
.join("lazaret-host-coordinator");
|
||||
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).map_err(failure)?;
|
||||
let mut target = TargetCommand::new(coordinator)
|
||||
.argument(PRIVATE_CLIENT_BOOTSTRAP_ROLE)
|
||||
.argument("--profile")
|
||||
.argument(&command.profile)
|
||||
.argument("--workspace")
|
||||
.argument(command.workspace.as_os_str())
|
||||
.with_lifetime_pipe_parent_edge();
|
||||
if let Some(configuration) = &command.configuration {
|
||||
target = target
|
||||
.argument("--config")
|
||||
.argument(configuration.as_os_str());
|
||||
}
|
||||
target = target
|
||||
.argument("--client-id")
|
||||
.argument(hex(self.identity.cli_id().as_bytes()))
|
||||
.argument("--session-id")
|
||||
.argument(hex(self.identity.session_id().as_bytes()))
|
||||
.argument("--anchor-id")
|
||||
.argument(hex(self.identity.session_anchor_id().as_bytes()))
|
||||
.argument("--domain-id")
|
||||
.argument(hex(self.configuration()?.domain_id().as_bytes()))
|
||||
.argument("--configuration-fingerprint")
|
||||
.argument(hex(self
|
||||
.configuration()?
|
||||
.configuration_fingerprint()
|
||||
.as_bytes()));
|
||||
let mut child = child_broker()
|
||||
.spawn_bootstrap(&target, File::from(reader))
|
||||
.map_err(failure)?;
|
||||
match child.acquisition_status().map_err(failure)? {
|
||||
b'L' => {
|
||||
self.coordinator = Some(child);
|
||||
self.bootstrap_owner = Some(writer);
|
||||
Ok(BootstrapAcquisition::Leader)
|
||||
}
|
||||
b'W' => {
|
||||
child.reap().map_err(failure)?;
|
||||
Ok(BootstrapAcquisition::Waiter)
|
||||
}
|
||||
b'E' => {
|
||||
let result = child.reap().map_err(failure)?;
|
||||
Err(failure(format!(
|
||||
"Host Coordinator refused bootstrap with {result:?}"
|
||||
)))
|
||||
}
|
||||
_ => Err(failure("Host Coordinator sent an invalid bootstrap status")),
|
||||
}
|
||||
}
|
||||
|
||||
fn await_publication(&mut self) -> Result<ControlEndpoint, ClientError<ProductionClientError>> {
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||
Discovery::Absent => {}
|
||||
}
|
||||
if self
|
||||
.coordinator
|
||||
.as_ref()
|
||||
.ok_or_else(|| ClientError::Backend(failure("Host Coordinator is absent")))?
|
||||
.has_exited()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?
|
||||
{
|
||||
let status = self
|
||||
.coordinator
|
||||
.take()
|
||||
.expect("checked coordinator remains present")
|
||||
.reap()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
return Err(ClientError::Backend(failure(format!(
|
||||
"Host Coordinator exited before publication with {status:?}"
|
||||
))));
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientBackend for ProductionClientBackend {
|
||||
type Endpoint = ControlEndpoint;
|
||||
type Error = ProductionClientError;
|
||||
|
||||
fn validate(&mut self, command: &ClientCommand) -> Result<(), ClientError<Self::Error>> {
|
||||
let kernel_release = std::fs::read_to_string("/proc/sys/kernel/osrelease")
|
||||
.map_err(|error| ClientError::UnsupportedHost(error.to_string()))?;
|
||||
check_platform(&PlatformObservation {
|
||||
operating_system: std::env::consts::OS,
|
||||
architecture: std::env::consts::ARCH,
|
||||
kernel_release: kernel_release.trim(),
|
||||
})
|
||||
.map_err(|error| ClientError::UnsupportedHost(error.to_string()))?;
|
||||
self.configuration = Some(
|
||||
load_configuration(command)
|
||||
.map_err(|error| ClientError::Configuration(error.to_string()))?,
|
||||
);
|
||||
self.command = Some(command.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error> {
|
||||
match self.discover()? {
|
||||
Discovery::Ready(endpoint) => return Ok(DomainStatus::Ready(*endpoint)),
|
||||
Discovery::Stale => return Ok(DomainStatus::Stale),
|
||||
Discovery::Absent => {}
|
||||
}
|
||||
if self.domain_lock_is_available()? {
|
||||
Ok(DomainStatus::Absent)
|
||||
} else {
|
||||
Ok(DomainStatus::Starting)
|
||||
}
|
||||
}
|
||||
|
||||
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
|
||||
let command = self
|
||||
.command
|
||||
.clone()
|
||||
.ok_or_else(|| failure("client command was not validated"))?;
|
||||
self.launch_coordinator(&command)
|
||||
}
|
||||
|
||||
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
self.await_publication()
|
||||
}
|
||||
|
||||
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
let configuration = self.configuration().map_err(ClientError::Backend)?;
|
||||
let runtime_authority = configuration
|
||||
.runtime_domain_root()
|
||||
.open_authority()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
let socket = PathBuf::from(format!(
|
||||
"/proc/self/fd/{}/{}",
|
||||
runtime_authority.as_raw_fd(),
|
||||
BOOTSTRAP_WAITER_SOCKET_NAME
|
||||
));
|
||||
let request = BootstrapWaitRequest {
|
||||
protocol_version: lazaret_core::records::ProtocolVersion::V1,
|
||||
domain_id: configuration.domain_id(),
|
||||
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||
cli_id: self.identity.cli_id(),
|
||||
session_id: self.identity.session_id(),
|
||||
session_anchor_id: self.identity.session_anchor_id(),
|
||||
};
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||
Discovery::Absent => {}
|
||||
}
|
||||
match std::os::unix::net::UnixStream::connect(&socket) {
|
||||
Ok(mut stream) => {
|
||||
write_frame(&mut stream, &request).map_err(|error| {
|
||||
ClientError::Backend(failure(format!(
|
||||
"bootstrap waiter request failed: {error}"
|
||||
)))
|
||||
})?;
|
||||
let first: BootstrapWaitOutcome = read_frame(&mut stream).map_err(|error| {
|
||||
ClientError::Backend(failure(format!(
|
||||
"bootstrap waiter admission failed: {error}"
|
||||
)))
|
||||
})?;
|
||||
if first != BootstrapWaitOutcome::Waiting {
|
||||
return match first {
|
||||
BootstrapWaitOutcome::Refused => Err(ClientError::StaleCoordinator),
|
||||
BootstrapWaitOutcome::Failed => Err(ClientError::WaitingDomainFailed),
|
||||
BootstrapWaitOutcome::Published => {
|
||||
self.await_ready_publication(Some(stream))
|
||||
}
|
||||
BootstrapWaitOutcome::Waiting => unreachable!(),
|
||||
};
|
||||
}
|
||||
return match read_frame(&mut stream) {
|
||||
Ok(BootstrapWaitOutcome::Published) => {
|
||||
self.await_ready_publication(Some(stream))
|
||||
}
|
||||
Ok(BootstrapWaitOutcome::Failed) | Err(_) => {
|
||||
Err(ClientError::WaitingDomainFailed)
|
||||
}
|
||||
Ok(BootstrapWaitOutcome::Waiting | BootstrapWaitOutcome::Refused) => {
|
||||
Err(ClientError::Protocol(
|
||||
"bootstrap waiter sent an invalid terminal outcome",
|
||||
))
|
||||
}
|
||||
};
|
||||
}
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused
|
||||
) => {}
|
||||
Err(error) => return Err(ClientError::Backend(failure(error))),
|
||||
}
|
||||
if self
|
||||
.domain_lock_is_available()
|
||||
.map_err(ClientError::Backend)?
|
||||
{
|
||||
return Err(ClientError::WaitingDomainFailed);
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&mut self,
|
||||
endpoint: Self::Endpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<Self::Error>> {
|
||||
let exit = execute_over_control_face_as(&endpoint, argv, stdout, stderr, &self.identity)?;
|
||||
if let Some(coordinator) = self.coordinator.take() {
|
||||
let status = coordinator
|
||||
.reap()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
if status.exit_code != Some(0) || status.terminating_signal.is_some() {
|
||||
return Err(ClientError::Backend(failure(format!(
|
||||
"Host Coordinator teardown failed with {status:?}"
|
||||
))));
|
||||
}
|
||||
}
|
||||
self.bootstrap_owner.take();
|
||||
Ok(exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl ProductionClientBackend {
|
||||
fn await_ready_publication(
|
||||
&self,
|
||||
waiter: Option<std::os::unix::net::UnixStream>,
|
||||
) -> Result<ControlEndpoint, ClientError<ProductionClientError>> {
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => {
|
||||
let endpoint = *endpoint;
|
||||
return Ok(match waiter {
|
||||
Some(stream) => endpoint.retain_waiter_transfer(stream),
|
||||
None => endpoint,
|
||||
});
|
||||
}
|
||||
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||
Discovery::Absent => thread::sleep(Duration::from_millis(1)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn open_private_domain_lock(root: &impl AsFd) -> Result<File, ProductionClientError> {
|
||||
let file = File::from(
|
||||
openat(
|
||||
root,
|
||||
"domain.lock",
|
||||
OFlags::RDWR | OFlags::CREATE | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)
|
||||
.map_err(failure)?,
|
||||
);
|
||||
let metadata = file.metadata().map_err(failure)?;
|
||||
if !metadata.file_type().is_file()
|
||||
|| metadata.uid() != rustix::process::getuid().as_raw()
|
||||
|| metadata.mode() & 0o7777 != 0o600
|
||||
|| metadata.nlink() != 1
|
||||
{
|
||||
return Err(failure(
|
||||
"Domain lock is not exact invoking-user mode-0600 authority",
|
||||
));
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn publication_matches(
|
||||
configuration: &ResolvedDomainConfiguration,
|
||||
publication: &lazaret_core::records::ProtocolPublicationIdentity,
|
||||
) -> bool {
|
||||
publication.generation_identity.domain_id == configuration.domain_id()
|
||||
&& publication.generation_identity.configuration_fingerprint
|
||||
== configuration.configuration_fingerprint()
|
||||
}
|
||||
|
||||
fn load_configuration(
|
||||
command: &ClientCommand,
|
||||
) -> Result<ResolvedDomainConfiguration, Box<dyn std::error::Error>> {
|
||||
Ok(load_trusted_configuration(ConfigurationLoadRequest {
|
||||
explicit_configuration_path: command.configuration.clone(),
|
||||
selected_profile: command.profile.clone(),
|
||||
workspace: command.workspace.clone(),
|
||||
home: required_environment_path("HOME")?,
|
||||
xdg_config_home: optional_environment_path("XDG_CONFIG_HOME"),
|
||||
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
|
||||
xdg_runtime_dir: required_environment_path("XDG_RUNTIME_DIR")?,
|
||||
environment: environment(),
|
||||
sandbox_layout: production_sandbox_layout()?,
|
||||
})?)
|
||||
}
|
||||
|
||||
fn required_environment_path(name: &str) -> Result<PathBuf, ProductionClientError> {
|
||||
std::env::var_os(name)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.ok_or_else(|| failure(format!("{name} is not set")))
|
||||
}
|
||||
|
||||
fn optional_environment_path(name: &str) -> Option<PathBuf> {
|
||||
std::env::var_os(name)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(PathBuf::from)
|
||||
}
|
||||
|
||||
fn environment() -> BTreeMap<String, OsString> {
|
||||
std::env::vars_os()
|
||||
.filter_map(|(name, value)| name.into_string().ok().map(|name| (name, value)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
use fmt::Write as _;
|
||||
|
||||
bytes.iter().fold(String::new(), |mut output, byte| {
|
||||
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
|
||||
output
|
||||
})
|
||||
}
|
||||
|
||||
enum Discovery {
|
||||
Absent,
|
||||
Ready(Box<ControlEndpoint>),
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
|
||||
use super::*;
|
||||
use lazaret_core::codec::encode;
|
||||
use lazaret_core::identity::{
|
||||
CertificateDigest, CoordinatorId, GenerationId, SandboxId, TrustManifestDigest,
|
||||
};
|
||||
use lazaret_core::records::{GenerationIdentity, ProtocolPublicationIdentity, ProtocolVersion};
|
||||
|
||||
#[test]
|
||||
fn replaced_domain_roots_cannot_redirect_client_lock_or_control_authority() {
|
||||
let persistent_tree = tempfile::Builder::new()
|
||||
.prefix("lz-")
|
||||
.tempdir_in("/tmp")
|
||||
.unwrap();
|
||||
let persistent_configuration = configuration(persistent_tree.path());
|
||||
let persistent_path = path(
|
||||
persistent_configuration
|
||||
.persistent_domain_root()
|
||||
.canonical_label()
|
||||
.as_bytes(),
|
||||
);
|
||||
fs::rename(&persistent_path, persistent_path.with_extension("old")).unwrap();
|
||||
fs::create_dir(&persistent_path).unwrap();
|
||||
fs::set_permissions(&persistent_path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert!(
|
||||
persistent_configuration
|
||||
.persistent_domain_root()
|
||||
.open_authority()
|
||||
.is_err()
|
||||
);
|
||||
assert!(!persistent_path.join("domain.lock").exists());
|
||||
|
||||
let runtime_tree = tempfile::Builder::new()
|
||||
.prefix("lz-")
|
||||
.tempdir_in("/tmp")
|
||||
.unwrap();
|
||||
let runtime_configuration = configuration(runtime_tree.path());
|
||||
let runtime_path = path(
|
||||
runtime_configuration
|
||||
.runtime_domain_root()
|
||||
.canonical_label()
|
||||
.as_bytes(),
|
||||
);
|
||||
fs::rename(&runtime_path, runtime_path.with_extension("old")).unwrap();
|
||||
fs::create_dir(&runtime_path).unwrap();
|
||||
fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert!(
|
||||
PublishedControlFace::discover_confined(
|
||||
runtime_configuration.runtime_domain_root(),
|
||||
rustix::process::getuid().as_raw(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confined_control_discovery_connects_through_the_retained_root() {
|
||||
let tree = tempfile::Builder::new()
|
||||
.prefix("lz-")
|
||||
.tempdir_in("/tmp")
|
||||
.unwrap();
|
||||
let configuration = configuration(tree.path());
|
||||
let runtime_path = path(
|
||||
configuration
|
||||
.runtime_domain_root()
|
||||
.canonical_label()
|
||||
.as_bytes(),
|
||||
);
|
||||
let listener = UnixListener::bind(runtime_path.join("c.sock")).unwrap();
|
||||
fs::set_permissions(
|
||||
runtime_path.join("c.sock"),
|
||||
fs::Permissions::from_mode(0o600),
|
||||
)
|
||||
.unwrap();
|
||||
let publication = ProtocolPublicationIdentity {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
generation_identity: GenerationIdentity {
|
||||
agent_profile: "agent".into(),
|
||||
domain_id: configuration.domain_id(),
|
||||
generation_id: GenerationId::new(1),
|
||||
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||
coordinator_id: CoordinatorId::new([3; 32]),
|
||||
sandbox_id: SandboxId::new([4; 32]),
|
||||
},
|
||||
tls_ca_certificate_der: vec![1],
|
||||
trust_manifest_sha256: TrustManifestDigest::new([5; 32]),
|
||||
coordinator_tls_certificate_sha256: CertificateDigest::new([6; 32]),
|
||||
};
|
||||
let mut mismatch = publication.clone();
|
||||
mismatch.generation_identity.configuration_fingerprint =
|
||||
lazaret_core::identity::ConfigurationFingerprint::new([7; 32]);
|
||||
assert!(publication_matches(&configuration, &publication));
|
||||
assert!(!publication_matches(&configuration, &mismatch));
|
||||
fs::write(
|
||||
runtime_path.join("protocol-identity.postcard"),
|
||||
encode(&publication).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
fs::set_permissions(
|
||||
runtime_path.join("protocol-identity.postcard"),
|
||||
fs::Permissions::from_mode(0o600),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (authority, socket, observed) = PublishedControlFace::discover_confined(
|
||||
configuration.runtime_domain_root(),
|
||||
rustix::process::getuid().as_raw(),
|
||||
)
|
||||
.unwrap();
|
||||
let client = UnixStream::connect(socket).unwrap();
|
||||
let (server, _) = listener.accept().unwrap();
|
||||
|
||||
assert_eq!(observed, publication);
|
||||
drop((authority, client, server));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiter_transfer_remains_live_until_control_registration_acknowledges_it() {
|
||||
let (client, mut coordinator) = UnixStream::pair().unwrap();
|
||||
coordinator.set_nonblocking(true).unwrap();
|
||||
let endpoint = ControlEndpoint::new(
|
||||
PathBuf::from("unused"),
|
||||
ProtocolPublicationIdentity {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
generation_identity: GenerationIdentity {
|
||||
agent_profile: "agent".into(),
|
||||
domain_id: lazaret_core::identity::DomainId::new([1; 32]),
|
||||
generation_id: GenerationId::new(1),
|
||||
configuration_fingerprint:
|
||||
lazaret_core::identity::ConfigurationFingerprint::new([2; 32]),
|
||||
coordinator_id: CoordinatorId::new([3; 32]),
|
||||
sandbox_id: SandboxId::new([4; 32]),
|
||||
},
|
||||
tls_ca_certificate_der: vec![1],
|
||||
trust_manifest_sha256: TrustManifestDigest::new([5; 32]),
|
||||
coordinator_tls_certificate_sha256: CertificateDigest::new([6; 32]),
|
||||
},
|
||||
)
|
||||
.retain_waiter_transfer(client);
|
||||
assert_eq!(
|
||||
coordinator.read(&mut [0_u8; 1]).unwrap_err().kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
|
||||
endpoint
|
||||
.acknowledge_waiter_transfer::<std::convert::Infallible>()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(coordinator.read(&mut [0_u8; 1]).unwrap(), 0);
|
||||
}
|
||||
|
||||
fn configuration(root: &std::path::Path) -> ResolvedDomainConfiguration {
|
||||
let configuration_directory = root.join("configuration");
|
||||
let workspace = root.join("workspace");
|
||||
let runtime = root.join("r");
|
||||
fs::create_dir(&configuration_directory).unwrap();
|
||||
fs::create_dir(&workspace).unwrap();
|
||||
fs::create_dir(&runtime).unwrap();
|
||||
let configuration_path = configuration_directory.join("lazaret.toml");
|
||||
fs::write(
|
||||
&configuration_path,
|
||||
format!(
|
||||
"schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.agent]\nimage = \"example.invalid/image:v1\"\n[profiles.agent.readiness]\ndenied = [{{ address = \"10.0.0.1\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
|
||||
root.join("data").display()
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
load_trusted_configuration(ConfigurationLoadRequest {
|
||||
explicit_configuration_path: Some(configuration_path),
|
||||
selected_profile: "agent".into(),
|
||||
workspace,
|
||||
home: root.join("home"),
|
||||
xdg_config_home: None,
|
||||
xdg_data_home: None,
|
||||
xdg_runtime_dir: runtime,
|
||||
environment: BTreeMap::new(),
|
||||
sandbox_layout: production_sandbox_layout().unwrap(),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn path(bytes: &[u8]) -> PathBuf {
|
||||
PathBuf::from(OsString::from_vec(bytes.to_vec()))
|
||||
}
|
||||
}
|
||||
255
crates/lazaret-client/tests/cli_contract.rs
Normal file
255
crates/lazaret-client/tests/cli_contract.rs
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
use std::collections::VecDeque;
|
||||
use std::convert::Infallible;
|
||||
use std::ffi::OsString;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lazaret_client::{
|
||||
BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, DomainStatus,
|
||||
parse_command_line, run,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum Endpoint {
|
||||
Existing,
|
||||
Started,
|
||||
Published,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeBackend {
|
||||
observations: VecDeque<DomainStatus<Endpoint>>,
|
||||
acquisition: Option<BootstrapAcquisition>,
|
||||
bootstrap: Option<Result<Endpoint, ClientError<Infallible>>>,
|
||||
waited: Option<Result<Endpoint, ClientError<Infallible>>>,
|
||||
execution: Option<Result<i32, ClientError<Infallible>>>,
|
||||
actions: Vec<&'static str>,
|
||||
validation: Option<ClientError<Infallible>>,
|
||||
}
|
||||
|
||||
impl ClientBackend for FakeBackend {
|
||||
type Endpoint = Endpoint;
|
||||
type Error = Infallible;
|
||||
|
||||
fn validate(&mut self, _command: &ClientCommand) -> Result<(), ClientError<Self::Error>> {
|
||||
self.actions.push("validate");
|
||||
self.validation.take().map_or(Ok(()), Err)
|
||||
}
|
||||
|
||||
fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error> {
|
||||
self.actions.push("observe");
|
||||
Ok(self.observations.pop_front().unwrap())
|
||||
}
|
||||
|
||||
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
|
||||
self.actions.push("acquire");
|
||||
Ok(self.acquisition.unwrap())
|
||||
}
|
||||
|
||||
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
self.actions.push("bootstrap");
|
||||
self.bootstrap.take().unwrap()
|
||||
}
|
||||
|
||||
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
self.actions.push("wait");
|
||||
self.waited.take().unwrap()
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&mut self,
|
||||
_endpoint: Self::Endpoint,
|
||||
_argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<Self::Error>> {
|
||||
self.actions.push("execute");
|
||||
stdout.write_all(b"out").unwrap();
|
||||
stderr.write_all(b"err").unwrap();
|
||||
self.execution.take().unwrap_or(Ok(0))
|
||||
}
|
||||
}
|
||||
|
||||
fn command() -> ClientCommand {
|
||||
ClientCommand {
|
||||
profile: "agent".into(),
|
||||
workspace: PathBuf::from("/workspace"),
|
||||
configuration: None,
|
||||
argv: vec![b"sh".to_vec(), b"-c".to_vec(), b"exit 0".to_vec()],
|
||||
}
|
||||
}
|
||||
|
||||
fn invoke(backend: &mut FakeBackend) -> Result<(i32, Vec<u8>, Vec<u8>), String> {
|
||||
let mut stdout = Vec::new();
|
||||
let mut stderr = Vec::new();
|
||||
run(backend, &command(), &mut stdout, &mut stderr)
|
||||
.map(|status| (status, stdout, stderr))
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_start_leader_bootstraps_before_forwarding_streams() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Absent].into(),
|
||||
acquisition: Some(BootstrapAcquisition::Leader),
|
||||
bootstrap: Some(Ok(Endpoint::Started)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
invoke(&mut backend),
|
||||
Ok((0, b"out".to_vec(), b"err".to_vec()))
|
||||
);
|
||||
assert_eq!(
|
||||
backend.actions,
|
||||
["validate", "observe", "acquire", "bootstrap", "execute"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_domain_is_joined_without_bootstrap() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(invoke(&mut backend).unwrap().0, 0);
|
||||
assert_eq!(backend.actions, ["validate", "observe", "execute"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_owner_disconnect_fails_that_closed_session() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Absent].into(),
|
||||
acquisition: Some(BootstrapAcquisition::Leader),
|
||||
bootstrap: Some(Err(ClientError::BootstrapOwnerDisconnected)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
invoke(&mut backend)
|
||||
.unwrap_err()
|
||||
.contains("bootstrap owner disconnected")
|
||||
);
|
||||
assert_eq!(
|
||||
backend.actions,
|
||||
["validate", "observe", "acquire", "bootstrap"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_waiter_joins_after_publication() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Starting].into(),
|
||||
waited: Some(Ok(Endpoint::Published)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(invoke(&mut backend).unwrap().0, 0);
|
||||
assert_eq!(backend.actions, ["validate", "observe", "wait", "execute"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_waiter_is_not_restarted_after_owner_loss() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Starting].into(),
|
||||
waited: Some(Err(ClientError::WaitingDomainFailed)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
invoke(&mut backend)
|
||||
.unwrap_err()
|
||||
.contains("failed while this Client was waiting")
|
||||
);
|
||||
assert_eq!(backend.actions, ["validate", "observe", "wait"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonzero_guest_exit_is_the_cli_status() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
|
||||
execution: Some(Ok(23)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(invoke(&mut backend).unwrap().0, 23);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_and_lost_coordinators_are_actionable_failures() {
|
||||
let mut stale = FakeBackend {
|
||||
observations: [DomainStatus::Stale].into(),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
assert!(invoke(&mut stale).unwrap_err().contains("stale"));
|
||||
|
||||
let mut lost = FakeBackend {
|
||||
observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
|
||||
execution: Some(Err(ClientError::LostConnection)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
assert!(
|
||||
invoke(&mut lost)
|
||||
.unwrap_err()
|
||||
.contains("connection was lost")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_host_and_configuration_errors_keep_their_cause() {
|
||||
for (error, expected) in [
|
||||
(
|
||||
ClientError::UnsupportedHost("Linux kernel 6.8 is below the required 6.9 floor".into()),
|
||||
"host is unsupported: Linux kernel 6.8 is below the required 6.9 floor",
|
||||
),
|
||||
(
|
||||
ClientError::Configuration("profile `agent` is not configured".into()),
|
||||
"configuration is invalid: profile `agent` is not configured",
|
||||
),
|
||||
] {
|
||||
let mut backend = FakeBackend {
|
||||
validation: Some(error),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
assert_eq!(invoke(&mut backend).unwrap_err(), expected);
|
||||
assert_eq!(backend.actions, ["validate"]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_line_requires_an_explicit_profile_and_preserves_command_bytes() {
|
||||
let parsed = parse_command_line(
|
||||
[
|
||||
"--profile",
|
||||
"agent",
|
||||
"--workspace",
|
||||
"/project",
|
||||
"--config",
|
||||
"/host/config.toml",
|
||||
"--",
|
||||
"tool",
|
||||
"argument",
|
||||
]
|
||||
.map(OsString::from),
|
||||
PathBuf::from("/current"),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(parsed.profile, "agent");
|
||||
assert_eq!(parsed.workspace, PathBuf::from("/project"));
|
||||
assert_eq!(
|
||||
parsed.configuration,
|
||||
Some(PathBuf::from("/host/config.toml"))
|
||||
);
|
||||
assert_eq!(parsed.argv, [b"tool".to_vec(), b"argument".to_vec()]);
|
||||
|
||||
assert!(
|
||||
parse_command_line(
|
||||
["--", "tool"].map(OsString::from),
|
||||
PathBuf::from("/current")
|
||||
)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("profile")
|
||||
);
|
||||
}
|
||||
164
crates/lazaret-client/tests/control_face.rs
Normal file
164
crates/lazaret-client/tests/control_face.rs
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
use std::fs::OpenOptions;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use lazaret_client::{ControlEndpoint, execute_over_control_face};
|
||||
use lazaret_core::identity::{
|
||||
ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, HostPath, NetworkSidecarId,
|
||||
SandboxId,
|
||||
};
|
||||
use lazaret_core::records::{
|
||||
Domain, DurableDecision, Generation, GenerationIdentity, NonInteractiveOutput,
|
||||
NonInteractiveSessionMessage, NonInteractiveSessionRequest, SessionExit,
|
||||
};
|
||||
use lazaret_host_coordinator::control::{
|
||||
ControlError, NonInteractiveSessionExecutor, PublishedControlFace, TrustAnchor,
|
||||
};
|
||||
use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||
|
||||
struct TestDirectory(PathBuf);
|
||||
|
||||
impl TestDirectory {
|
||||
fn new() -> Self {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"lazaret-client-control-{}-{}",
|
||||
std::process::id(),
|
||||
random_suffix()
|
||||
));
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
Self(path)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestDirectory {
|
||||
fn drop(&mut self) {
|
||||
std::fs::remove_dir_all(&self.0).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
struct Executor;
|
||||
|
||||
impl NonInteractiveSessionExecutor for Executor {
|
||||
fn execute(
|
||||
&self,
|
||||
request: &NonInteractiveSessionRequest,
|
||||
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<SessionExit, ControlError> {
|
||||
assert_eq!(request.argv, [b"tool".to_vec(), b"argument".to_vec()]);
|
||||
emit(NonInteractiveSessionMessage::Stdout(NonInteractiveOutput {
|
||||
session_id: request.session_id,
|
||||
bytes: b"first".to_vec(),
|
||||
}))?;
|
||||
emit(NonInteractiveSessionMessage::Stderr(NonInteractiveOutput {
|
||||
session_id: request.session_id,
|
||||
bytes: b"second".to_vec(),
|
||||
}))?;
|
||||
Ok(SessionExit::exited(request.session_id, 19))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_enrolls_registers_forwards_both_streams_and_returns_guest_status() {
|
||||
let directory = TestDirectory::new();
|
||||
let identity = generation_identity();
|
||||
let store = open_store(&directory, &identity);
|
||||
let face = Arc::new(
|
||||
PublishedControlFace::publish_with_executor(
|
||||
&directory.0,
|
||||
rustix::process::getuid().as_raw(),
|
||||
TrustAnchor::create(identity).unwrap(),
|
||||
store,
|
||||
Arc::new(Executor),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let endpoint = ControlEndpoint::new(
|
||||
face.socket_path().to_path_buf(),
|
||||
face.publication_identity().clone(),
|
||||
);
|
||||
let server = {
|
||||
let face = Arc::clone(&face);
|
||||
std::thread::spawn(move || {
|
||||
face.serve_next().unwrap();
|
||||
face.serve_next().unwrap();
|
||||
})
|
||||
};
|
||||
let mut stdout = Vec::new();
|
||||
let mut stderr = Vec::new();
|
||||
|
||||
let status = execute_over_control_face(
|
||||
&endpoint,
|
||||
&[b"tool".to_vec(), b"argument".to_vec()],
|
||||
&mut stdout,
|
||||
&mut stderr,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
server.join().unwrap();
|
||||
assert_eq!(status, 19);
|
||||
assert_eq!(stdout, b"first");
|
||||
assert_eq!(stderr, b"second");
|
||||
}
|
||||
|
||||
fn generation_identity() -> GenerationIdentity {
|
||||
GenerationIdentity {
|
||||
agent_profile: "trusted".into(),
|
||||
domain_id: DomainId::new([1; 32]),
|
||||
generation_id: GenerationId::new(1),
|
||||
configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
|
||||
coordinator_id: CoordinatorId::new([3; 32]),
|
||||
sandbox_id: SandboxId::new([4; 32]),
|
||||
}
|
||||
}
|
||||
|
||||
fn open_store(
|
||||
directory: &TestDirectory,
|
||||
identity: &GenerationIdentity,
|
||||
) -> Arc<HostCoordinatorDomainStore> {
|
||||
let lock = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.0.join("domain.lock"))
|
||||
.unwrap();
|
||||
let domain = Domain {
|
||||
domain_id: identity.domain_id,
|
||||
profile_name: identity.agent_profile.clone(),
|
||||
canonical_workspace: HostPath::new(b"/workspace/project".to_vec()),
|
||||
};
|
||||
let store = Arc::new(
|
||||
HostCoordinatorDomainStore::open(
|
||||
try_exclusive_flock(lock).unwrap(),
|
||||
&directory.0.join("domain.redb"),
|
||||
&domain,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
for durable_decision in [
|
||||
DurableDecision::None,
|
||||
DurableDecision::Preparing,
|
||||
DurableDecision::Published,
|
||||
] {
|
||||
store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(Generation {
|
||||
domain_id: identity.domain_id,
|
||||
generation_id: identity.generation_id,
|
||||
configuration_fingerprint: identity.configuration_fingerprint,
|
||||
coordinator_id: identity.coordinator_id,
|
||||
network_sidecar_id: NetworkSidecarId::new([5; 32]),
|
||||
sandbox_id: identity.sandbox_id,
|
||||
durable_decision,
|
||||
})])
|
||||
.unwrap();
|
||||
}
|
||||
store
|
||||
}
|
||||
|
||||
fn random_suffix() -> u64 {
|
||||
let mut bytes = [0_u8; 8];
|
||||
getrandom::fill(&mut bytes).unwrap();
|
||||
u64::from_ne_bytes(bytes)
|
||||
}
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
use super::{
|
||||
CapabilityId, CertificateDigest, ClientId, ConfigurationFingerprint, CoordinatorId,
|
||||
Deserialize, DomainId, GenerationId, ProtocolCredentialId, RequestDigest, RequestId, SandboxId,
|
||||
Serialize, SessionId, StreamDigest, TrustManifestDigest, fmt,
|
||||
Deserialize, DomainId, GenerationId, NonInteractiveSessionRequest, ProtocolCredentialId,
|
||||
RequestDigest, RequestId, SandboxId, Serialize, SessionId, StreamDigest, TrustManifestDigest,
|
||||
fmt,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
|
|
@ -227,6 +228,7 @@ pub struct SessionRegistrationResponse {
|
|||
pub enum ControlRequest {
|
||||
Enroll(ClientEnrollmentRequest),
|
||||
Register(SessionRegistrationRequest),
|
||||
Execute(NonInteractiveSessionRequest),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ use super::{
|
|||
HelperOwnership, HostProcessIdentity, HostProcessOwnershipSlot, Mailbox, MailboxCancellation,
|
||||
MailboxRequest, MailboxResult, MailboxResultAck, MailboxResultChunk, MountIdentity,
|
||||
NamespaceIdentity, NetworkObservation, NetworkOwnership, NetworkReachabilityProbe,
|
||||
NetworkReadinessAttestation, NetworkReadinessEvidence, NetworkReadinessIdentity, Participant,
|
||||
NetworkReadinessAttestation, NetworkReadinessEvidence, NetworkReadinessIdentity,
|
||||
NonInteractiveOutput, NonInteractiveSessionMessage, NonInteractiveSessionRequest, Participant,
|
||||
ParticipantOwnership, ParticipantState, PermittedDestinationObservation,
|
||||
PersistedHostProcessOwnership, PidfsProcessAuthorityV1, PidfsProcessIdentityV1, PolicyAction,
|
||||
PolicyDelivery, PrefixRule, PrepareOperation, PrepareStageState, PreparedArtifact,
|
||||
|
|
@ -71,6 +72,9 @@ schema_types!(
|
|||
"TerminalSignalKind" => TerminalSignalKind,
|
||||
"SessionClose" => SessionClose,
|
||||
"SessionTerminalMessage" => SessionTerminalMessage,
|
||||
"NonInteractiveSessionRequest" => NonInteractiveSessionRequest,
|
||||
"NonInteractiveOutput" => NonInteractiveOutput,
|
||||
"NonInteractiveSessionMessage" => NonInteractiveSessionMessage,
|
||||
"TerminalOutput" => TerminalOutput,
|
||||
"SessionExit" => SessionExit,
|
||||
"SessionExitOutcome" => SessionExitOutcome,
|
||||
|
|
|
|||
|
|
@ -136,3 +136,23 @@ pub enum SessionTerminalMessage {
|
|||
Output(TerminalOutput),
|
||||
Exit(SessionExit),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct NonInteractiveSessionRequest {
|
||||
pub protocol_version: super::ProtocolVersion,
|
||||
pub session_id: SessionId,
|
||||
pub argv: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct NonInteractiveOutput {
|
||||
pub session_id: SessionId,
|
||||
pub bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub enum NonInteractiveSessionMessage {
|
||||
Stdout(NonInteractiveOutput),
|
||||
Stderr(NonInteractiveOutput),
|
||||
Exit(SessionExit),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -177,6 +177,25 @@ fn every_schema_shape_round_trips_and_rejects_malformed_or_unknown_versions() {
|
|||
);
|
||||
verify_shape(SessionExit::exited(session_id(), 3), &mut seen);
|
||||
verify_shape(SessionExitOutcome::RuntimeLost, &mut seen);
|
||||
verify_shape(
|
||||
NonInteractiveSessionRequest {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
session_id: session_id(),
|
||||
argv: vec![b"tool".to_vec()],
|
||||
},
|
||||
&mut seen,
|
||||
);
|
||||
verify_shape(
|
||||
NonInteractiveOutput {
|
||||
session_id: session_id(),
|
||||
bytes: vec![53],
|
||||
},
|
||||
&mut seen,
|
||||
);
|
||||
verify_shape(
|
||||
NonInteractiveSessionMessage::Exit(SessionExit::exited(session_id(), 4)),
|
||||
&mut seen,
|
||||
);
|
||||
verify_shape(generation_identity(), &mut seen);
|
||||
verify_shape(trust_manifest(), &mut seen);
|
||||
verify_shape(tls_credential.clone(), &mut seen);
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ rustls.workspace = true
|
|||
serde.workspace = true
|
||||
sha2.workspace = true
|
||||
time.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "=3.27.0"
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ use lazaret_storage_catalog::Catalog;
|
|||
|
||||
use crate::control::ControlSessionAdmission;
|
||||
use crate::control::{
|
||||
ControlError, PublishedControlFace, SessionRegistry, StagedControlFace, TrustAnchor,
|
||||
ControlError, NonInteractiveSessionExecutor, PublishedControlFace, SessionRegistry,
|
||||
StagedControlFace, TrustAnchor,
|
||||
};
|
||||
use crate::domain_store::HostCoordinatorDomainStore;
|
||||
use crate::lifecycle::{
|
||||
|
|
@ -39,6 +40,12 @@ impl fmt::Display for ProductionLifecycleError {
|
|||
|
||||
impl std::error::Error for ProductionLifecycleError {}
|
||||
|
||||
impl ProductionLifecycleError {
|
||||
pub(crate) fn service(error: impl fmt::Display) -> Self {
|
||||
Self(error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn failure(error: impl fmt::Display) -> ProductionLifecycleError {
|
||||
ProductionLifecycleError(error.to_string())
|
||||
}
|
||||
|
|
@ -59,6 +66,7 @@ pub struct ProductionBootstrapPlan<'a> {
|
|||
pub runtime_root: PathBuf,
|
||||
pub bundle_digest: BundleDigest,
|
||||
pub invoking_uid: u32,
|
||||
pub non_interactive_executor: Arc<dyn NonInteractiveSessionExecutor>,
|
||||
}
|
||||
|
||||
pub struct ProductionLifecycleAdapter<'a> {
|
||||
|
|
@ -332,7 +340,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
|||
.take()
|
||||
.ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?;
|
||||
self.staged_control = Some(
|
||||
StagedControlFace::stage(
|
||||
StagedControlFace::stage_with_executor(
|
||||
path_from_host(
|
||||
self.plan
|
||||
.configuration
|
||||
|
|
@ -343,6 +351,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
|||
self.plan.invoking_uid,
|
||||
anchor,
|
||||
Arc::clone(&self.registry),
|
||||
Arc::clone(&self.plan.non_interactive_executor),
|
||||
)
|
||||
.map_err(failure)?,
|
||||
);
|
||||
|
|
@ -507,10 +516,13 @@ pub fn serve_production_control(
|
|||
.map_err(|error| ControlError::Lifecycle(error.to_string()))
|
||||
},
|
||||
|session_id| {
|
||||
let mut lifecycle = lifecycle.lock().map_err(|_| ControlError::LockPoisoned)?;
|
||||
lifecycle
|
||||
.lock()
|
||||
.map_err(|_| ControlError::LockPoisoned)?
|
||||
.disconnect_session(session_id)
|
||||
.map_err(|error| ControlError::Lifecycle(error.to_string()))?;
|
||||
lifecycle
|
||||
.teardown_if_idle()
|
||||
.map(|_| ())
|
||||
.map_err(|error| ControlError::Lifecycle(error.to_string()))
|
||||
},
|
||||
)
|
||||
|
|
|
|||
466
crates/lazaret-host-coordinator/src/client_bootstrap.rs
Normal file
466
crates/lazaret-host-coordinator/src/client_bootstrap.rs
Normal file
|
|
@ -0,0 +1,466 @@
|
|||
use std::cell::RefCell;
|
||||
use std::collections::BTreeMap;
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::os::fd::AsFd;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use lazaret_core::configuration::{
|
||||
ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
|
||||
};
|
||||
use lazaret_core::identity::{
|
||||
ClientId, ConfigurationFingerprint, DomainId, SessionAnchorId, SessionId,
|
||||
};
|
||||
use lazaret_core::records::{Domain, Session, SessionPhase};
|
||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
|
||||
use crate::bootstrap::{ProductionHostCoordinatorLifecycle, serve_production_control};
|
||||
use crate::domain_store::HostCoordinatorDomainStore;
|
||||
use crate::production::{BootstrapOwnerGuard, ProductionColdStart, ProductionColdStartError};
|
||||
use crate::production_selector::{InstalledProductPaths, select_production_cold_start};
|
||||
use crate::waiter::BootstrapWaiterFace;
|
||||
|
||||
pub const PRIVATE_CLIENT_BOOTSTRAP_ROLE: &str = "--lazaret-private-client-bootstrap-v1";
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PrivateClientBootstrapError(String);
|
||||
|
||||
impl fmt::Display for PrivateClientBootstrapError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PrivateClientBootstrapError {}
|
||||
|
||||
fn failure(error: impl fmt::Display) -> PrivateClientBootstrapError {
|
||||
PrivateClientBootstrapError(error.to_string())
|
||||
}
|
||||
|
||||
struct BootstrapArguments {
|
||||
profile: String,
|
||||
workspace: PathBuf,
|
||||
configuration: Option<PathBuf>,
|
||||
cli_id: ClientId,
|
||||
session_id: SessionId,
|
||||
session_anchor_id: SessionAnchorId,
|
||||
expected_domain_id: DomainId,
|
||||
expected_configuration_fingerprint: ConfigurationFingerprint,
|
||||
}
|
||||
|
||||
/// Dispatches the exact private Client cold-start entry.
|
||||
#[must_use]
|
||||
pub fn dispatch_private_client_bootstrap(
|
||||
arguments: &[OsString],
|
||||
) -> Option<Result<(), PrivateClientBootstrapError>> {
|
||||
if arguments
|
||||
.first()
|
||||
.is_none_or(|argument| argument != PRIVATE_CLIENT_BOOTSTRAP_ROLE)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(run_private_client_bootstrap(arguments))
|
||||
}
|
||||
|
||||
fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateClientBootstrapError> {
|
||||
let parsed = parse_arguments(arguments).inspect_err(|_| signal(b'E'))?;
|
||||
let owner = BootstrapOwnerGuard::from_stdin()
|
||||
.map_err(failure)
|
||||
.inspect_err(|_| signal(b'E'))?;
|
||||
let configuration = load_configuration(&parsed)
|
||||
.map_err(failure)
|
||||
.inspect_err(|_| signal(b'E'))?;
|
||||
if !matches_client_expectation(
|
||||
configuration.domain_id(),
|
||||
configuration.configuration_fingerprint(),
|
||||
&parsed,
|
||||
) {
|
||||
signal(b'E');
|
||||
return Err(failure(
|
||||
"independently resolved Domain identity differs from the Client expectation",
|
||||
));
|
||||
}
|
||||
let persistent_domain = configuration
|
||||
.persistent_domain_root()
|
||||
.open_authority()
|
||||
.map_err(failure)
|
||||
.inspect_err(|_| signal(b'E'))?;
|
||||
let lock_file = open_private_domain_file(&persistent_domain, "domain.lock")
|
||||
.inspect_err(|_| signal(b'E'))?;
|
||||
let domain_lock = match try_exclusive_flock(lock_file) {
|
||||
Ok(lock) => lock,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||
signal(b'W');
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => {
|
||||
signal(b'E');
|
||||
return Err(failure(error));
|
||||
}
|
||||
};
|
||||
let domain = Domain {
|
||||
domain_id: configuration.domain_id(),
|
||||
profile_name: configuration.selected_profile().to_owned(),
|
||||
canonical_workspace: configuration.workspace().canonical_label().clone(),
|
||||
};
|
||||
let store = Arc::new(
|
||||
HostCoordinatorDomainStore::open_file(
|
||||
domain_lock,
|
||||
open_private_domain_file(&persistent_domain, "domain.redb")
|
||||
.inspect_err(|_| signal(b'E'))?,
|
||||
&domain,
|
||||
)
|
||||
.map_err(failure)
|
||||
.inspect_err(|_| signal(b'E'))?,
|
||||
);
|
||||
signal(b'L');
|
||||
|
||||
let executable = std::env::current_exe().map_err(failure)?;
|
||||
let installed = executable
|
||||
.parent()
|
||||
.ok_or_else(|| failure("Host Coordinator executable has no installation directory"))?;
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(failure)?;
|
||||
let selection = runtime
|
||||
.block_on(select_production_cold_start(
|
||||
&configuration,
|
||||
Arc::clone(&store),
|
||||
InstalledProductPaths {
|
||||
escalation_client: &installed.join("lazaret-escalation-client"),
|
||||
sandbox_init: &executable,
|
||||
},
|
||||
))
|
||||
.map_err(failure)?;
|
||||
let inputs = selection
|
||||
.cold_start_inputs(&configuration)
|
||||
.map_err(failure)?;
|
||||
let bootstrap_owner = Session {
|
||||
generation_id: inputs.generation.generation_id,
|
||||
session_id: parsed.session_id,
|
||||
cli_id: parsed.cli_id,
|
||||
session_anchor_id: parsed.session_anchor_id,
|
||||
session_phase: SessionPhase::New,
|
||||
};
|
||||
ProductionColdStart::stage(inputs)
|
||||
.map_err(failure)?
|
||||
.run(bootstrap_owner, &owner, serve_published)
|
||||
.map_err(failure)
|
||||
}
|
||||
|
||||
fn serve_published(
|
||||
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
|
||||
waiters: &RefCell<BootstrapWaiterFace>,
|
||||
bootstrap_owner: &BootstrapOwnerGuard,
|
||||
bootstrap_owner_session_id: SessionId,
|
||||
) -> Result<(), ProductionColdStartError> {
|
||||
lifecycle
|
||||
.lock()
|
||||
.map_err(|_| ProductionColdStartError::service("production lifecycle lock is poisoned"))?
|
||||
.adapter()
|
||||
.published_control()
|
||||
.ok_or_else(|| ProductionColdStartError::service("control face is absent"))?
|
||||
.set_nonblocking(true)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
loop {
|
||||
{
|
||||
let mut lifecycle = lifecycle.lock().map_err(|_| {
|
||||
ProductionColdStartError::service("production lifecycle lock is poisoned")
|
||||
})?;
|
||||
lifecycle
|
||||
.reconcile_published_bootstrap_owner(
|
||||
bootstrap_owner_session_id,
|
||||
bootstrap_owner
|
||||
.is_live()
|
||||
.map_err(ProductionColdStartError::service)?,
|
||||
)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
if lifecycle.adapter().published_control().is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
waiters
|
||||
.borrow_mut()
|
||||
.reconcile_published(&mut lifecycle)
|
||||
.map_err(ProductionColdStartError::service)?;
|
||||
}
|
||||
match serve_production_control(lifecycle) {
|
||||
Ok(()) => {}
|
||||
Err(crate::control::ControlError::Io(error))
|
||||
if error.kind() == std::io::ErrorKind::WouldBlock =>
|
||||
{
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
Err(error) => return Err(ProductionColdStartError::service(error)),
|
||||
}
|
||||
if lifecycle
|
||||
.lock()
|
||||
.map_err(|_| {
|
||||
ProductionColdStartError::service("production lifecycle lock is poisoned")
|
||||
})?
|
||||
.adapter()
|
||||
.published_control()
|
||||
.is_none()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn matches_client_expectation(
|
||||
domain_id: DomainId,
|
||||
configuration_fingerprint: ConfigurationFingerprint,
|
||||
arguments: &BootstrapArguments,
|
||||
) -> bool {
|
||||
domain_id == arguments.expected_domain_id
|
||||
&& configuration_fingerprint == arguments.expected_configuration_fingerprint
|
||||
}
|
||||
|
||||
fn parse_arguments(
|
||||
arguments: &[OsString],
|
||||
) -> Result<BootstrapArguments, PrivateClientBootstrapError> {
|
||||
let mut arguments = arguments.iter().skip(1);
|
||||
require_option(&mut arguments, "--profile")?;
|
||||
let profile = required(&mut arguments, "profile")?
|
||||
.to_str()
|
||||
.ok_or_else(|| failure("profile is not UTF-8"))?
|
||||
.to_owned();
|
||||
require_option(&mut arguments, "--workspace")?;
|
||||
let workspace = PathBuf::from(required(&mut arguments, "workspace")?);
|
||||
let mut configuration = None;
|
||||
let next = required(&mut arguments, "identity option")?;
|
||||
let client_option = if next == OsStr::new("--config") {
|
||||
configuration = Some(PathBuf::from(required(&mut arguments, "configuration")?));
|
||||
required(&mut arguments, "Client ID option")?
|
||||
} else {
|
||||
next
|
||||
};
|
||||
if client_option != OsStr::new("--client-id") {
|
||||
return Err(failure("expected --client-id"));
|
||||
}
|
||||
let cli_id = ClientId::new(decode_id(required(&mut arguments, "Client ID")?)?);
|
||||
require_option(&mut arguments, "--session-id")?;
|
||||
let session_id = SessionId::new(decode_id(required(&mut arguments, "Session ID")?)?);
|
||||
require_option(&mut arguments, "--anchor-id")?;
|
||||
let session_anchor_id =
|
||||
SessionAnchorId::new(decode_id(required(&mut arguments, "Session Anchor ID")?)?);
|
||||
require_option(&mut arguments, "--domain-id")?;
|
||||
let expected_domain_id = DomainId::new(decode_id(required(&mut arguments, "Domain ID")?)?);
|
||||
require_option(&mut arguments, "--configuration-fingerprint")?;
|
||||
let expected_configuration_fingerprint = ConfigurationFingerprint::new(decode_id(required(
|
||||
&mut arguments,
|
||||
"Configuration Fingerprint",
|
||||
)?)?);
|
||||
if arguments.next().is_some() {
|
||||
return Err(failure("unexpected private bootstrap argument"));
|
||||
}
|
||||
Ok(BootstrapArguments {
|
||||
profile,
|
||||
workspace,
|
||||
configuration,
|
||||
cli_id,
|
||||
session_id,
|
||||
session_anchor_id,
|
||||
expected_domain_id,
|
||||
expected_configuration_fingerprint,
|
||||
})
|
||||
}
|
||||
|
||||
fn open_private_domain_file(
|
||||
root: &impl AsFd,
|
||||
name: &str,
|
||||
) -> Result<File, PrivateClientBootstrapError> {
|
||||
let descriptor = openat(
|
||||
root,
|
||||
name,
|
||||
OFlags::RDWR | OFlags::CREATE | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)
|
||||
.map_err(failure)?;
|
||||
let file = File::from(descriptor);
|
||||
let metadata = file.metadata().map_err(failure)?;
|
||||
if !metadata.file_type().is_file()
|
||||
|| metadata.uid() != rustix::process::getuid().as_raw()
|
||||
|| metadata.mode() & 0o7777 != 0o600
|
||||
|| metadata.nlink() != 1
|
||||
{
|
||||
return Err(failure(
|
||||
"private Domain file is not exact invoking-user mode-0600 authority",
|
||||
));
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn required<'a>(
|
||||
arguments: &mut impl Iterator<Item = &'a OsString>,
|
||||
label: &str,
|
||||
) -> Result<&'a OsStr, PrivateClientBootstrapError> {
|
||||
arguments
|
||||
.next()
|
||||
.map(OsString::as_os_str)
|
||||
.ok_or_else(|| failure(format!("missing {label}")))
|
||||
}
|
||||
|
||||
fn require_option<'a>(
|
||||
arguments: &mut impl Iterator<Item = &'a OsString>,
|
||||
expected: &str,
|
||||
) -> Result<(), PrivateClientBootstrapError> {
|
||||
if required(arguments, expected)? == OsStr::new(expected) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(failure(format!("expected {expected}")))
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_id(value: &OsStr) -> Result<[u8; 32], PrivateClientBootstrapError> {
|
||||
let value = value
|
||||
.to_str()
|
||||
.ok_or_else(|| failure("identity is not UTF-8"))?;
|
||||
if value.len() != 64 {
|
||||
return Err(failure("identity is not 32-byte lowercase hexadecimal"));
|
||||
}
|
||||
let mut decoded = [0_u8; 32];
|
||||
for (output, pair) in decoded.iter_mut().zip(value.as_bytes().chunks_exact(2)) {
|
||||
*output = match (hex_nibble(pair[0]), hex_nibble(pair[1])) {
|
||||
(Some(high), Some(low)) => high << 4 | low,
|
||||
_ => return Err(failure("identity is not 32-byte lowercase hexadecimal")),
|
||||
};
|
||||
}
|
||||
Ok(decoded)
|
||||
}
|
||||
|
||||
const fn hex_nibble(byte: u8) -> Option<u8> {
|
||||
match byte {
|
||||
b'0'..=b'9' => Some(byte - b'0'),
|
||||
b'a'..=b'f' => Some(byte - b'a' + 10),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn load_configuration(
|
||||
arguments: &BootstrapArguments,
|
||||
) -> Result<ResolvedDomainConfiguration, Box<dyn std::error::Error>> {
|
||||
let home = required_environment_path("HOME")?;
|
||||
let runtime = required_environment_path("XDG_RUNTIME_DIR")?;
|
||||
Ok(load_trusted_configuration(ConfigurationLoadRequest {
|
||||
explicit_configuration_path: arguments.configuration.clone(),
|
||||
selected_profile: arguments.profile.clone(),
|
||||
workspace: arguments.workspace.clone(),
|
||||
home,
|
||||
xdg_config_home: optional_environment_path("XDG_CONFIG_HOME"),
|
||||
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
|
||||
xdg_runtime_dir: runtime,
|
||||
environment: environment(),
|
||||
sandbox_layout: production_sandbox_layout()?,
|
||||
})?)
|
||||
}
|
||||
|
||||
fn required_environment_path(name: &str) -> Result<PathBuf, PrivateClientBootstrapError> {
|
||||
std::env::var_os(name)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.ok_or_else(|| failure(format!("{name} is not set")))
|
||||
}
|
||||
|
||||
fn optional_environment_path(name: &str) -> Option<PathBuf> {
|
||||
std::env::var_os(name)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(PathBuf::from)
|
||||
}
|
||||
|
||||
fn environment() -> BTreeMap<String, OsString> {
|
||||
std::env::vars_os()
|
||||
.filter_map(|(name, value)| name.into_string().ok().map(|name| (name, value)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn signal(byte: u8) {
|
||||
let _ = std::io::stdout().write_all(&[byte]);
|
||||
let _ = std::io::stdout().flush();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn private_bootstrap_arguments_are_exact_and_public() {
|
||||
let arguments = [
|
||||
PRIVATE_CLIENT_BOOTSTRAP_ROLE,
|
||||
"--profile",
|
||||
"agent",
|
||||
"--workspace",
|
||||
"/workspace/with spaces",
|
||||
"--config",
|
||||
"/host/config.toml",
|
||||
"--client-id",
|
||||
&"11".repeat(32),
|
||||
"--session-id",
|
||||
&"22".repeat(32),
|
||||
"--anchor-id",
|
||||
&"33".repeat(32),
|
||||
"--domain-id",
|
||||
&"44".repeat(32),
|
||||
"--configuration-fingerprint",
|
||||
&"55".repeat(32),
|
||||
]
|
||||
.map(OsString::from);
|
||||
let parsed = parse_arguments(&arguments).unwrap();
|
||||
assert_eq!(parsed.profile, "agent");
|
||||
assert_eq!(parsed.workspace, Path::new("/workspace/with spaces"));
|
||||
assert_eq!(
|
||||
parsed.configuration.as_deref(),
|
||||
Some(Path::new("/host/config.toml"))
|
||||
);
|
||||
assert_eq!(parsed.cli_id, ClientId::new([0x11; 32]));
|
||||
assert_eq!(parsed.session_id, SessionId::new([0x22; 32]));
|
||||
assert_eq!(parsed.session_anchor_id, SessionAnchorId::new([0x33; 32]));
|
||||
assert_eq!(parsed.expected_domain_id, DomainId::new([0x44; 32]));
|
||||
assert_eq!(
|
||||
parsed.expected_configuration_fingerprint,
|
||||
ConfigurationFingerprint::new([0x55; 32])
|
||||
);
|
||||
assert!(matches_client_expectation(
|
||||
DomainId::new([0x44; 32]),
|
||||
ConfigurationFingerprint::new([0x55; 32]),
|
||||
&parsed
|
||||
));
|
||||
assert!(!matches_client_expectation(
|
||||
DomainId::new([0x44; 32]),
|
||||
ConfigurationFingerprint::new([0x56; 32]),
|
||||
&parsed
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn private_bootstrap_dispatch_and_identity_grammar_are_closed() {
|
||||
assert!(dispatch_private_client_bootstrap(&[]).is_none());
|
||||
let extra = [
|
||||
PRIVATE_CLIENT_BOOTSTRAP_ROLE,
|
||||
"--profile",
|
||||
"agent",
|
||||
"--workspace",
|
||||
"/workspace",
|
||||
"--client-id",
|
||||
&"11".repeat(32),
|
||||
"--session-id",
|
||||
&"22".repeat(32),
|
||||
"--anchor-id",
|
||||
&"AA".repeat(32),
|
||||
"--domain-id",
|
||||
&"44".repeat(32),
|
||||
"--configuration-fingerprint",
|
||||
&"55".repeat(32),
|
||||
]
|
||||
.map(OsString::from);
|
||||
assert!(parse_arguments(&extra).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,16 +1,19 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::fmt;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Read, Seek, SeekFrom, Write};
|
||||
use std::os::fd::AsFd;
|
||||
use std::os::fd::{AsFd, AsRawFd};
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
|
||||
use ed25519_dalek::SigningKey;
|
||||
use lazaret_core::codec::{CodecError, decode, encode};
|
||||
use lazaret_core::configuration::ConfinedPathPlan;
|
||||
use lazaret_core::identity::{
|
||||
CertificateDigest, ClientId, SessionAnchorId, SessionId, TrustManifestDigest,
|
||||
CertificateDigest, ClientId, ContainerId, SessionAnchorId, SessionId, TrustManifestDigest,
|
||||
};
|
||||
use lazaret_core::protocol::{
|
||||
ProtocolValidationError, issue_protocol_credential, issue_session_capability,
|
||||
|
|
@ -18,10 +21,11 @@ use lazaret_core::protocol::{
|
|||
};
|
||||
use lazaret_core::records::{
|
||||
ClientEnrollmentRequest, ClientEnrollmentResponse, ControlRequest, ControlResponse,
|
||||
DurableDecision, GenerationIdentity, ProtocolCredential, ProtocolCredentialRole,
|
||||
DurableDecision, GenerationIdentity, NonInteractiveSessionMessage,
|
||||
NonInteractiveSessionRequest, ProtocolCredential, ProtocolCredentialRole,
|
||||
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, SandboxCredentialBundle,
|
||||
Session, SessionBinding, SessionPhase, SessionRegistrationRequest, SessionRegistrationResponse,
|
||||
TlsCredential, TlsRole, TlsSubjectId, TrustManifest,
|
||||
Session, SessionBinding, SessionExit, SessionPhase, SessionRegistrationRequest,
|
||||
SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId, TrustManifest,
|
||||
};
|
||||
use rcgen::{
|
||||
BasicConstraints, CertificateParams, CertificateSigningRequestParams, CertifiedIssuer,
|
||||
|
|
@ -42,14 +46,16 @@ use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinator
|
|||
use crate::lifecycle::{
|
||||
HostCoordinatorLifecycle, LifecycleAction, SessionAdmissionIdentity, SessionLifecycleAdapter,
|
||||
};
|
||||
use lazaret_linux::process::{ProcessExit, TargetCommand, child_broker};
|
||||
use lazaret_runtime_supervisor::sandbox::RunscCommand;
|
||||
|
||||
pub const CONTROL_ALPN: &[u8] = b"lazaret-control/1";
|
||||
pub const CONTROL_SERVER_NAME: &str = "coordinator.lazaret.invalid";
|
||||
pub const MAX_ENROLLMENT_BYTES: usize = 8_192;
|
||||
const MAX_CONTROL_BYTES: usize = 65_536;
|
||||
const CONTROL_SOCKET_NAME: &str = "control.sock";
|
||||
const CONTROL_SOCKET_NAME: &str = "c.sock";
|
||||
const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard";
|
||||
const STAGED_CONTROL_SOCKET_NAME: &str = ".control.sock.staged";
|
||||
const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
|
||||
const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
@ -824,6 +830,167 @@ pub struct PublishedControlFace {
|
|||
anchor: TrustAnchor,
|
||||
server_config: Arc<ServerConfig>,
|
||||
registry: Arc<SessionRegistry>,
|
||||
executor: Arc<dyn NonInteractiveSessionExecutor>,
|
||||
}
|
||||
|
||||
pub trait NonInteractiveSessionExecutor: Send + Sync {
|
||||
/// # Errors
|
||||
/// Returns a runtime, stream, or exact-session execution failure.
|
||||
fn execute(
|
||||
&self,
|
||||
request: &NonInteractiveSessionRequest,
|
||||
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<SessionExit, ControlError>;
|
||||
}
|
||||
|
||||
struct ExecutionUnavailable;
|
||||
|
||||
impl NonInteractiveSessionExecutor for ExecutionUnavailable {
|
||||
fn execute(
|
||||
&self,
|
||||
_request: &NonInteractiveSessionRequest,
|
||||
_emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<SessionExit, ControlError> {
|
||||
Err(ControlError::InvalidRequest)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RunscNonInteractiveExecutor {
|
||||
program: OsString,
|
||||
global_arguments: Vec<OsString>,
|
||||
container: OsString,
|
||||
runner: Arc<dyn NonInteractiveCommandRunner>,
|
||||
}
|
||||
|
||||
trait NonInteractiveCommandRunner: Send + Sync {
|
||||
fn run(
|
||||
&self,
|
||||
target: &TargetCommand,
|
||||
session_id: SessionId,
|
||||
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<ProcessExit, ControlError>;
|
||||
}
|
||||
|
||||
struct ChildBrokerCommandRunner;
|
||||
|
||||
impl NonInteractiveCommandRunner for ChildBrokerCommandRunner {
|
||||
fn run(
|
||||
&self,
|
||||
target: &TargetCommand,
|
||||
session_id: SessionId,
|
||||
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<ProcessExit, ControlError> {
|
||||
let mut child = child_broker().run_transient_captured(target)?;
|
||||
let mut stdout_done = false;
|
||||
let mut stderr_done = false;
|
||||
while !(stdout_done && stderr_done && child.process().has_exited()?) {
|
||||
let (stdout, stderr) = child.streams();
|
||||
stdout_done |= emit_available(stdout, session_id, true, emit)?;
|
||||
stderr_done |= emit_available(stderr, session_id, false, emit)?;
|
||||
if !(stdout_done && stderr_done) {
|
||||
std::thread::yield_now();
|
||||
}
|
||||
}
|
||||
child_broker()
|
||||
.reap_status(child.into_process())
|
||||
.map_err(Into::into)
|
||||
}
|
||||
}
|
||||
|
||||
impl RunscNonInteractiveExecutor {
|
||||
/// Derives the exact `runsc exec` prefix from the generation's attached `runsc run` command.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a non-runsc command or a command without one exact run/container suffix.
|
||||
pub fn from_run_command(
|
||||
command: &RunscCommand,
|
||||
container_id: ContainerId,
|
||||
) -> Result<Self, ControlError> {
|
||||
if command.program() != OsStr::new("runsc") {
|
||||
return Err(ControlError::InvalidRequest);
|
||||
}
|
||||
let run = command
|
||||
.arguments()
|
||||
.iter()
|
||||
.position(|argument| argument == OsStr::new("run"))
|
||||
.ok_or(ControlError::InvalidRequest)?;
|
||||
if run + 3 != command.arguments().len()
|
||||
|| !command.arguments()[run + 1]
|
||||
.as_encoded_bytes()
|
||||
.starts_with(b"-bundle=")
|
||||
{
|
||||
return Err(ControlError::InvalidRequest);
|
||||
}
|
||||
let container = OsString::from(hex_lower(container_id.as_bytes()));
|
||||
if command.arguments()[run + 2] != container {
|
||||
return Err(ControlError::InvalidRequest);
|
||||
}
|
||||
Ok(Self {
|
||||
program: command.program().to_owned(),
|
||||
global_arguments: command.arguments()[..run].to_vec(),
|
||||
container,
|
||||
runner: Arc::new(ChildBrokerCommandRunner),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl NonInteractiveSessionExecutor for RunscNonInteractiveExecutor {
|
||||
fn execute(
|
||||
&self,
|
||||
request: &NonInteractiveSessionRequest,
|
||||
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<SessionExit, ControlError> {
|
||||
let mut target = TargetCommand::new(self.program.clone());
|
||||
for argument in &self.global_arguments {
|
||||
target = target.argument(argument.clone());
|
||||
}
|
||||
target = target.argument("exec").argument(self.container.clone());
|
||||
for argument in &request.argv {
|
||||
target = target.argument(OsString::from_vec(argument.clone()));
|
||||
}
|
||||
let process_status = self.runner.run(&target, request.session_id, emit)?;
|
||||
match (process_status.exit_code, process_status.terminating_signal) {
|
||||
(Some(code), None) => Ok(SessionExit::exited(request.session_id, code)),
|
||||
(None, Some(_)) => Ok(SessionExit::terminated(request.session_id)),
|
||||
_ => Err(ControlError::InvalidRequest),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_available(
|
||||
stream: &mut File,
|
||||
session_id: SessionId,
|
||||
stdout: bool,
|
||||
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<bool, ControlError> {
|
||||
let mut bytes = [0_u8; 16 * 1024];
|
||||
loop {
|
||||
match stream.read(&mut bytes) {
|
||||
Ok(0) => return Ok(true),
|
||||
Ok(length) => {
|
||||
let output = lazaret_core::records::NonInteractiveOutput {
|
||||
session_id,
|
||||
bytes: bytes[..length].to_vec(),
|
||||
};
|
||||
emit(if stdout {
|
||||
NonInteractiveSessionMessage::Stdout(output)
|
||||
} else {
|
||||
NonInteractiveSessionMessage::Stderr(output)
|
||||
})?;
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::WouldBlock => return Ok(false),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_lower(bytes: &[u8]) -> String {
|
||||
use fmt::Write as _;
|
||||
|
||||
bytes.iter().fold(String::new(), |mut output, byte| {
|
||||
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
|
||||
output
|
||||
})
|
||||
}
|
||||
|
||||
pub struct StagedControlFace {
|
||||
|
|
@ -838,6 +1005,7 @@ pub struct StagedControlFace {
|
|||
anchor: Option<TrustAnchor>,
|
||||
server_config: Arc<ServerConfig>,
|
||||
registry: Arc<SessionRegistry>,
|
||||
executor: Arc<dyn NonInteractiveSessionExecutor>,
|
||||
}
|
||||
|
||||
impl StagedControlFace {
|
||||
|
|
@ -850,6 +1018,26 @@ impl StagedControlFace {
|
|||
uid: u32,
|
||||
anchor: TrustAnchor,
|
||||
registry: Arc<SessionRegistry>,
|
||||
) -> Result<Self, ControlError> {
|
||||
Self::stage_with_executor(
|
||||
runtime_domain_root,
|
||||
uid,
|
||||
anchor,
|
||||
registry,
|
||||
Arc::new(ExecutionUnavailable),
|
||||
)
|
||||
}
|
||||
|
||||
/// Stages a control face with its generation-bound non-interactive executor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses the same invalid staging conditions as [`Self::stage`].
|
||||
pub fn stage_with_executor(
|
||||
runtime_domain_root: &Path,
|
||||
uid: u32,
|
||||
anchor: TrustAnchor,
|
||||
registry: Arc<SessionRegistry>,
|
||||
executor: Arc<dyn NonInteractiveSessionExecutor>,
|
||||
) -> Result<Self, ControlError> {
|
||||
if registry.generation_identity != anchor.generation_identity {
|
||||
return Err(ControlError::StaleGeneration);
|
||||
|
|
@ -902,6 +1090,7 @@ impl StagedControlFace {
|
|||
anchor: Some(anchor),
|
||||
server_config,
|
||||
registry,
|
||||
executor,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -959,6 +1148,7 @@ impl StagedControlFace {
|
|||
))?,
|
||||
server_config: Arc::clone(&self.server_config),
|
||||
registry: Arc::clone(&self.registry),
|
||||
executor: Arc::clone(&self.executor),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -971,6 +1161,128 @@ impl Drop for StagedControlFace {
|
|||
}
|
||||
|
||||
impl PublishedControlFace {
|
||||
/// Selects blocking or polling admission for the publication listener.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an operating-system failure changing the listener mode.
|
||||
pub fn set_nonblocking(&self, nonblocking: bool) -> Result<(), ControlError> {
|
||||
self.listener.set_nonblocking(nonblocking)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Discovers one publication through the retained runtime Domain authority.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses changed root authority or absent, replaced, non-private, oversized, or malformed
|
||||
/// publication objects.
|
||||
pub fn discover_confined(
|
||||
runtime_domain_root: &ConfinedPathPlan,
|
||||
uid: u32,
|
||||
) -> Result<(File, PathBuf, ProtocolPublicationIdentity), ControlError> {
|
||||
let root = File::from(
|
||||
runtime_domain_root
|
||||
.open_authority()
|
||||
.map_err(|_| ControlError::InvalidPath("runtime Domain authority changed"))?,
|
||||
);
|
||||
let root_metadata = root.metadata()?;
|
||||
if !root_metadata.is_dir()
|
||||
|| root_metadata.uid() != uid
|
||||
|| root_metadata.mode() & 0o7777 != 0o700
|
||||
{
|
||||
return Err(ControlError::InvalidPath(
|
||||
"runtime Domain root is not invoking-user-owned mode 0700",
|
||||
));
|
||||
}
|
||||
let descriptor = openat(
|
||||
root.as_fd(),
|
||||
PUBLICATION_IDENTITY_NAME,
|
||||
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let mut identity = File::from(descriptor);
|
||||
let identity_metadata = identity.metadata()?;
|
||||
if !identity_metadata.file_type().is_file()
|
||||
|| identity_metadata.uid() != uid
|
||||
|| identity_metadata.mode() & 0o7777 != 0o600
|
||||
|| identity_metadata.nlink() != 1
|
||||
|| identity_metadata.len() > MAX_CONTROL_BYTES as u64
|
||||
{
|
||||
return Err(ControlError::InvalidPublication(
|
||||
"publication identity is not an exact private regular file",
|
||||
));
|
||||
}
|
||||
let socket = File::from(
|
||||
openat(
|
||||
root.as_fd(),
|
||||
CONTROL_SOCKET_NAME,
|
||||
OFlags::PATH | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(io::Error::from)?,
|
||||
);
|
||||
let socket_metadata = socket.metadata()?;
|
||||
if !socket_metadata.file_type().is_socket()
|
||||
|| socket_metadata.uid() != uid
|
||||
|| socket_metadata.mode() & 0o7777 != 0o600
|
||||
{
|
||||
return Err(ControlError::InvalidPublication(
|
||||
"control socket is not an invoking-user-private socket",
|
||||
));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
identity.read_to_end(&mut bytes)?;
|
||||
let socket_path = PathBuf::from(format!(
|
||||
"/proc/self/fd/{}/{}",
|
||||
root.as_raw_fd(),
|
||||
CONTROL_SOCKET_NAME
|
||||
));
|
||||
Ok((root, socket_path, decode(&bytes)?))
|
||||
}
|
||||
|
||||
/// Discovers one exact current control publication beneath a validated runtime Domain root.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses absent, replaced, non-private, oversized, or malformed publication objects.
|
||||
pub fn discover(
|
||||
runtime_domain_root: &Path,
|
||||
uid: u32,
|
||||
) -> Result<(PathBuf, ProtocolPublicationIdentity), ControlError> {
|
||||
let paths = ControlPaths::open(runtime_domain_root, uid)?;
|
||||
let descriptor = openat(
|
||||
paths.root_file.as_fd(),
|
||||
PUBLICATION_IDENTITY_NAME,
|
||||
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(io::Error::from)?;
|
||||
let mut identity = File::from(descriptor);
|
||||
let descriptor_metadata = identity.metadata()?;
|
||||
let path_metadata = std::fs::symlink_metadata(&paths.identity)?;
|
||||
let expected = FileIdentity::from_metadata(&descriptor_metadata);
|
||||
if FileIdentity::from_metadata(&path_metadata) != expected
|
||||
|| !descriptor_metadata.file_type().is_file()
|
||||
|| descriptor_metadata.uid() != uid
|
||||
|| descriptor_metadata.mode() & 0o7777 != 0o600
|
||||
|| descriptor_metadata.nlink() != 1
|
||||
|| descriptor_metadata.len() > MAX_CONTROL_BYTES as u64
|
||||
{
|
||||
return Err(ControlError::InvalidPublication(
|
||||
"publication identity is not an exact private regular file",
|
||||
));
|
||||
}
|
||||
let socket = std::fs::symlink_metadata(&paths.socket)?;
|
||||
if !socket.file_type().is_socket() || socket.uid() != uid || socket.mode() & 0o7777 != 0o600
|
||||
{
|
||||
return Err(ControlError::InvalidPublication(
|
||||
"control socket is not an invoking-user-private socket",
|
||||
));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
identity.read_to_end(&mut bytes)?;
|
||||
Ok((paths.socket, decode(&bytes)?))
|
||||
}
|
||||
|
||||
/// Publishes one generation control face in the fixed host-private runtime path.
|
||||
///
|
||||
/// # Errors
|
||||
|
|
@ -981,12 +1293,39 @@ impl PublishedControlFace {
|
|||
uid: u32,
|
||||
anchor: TrustAnchor,
|
||||
store: Arc<HostCoordinatorDomainStore>,
|
||||
) -> Result<Self, ControlError> {
|
||||
Self::publish_with_executor(
|
||||
runtime_domain_root,
|
||||
uid,
|
||||
anchor,
|
||||
store,
|
||||
Arc::new(ExecutionUnavailable),
|
||||
)
|
||||
}
|
||||
|
||||
/// Publishes one generation control face with non-interactive session execution.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses the same invalid publication conditions as [`Self::publish`].
|
||||
pub fn publish_with_executor(
|
||||
runtime_domain_root: &Path,
|
||||
uid: u32,
|
||||
anchor: TrustAnchor,
|
||||
store: Arc<HostCoordinatorDomainStore>,
|
||||
executor: Arc<dyn NonInteractiveSessionExecutor>,
|
||||
) -> Result<Self, ControlError> {
|
||||
let registry = Arc::new(SessionRegistry::new(
|
||||
store,
|
||||
anchor.generation_identity.clone(),
|
||||
));
|
||||
StagedControlFace::stage(runtime_domain_root, uid, anchor, registry)?.expose()
|
||||
StagedControlFace::stage_with_executor(
|
||||
runtime_domain_root,
|
||||
uid,
|
||||
anchor,
|
||||
registry,
|
||||
executor,
|
||||
)?
|
||||
.expose()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
|
|
@ -1132,13 +1471,44 @@ impl PublishedControlFace {
|
|||
session_capability: capability,
|
||||
}),
|
||||
)?;
|
||||
let mut extra = [0_u8; 1];
|
||||
match stream.read(&mut extra) {
|
||||
Ok(0) => Ok(()),
|
||||
Ok(_) => Err(ControlError::InvalidRequest),
|
||||
Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => Ok(()),
|
||||
Err(error) => Err(error.into()),
|
||||
let request: ControlRequest = match read_frame(stream, MAX_CONTROL_BYTES) {
|
||||
Ok(request) => request,
|
||||
Err(ControlError::Io(error))
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset
|
||||
) =>
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let ControlRequest::Execute(request) = request else {
|
||||
return Err(ControlError::InvalidRequest);
|
||||
};
|
||||
if request.session_id != session_id
|
||||
|| request.argv.is_empty()
|
||||
|| request.argv[0].is_empty()
|
||||
|| request.argv.iter().any(|argument| argument.contains(&0))
|
||||
{
|
||||
return Err(ControlError::InvalidRequest);
|
||||
}
|
||||
let mut send_message = |message: NonInteractiveSessionMessage| {
|
||||
if matches!(message, NonInteractiveSessionMessage::Exit(_)) {
|
||||
return Err(ControlError::InvalidRequest);
|
||||
}
|
||||
require_non_interactive_session(&message, request.session_id)?;
|
||||
write_frame(stream, &message)
|
||||
};
|
||||
let outcome = self.executor.execute(&request, &mut send_message)?;
|
||||
if outcome.session_id() != request.session_id {
|
||||
return Err(ControlError::InvalidRequest);
|
||||
}
|
||||
require_non_interactive_session(
|
||||
&NonInteractiveSessionMessage::Exit(outcome),
|
||||
request.session_id,
|
||||
)?;
|
||||
write_frame(stream, &NonInteractiveSessionMessage::Exit(outcome))
|
||||
})();
|
||||
let disconnected = disconnect(session_id);
|
||||
result.and(disconnected)
|
||||
|
|
@ -1301,6 +1671,22 @@ fn remove_if_identity_matches(path: &Path, expected: FileIdentity) {
|
|||
}
|
||||
}
|
||||
|
||||
fn require_non_interactive_session(
|
||||
message: &NonInteractiveSessionMessage,
|
||||
expected: SessionId,
|
||||
) -> Result<(), ControlError> {
|
||||
let actual = match message {
|
||||
NonInteractiveSessionMessage::Stdout(output)
|
||||
| NonInteractiveSessionMessage::Stderr(output) => output.session_id,
|
||||
NonInteractiveSessionMessage::Exit(exit) => exit.session_id(),
|
||||
};
|
||||
if actual == expected {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ControlError::InvalidRequest)
|
||||
}
|
||||
}
|
||||
|
||||
fn leaf_parameters(
|
||||
common_name: &str,
|
||||
usage: ExtendedKeyUsagePurpose,
|
||||
|
|
@ -1406,6 +1792,7 @@ fn write_frame<Record: serde::Serialize>(
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::ffi::OsString;
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
|
||||
|
|
@ -1483,6 +1870,64 @@ mod tests {
|
|||
));
|
||||
}
|
||||
|
||||
struct RecordingRunner {
|
||||
observed: Mutex<Option<(OsString, Vec<OsString>)>>,
|
||||
exit: ProcessExit,
|
||||
}
|
||||
|
||||
impl NonInteractiveCommandRunner for RecordingRunner {
|
||||
fn run(
|
||||
&self,
|
||||
target: &TargetCommand,
|
||||
_session_id: SessionId,
|
||||
_emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<ProcessExit, ControlError> {
|
||||
*self.observed.lock().unwrap() =
|
||||
Some((target.program().to_owned(), target.arguments().to_vec()));
|
||||
Ok(self.exit)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runsc_exec_uses_the_command_as_the_first_post_container_argument() {
|
||||
let runner = Arc::new(RecordingRunner {
|
||||
observed: Mutex::new(None),
|
||||
exit: ProcessExit {
|
||||
exit_code: Some(23),
|
||||
terminating_signal: None,
|
||||
},
|
||||
});
|
||||
let executor = RunscNonInteractiveExecutor {
|
||||
program: "runsc".into(),
|
||||
global_arguments: vec!["--root=/runtime".into()],
|
||||
container: "container-id".into(),
|
||||
runner: runner.clone(),
|
||||
};
|
||||
let session_id = SessionId::new([9; 32]);
|
||||
let exit = executor
|
||||
.execute(
|
||||
&NonInteractiveSessionRequest {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
session_id,
|
||||
argv: vec![b"printf".to_vec(), b"hello".to_vec()],
|
||||
},
|
||||
&mut |_| Ok(()),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(exit.exit_code(), Some(23));
|
||||
assert_eq!(
|
||||
runner.observed.lock().unwrap().as_ref().unwrap(),
|
||||
&(
|
||||
OsString::from("runsc"),
|
||||
["--root=/runtime", "exec", "container-id", "printf", "hello"]
|
||||
.into_iter()
|
||||
.map(OsString::from)
|
||||
.collect()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rustls_refuses_client_leaf_with_server_role() {
|
||||
let anchor = TrustAnchor::create(generation()).unwrap();
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
|
|
@ -159,6 +160,32 @@ impl HostCoordinatorDomainStore {
|
|||
Ok(store)
|
||||
}
|
||||
|
||||
/// Opens the Domain Store from a descriptor-relative file while retaining the Domain lock.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses I/O, database, metadata, row, or Domain identity failures.
|
||||
pub fn open_file(
|
||||
domain_lock: ExclusiveFileLock,
|
||||
database_file: File,
|
||||
expected_domain: &Domain,
|
||||
) -> Result<Self, DomainStoreError> {
|
||||
let is_new = database_file.metadata()?.len() == 0;
|
||||
let database = redb::Database::builder().create_file(database_file)?;
|
||||
let store = Self {
|
||||
database,
|
||||
_domain_lock: domain_lock,
|
||||
};
|
||||
if is_new {
|
||||
store.initialize(expected_domain)?;
|
||||
} else {
|
||||
store.validate_metadata()?;
|
||||
if store.snapshot()?.domain != *expected_domain {
|
||||
return Err(DomainStoreError::DomainMismatch);
|
||||
}
|
||||
}
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
/// Reads and validates every typed row as one consistent redb snapshot.
|
||||
///
|
||||
/// # Errors
|
||||
|
|
|
|||
|
|
@ -134,6 +134,22 @@ fn open_store(directory: &TestDirectory) -> HostCoordinatorDomainStore {
|
|||
HostCoordinatorDomainStore::open(lock, &directory.path("domain.redb"), &domain()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn descriptor_store_open_cannot_be_redirected_by_path_replacement() {
|
||||
let directory = TestDirectory::new("descriptor-open");
|
||||
let database_path = directory.path("domain.redb");
|
||||
let database_file = lock_file(&database_path);
|
||||
let retained_path = directory.path("retained.redb");
|
||||
std::fs::rename(&database_path, &retained_path).unwrap();
|
||||
drop(lock_file(&database_path));
|
||||
let lock = try_exclusive_flock(lock_file(&directory.path("domain.lock"))).unwrap();
|
||||
|
||||
drop(HostCoordinatorDomainStore::open_file(lock, database_file, &domain()).unwrap());
|
||||
|
||||
assert!(Database::open(retained_path).is_ok());
|
||||
assert_eq!(std::fs::metadata(database_path).unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transaction_reopen_and_interrupted_write_preserve_complete_documents() {
|
||||
let directory = TestDirectory::new("transaction");
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
#![doc = "Host-side coordination authority for one Lazaret Domain."]
|
||||
|
||||
pub mod bootstrap;
|
||||
pub mod client_bootstrap;
|
||||
pub mod control;
|
||||
pub mod domain_store;
|
||||
pub mod escalation;
|
||||
|
|
@ -11,3 +12,4 @@ pub mod production_selector;
|
|||
pub mod runtime;
|
||||
pub mod sandbox_init;
|
||||
pub mod state_anchor;
|
||||
pub mod waiter;
|
||||
|
|
|
|||
|
|
@ -293,15 +293,37 @@ where
|
|||
/// # Errors
|
||||
/// Returns the first invalid transition, durable update, or production adapter failure.
|
||||
pub fn bootstrap_new_domain_while_owner_live(
|
||||
&mut self,
|
||||
generation: Generation,
|
||||
session: Session,
|
||||
owner_live: impl FnMut() -> bool,
|
||||
) -> Result<bool, LifecycleError<A::Error>> {
|
||||
self.bootstrap_new_domain_while_owner_live_and_service(
|
||||
generation,
|
||||
session,
|
||||
owner_live,
|
||||
|_| Ok(()),
|
||||
|_| Ok(()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Drives cold start while servicing bounded pre-publication waiters between actions.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the first invalid transition, durable update, adapter, or waiter-service failure.
|
||||
pub fn bootstrap_new_domain_while_owner_live_and_service(
|
||||
&mut self,
|
||||
generation: Generation,
|
||||
session: Session,
|
||||
mut owner_live: impl FnMut() -> bool,
|
||||
mut service_waiters: impl FnMut(&mut Self) -> Result<(), A::Error>,
|
||||
mut withdraw_waiter_admission: impl FnMut(&mut Self) -> Result<(), A::Error>,
|
||||
) -> Result<bool, LifecycleError<A::Error>> {
|
||||
let session_id = session.session_id;
|
||||
self.start_new_domain(generation, session)?;
|
||||
macro_rules! checkpoint {
|
||||
() => {
|
||||
service_waiters(self).map_err(LifecycleError::Adapter)?;
|
||||
if !owner_live() {
|
||||
self.disconnect_session(session_id)?;
|
||||
return Ok(false);
|
||||
|
|
@ -327,6 +349,11 @@ where
|
|||
checkpoint!();
|
||||
self.start_sandbox()?;
|
||||
checkpoint!();
|
||||
withdraw_waiter_admission(self).map_err(LifecycleError::Adapter)?;
|
||||
if !owner_live() {
|
||||
self.disconnect_session(session_id)?;
|
||||
return Ok(false);
|
||||
}
|
||||
self.publish_domain()?;
|
||||
Ok(true)
|
||||
}
|
||||
|
|
@ -750,6 +777,48 @@ where
|
|||
})
|
||||
}
|
||||
|
||||
/// Performs the complete ordered normal teardown when the final session has disconnected.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the first physical or durable teardown failure.
|
||||
pub fn teardown_if_idle(&mut self) -> Result<bool, LifecycleError<A::Error>> {
|
||||
if !self.is_teardown_ready() {
|
||||
return Ok(false);
|
||||
}
|
||||
self.begin_teardown()?;
|
||||
self.stop_sandbox()?;
|
||||
self.stop_sidecar()?;
|
||||
self.unmount_state_anchor()?;
|
||||
self.release_prepared_lower_lease()?;
|
||||
self.graceful_supervisor_exit()?;
|
||||
self.finish_teardown()?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Transfers a published cold-start owner's lifetime edge to exact control admission.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses an unpublished Domain, unknown owner, or durable disconnect/teardown failure.
|
||||
pub fn reconcile_published_bootstrap_owner(
|
||||
&mut self,
|
||||
session_id: SessionId,
|
||||
bootstrap_owner_live: bool,
|
||||
) -> Result<(), LifecycleError<A::Error>> {
|
||||
Self::require(
|
||||
LifecycleAction::DisconnectSession,
|
||||
self.phase == DomainPhase::Ready && self.published,
|
||||
"Domain is not published",
|
||||
)?;
|
||||
let phase = self
|
||||
.session(session_id, LifecycleAction::DisconnectSession)?
|
||||
.session_phase;
|
||||
if phase == SessionPhase::Waiting && !bootstrap_owner_live {
|
||||
self.disconnect_session(session_id)?;
|
||||
self.teardown_if_idle()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Refuses while any session retains Domain interest.
|
||||
pub fn begin_teardown(&mut self) -> Result<(), LifecycleError<A::Error>> {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,14 @@
|
|||
fn main() {
|
||||
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
|
||||
if let Some(result) =
|
||||
lazaret_host_coordinator::client_bootstrap::dispatch_private_client_bootstrap(&arguments)
|
||||
{
|
||||
if let Err(error) = result {
|
||||
eprintln!("Host Coordinator bootstrap failed: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if lazaret_host_coordinator::sandbox_init::dispatch_private_sandbox_init(&arguments) {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use std::cell::RefCell;
|
||||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::fs::File;
|
||||
|
|
@ -28,13 +29,15 @@ use rustix::fs::{
|
|||
AtFlags, FileType, Mode, OFlags, RawDir, ResolveFlags, fchmod, fstat, mkdirat, openat2, statat,
|
||||
unlinkat,
|
||||
};
|
||||
use rustix::io::{FdFlags, fcntl_setfd};
|
||||
use rustix::io::{FdFlags, dup, fcntl_setfd};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::bootstrap::{
|
||||
ProductionBootstrapPlan, ProductionHostCoordinatorLifecycle, production_lifecycle,
|
||||
};
|
||||
use crate::control::RunscNonInteractiveExecutor;
|
||||
use crate::domain_store::HostCoordinatorDomainStore;
|
||||
use crate::waiter::{BootstrapWaitOutcome, BootstrapWaiterFace};
|
||||
|
||||
const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
|
||||
|
||||
|
|
@ -49,6 +52,12 @@ impl fmt::Display for ProductionColdStartError {
|
|||
|
||||
impl std::error::Error for ProductionColdStartError {}
|
||||
|
||||
impl ProductionColdStartError {
|
||||
pub(crate) fn service(error: impl fmt::Display) -> Self {
|
||||
Self(error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn failure(error: impl fmt::Display) -> ProductionColdStartError {
|
||||
ProductionColdStartError(error.to_string())
|
||||
}
|
||||
|
|
@ -74,6 +83,15 @@ pub struct BootstrapOwnerGuard {
|
|||
}
|
||||
|
||||
impl BootstrapOwnerGuard {
|
||||
/// Adopts the private bootstrap entry's pipe standard input.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a standard input that cannot be duplicated or is not a pipe.
|
||||
pub fn from_stdin() -> Result<Self, ProductionColdStartError> {
|
||||
let stdin = std::io::stdin();
|
||||
Self::from_inherited(dup(&stdin).map_err(failure)?)
|
||||
}
|
||||
|
||||
/// Adopts an inherited bootstrap-owner lifetime pipe without PID sampling.
|
||||
///
|
||||
/// # Errors
|
||||
|
|
@ -89,7 +107,11 @@ impl BootstrapOwnerGuard {
|
|||
Ok(Self { descriptor })
|
||||
}
|
||||
|
||||
fn is_live(&self) -> std::io::Result<bool> {
|
||||
/// Observes whether the exact retained bootstrap-owner pipe still has a writer.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a polling failure for the retained descriptor.
|
||||
pub fn is_live(&self) -> std::io::Result<bool> {
|
||||
let mut descriptors = [PollFd::new(&self.descriptor, PollFlags::IN)];
|
||||
poll(&mut descriptors, Some(&Timespec::default()))?;
|
||||
Ok(!descriptors[0]
|
||||
|
|
@ -222,23 +244,76 @@ impl<'a> ProductionColdStart<'a> {
|
|||
where
|
||||
Service: FnOnce(
|
||||
&Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
|
||||
&RefCell<BootstrapWaiterFace>,
|
||||
&BootstrapOwnerGuard,
|
||||
lazaret_core::identity::SessionId,
|
||||
) -> Result<(), ProductionColdStartError>,
|
||||
{
|
||||
let bootstrap_owner_session_id = bootstrap_owner.session_id;
|
||||
let waiters = RefCell::new(
|
||||
BootstrapWaiterFace::bind(
|
||||
self.configuration,
|
||||
self.generation.generation_id,
|
||||
self.invoking_uid,
|
||||
)
|
||||
.map_err(failure)?,
|
||||
);
|
||||
let store = Arc::clone(&self.store);
|
||||
let generation = self.generation.clone();
|
||||
let plan = self.take_plan()?;
|
||||
let mut lifecycle = production_lifecycle(&store, plan).map_err(failure)?;
|
||||
let published = lifecycle
|
||||
.bootstrap_new_domain_while_owner_live(generation, bootstrap_owner, || {
|
||||
owner_guard.is_live().unwrap_or(false)
|
||||
})
|
||||
.map_err(failure)?;
|
||||
if !published {
|
||||
return Err(ProductionColdStartError(
|
||||
"bootstrap owner disconnected before publication".into(),
|
||||
));
|
||||
let bootstrap = lifecycle.bootstrap_new_domain_while_owner_live_and_service(
|
||||
generation,
|
||||
bootstrap_owner,
|
||||
|| owner_guard.is_live().unwrap_or(false),
|
||||
|lifecycle| {
|
||||
waiters
|
||||
.borrow_mut()
|
||||
.admit_pending(lifecycle)
|
||||
.map_err(crate::bootstrap::ProductionLifecycleError::service)
|
||||
},
|
||||
|lifecycle| {
|
||||
let mut waiters = waiters.borrow_mut();
|
||||
waiters
|
||||
.admit_pending(lifecycle)
|
||||
.map_err(crate::bootstrap::ProductionLifecycleError::service)?;
|
||||
waiters
|
||||
.withdraw()
|
||||
.map_err(crate::bootstrap::ProductionLifecycleError::service)
|
||||
},
|
||||
);
|
||||
match bootstrap {
|
||||
Ok(true) => {
|
||||
for session_id in waiters.borrow_mut().notify(BootstrapWaitOutcome::Published) {
|
||||
lifecycle.disconnect_session(session_id).map_err(failure)?;
|
||||
}
|
||||
}
|
||||
Ok(false) => {
|
||||
let mut waiters = waiters.borrow_mut();
|
||||
let withdrawal = waiters.withdraw();
|
||||
let disconnected = waiters.notify(BootstrapWaitOutcome::Failed);
|
||||
withdrawal.map_err(failure)?;
|
||||
drop(waiters);
|
||||
for session_id in disconnected {
|
||||
lifecycle.disconnect_session(session_id).map_err(failure)?;
|
||||
}
|
||||
return Err(ProductionColdStartError(
|
||||
"bootstrap owner disconnected before publication".into(),
|
||||
));
|
||||
}
|
||||
Err(error) => {
|
||||
let mut waiters = waiters.borrow_mut();
|
||||
let _ = waiters.withdraw();
|
||||
let _ = waiters.notify(BootstrapWaitOutcome::Failed);
|
||||
return Err(failure(error));
|
||||
}
|
||||
}
|
||||
service(&Mutex::new(lifecycle))
|
||||
service(
|
||||
&Mutex::new(lifecycle),
|
||||
&waiters,
|
||||
owner_guard,
|
||||
bootstrap_owner_session_id,
|
||||
)
|
||||
}
|
||||
|
||||
fn take_plan(&mut self) -> Result<ProductionBootstrapPlan<'_>, ProductionColdStartError> {
|
||||
|
|
@ -260,6 +335,13 @@ impl<'a> ProductionColdStart<'a> {
|
|||
runtime_root: self.runtime_root.clone(),
|
||||
bundle_digest: self.bundle_digest,
|
||||
invoking_uid: self.invoking_uid,
|
||||
non_interactive_executor: Arc::new(
|
||||
RunscNonInteractiveExecutor::from_run_command(
|
||||
self.sandbox_bundle.command(),
|
||||
self.container_id,
|
||||
)
|
||||
.map_err(failure)?,
|
||||
),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue