Run non-interactive sessions through production coordination
This commit is contained in:
parent
308c5a1b56
commit
71b0266cf2
6 changed files with 3770 additions and 44 deletions
|
|
@ -10,7 +10,18 @@ name = "lazaret"
|
|||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
ed25519-dalek.workspace = true
|
||||
getrandom.workspace = true
|
||||
lazaret-core = { version = "=0.0.0", path = "../lazaret-core" }
|
||||
lazaret-host-coordinator = { version = "=0.0.0", path = "../lazaret-host-coordinator" }
|
||||
lazaret-linux = { version = "=0.0.0", path = "../lazaret-linux" }
|
||||
lazaret-runtime-supervisor = { version = "=0.0.0", path = "../lazaret-runtime-supervisor" }
|
||||
rustix.workspace = true
|
||||
rustls.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "=3.27.0"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
453
crates/lazaret-client/src/lib.rs
Normal file
453
crates/lazaret-client/src/lib.rs
Normal file
|
|
@ -0,0 +1,453 @@
|
|||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Write};
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use ed25519_dalek::SigningKey;
|
||||
use lazaret_core::identity::{ClientId, SessionAnchorId, SessionId};
|
||||
use lazaret_core::records::{
|
||||
ClientEnrollmentResponse, ControlRequest, ControlResponse, NonInteractiveSessionMessage,
|
||||
NonInteractiveSessionRequest, ProtocolPublicationIdentity, ProtocolVersion, SessionBinding,
|
||||
SessionExitOutcome, SessionRegistrationRequest, TlsSubjectId,
|
||||
};
|
||||
use lazaret_host_coordinator::control::{
|
||||
ControlError, PendingClientEnrollment, PublishedControlFace,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub mod production;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ClientCommand {
|
||||
pub profile: String,
|
||||
pub workspace: PathBuf,
|
||||
pub configuration: Option<PathBuf>,
|
||||
pub argv: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ArgumentError(&'static str);
|
||||
|
||||
impl fmt::Display for ArgumentError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ArgumentError {}
|
||||
|
||||
/// Parses `--profile`, optional host paths, and the exact command following `--`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a concise usage error for missing values, unknown options, or a missing command.
|
||||
pub fn parse_command_line(
|
||||
arguments: impl IntoIterator<Item = OsString>,
|
||||
current_workspace: PathBuf,
|
||||
) -> Result<ClientCommand, ArgumentError> {
|
||||
let mut arguments = arguments.into_iter();
|
||||
let mut profile = None;
|
||||
let mut workspace = None;
|
||||
let mut configuration = None;
|
||||
let mut argv = Vec::new();
|
||||
while let Some(argument) = arguments.next() {
|
||||
match argument.to_str() {
|
||||
Some("--profile") => profile = Some(required_utf8(&mut arguments, "--profile")?),
|
||||
Some("--workspace") => {
|
||||
workspace = Some(PathBuf::from(required_value(
|
||||
&mut arguments,
|
||||
"--workspace",
|
||||
)?));
|
||||
}
|
||||
Some("--config") => {
|
||||
configuration = Some(PathBuf::from(required_value(&mut arguments, "--config")?));
|
||||
}
|
||||
Some("--") => {
|
||||
argv.extend(arguments.map(OsString::into_vec));
|
||||
break;
|
||||
}
|
||||
Some(_) | None => return Err(ArgumentError("unknown option before `--`")),
|
||||
}
|
||||
}
|
||||
let profile = profile.ok_or(ArgumentError("`--profile <name>` is required"))?;
|
||||
if argv.is_empty() {
|
||||
return Err(ArgumentError("a command is required after `--`"));
|
||||
}
|
||||
Ok(ClientCommand {
|
||||
profile,
|
||||
workspace: workspace.unwrap_or(current_workspace),
|
||||
configuration,
|
||||
argv,
|
||||
})
|
||||
}
|
||||
|
||||
fn required_value(
|
||||
arguments: &mut impl Iterator<Item = OsString>,
|
||||
option: &'static str,
|
||||
) -> Result<OsString, ArgumentError> {
|
||||
arguments.next().ok_or(match option {
|
||||
"--profile" => ArgumentError("`--profile` requires a value"),
|
||||
"--workspace" => ArgumentError("`--workspace` requires a path"),
|
||||
"--config" => ArgumentError("`--config` requires a path"),
|
||||
_ => ArgumentError("option requires a value"),
|
||||
})
|
||||
}
|
||||
|
||||
fn required_utf8(
|
||||
arguments: &mut impl Iterator<Item = OsString>,
|
||||
option: &'static str,
|
||||
) -> Result<String, ArgumentError> {
|
||||
required_value(arguments, option)?
|
||||
.into_string()
|
||||
.map_err(|_| ArgumentError("profile name must be UTF-8"))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum DomainStatus<E> {
|
||||
Absent,
|
||||
Starting,
|
||||
Ready(E),
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum BootstrapAcquisition {
|
||||
Leader,
|
||||
Waiter,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ClientError<E> {
|
||||
Configuration(String),
|
||||
UnsupportedHost(String),
|
||||
StaleCoordinator,
|
||||
LostConnection,
|
||||
BootstrapOwnerDisconnected,
|
||||
WaitingDomainFailed,
|
||||
EmptyCommand,
|
||||
Backend(E),
|
||||
Io(io::Error),
|
||||
Control(ControlError),
|
||||
Protocol(&'static str),
|
||||
}
|
||||
|
||||
impl<E: fmt::Display> fmt::Display for ClientError<E> {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Configuration(message) => write!(formatter, "configuration is invalid: {message}"),
|
||||
Self::UnsupportedHost(message) => write!(formatter, "host is unsupported: {message}"),
|
||||
Self::StaleCoordinator => formatter.write_str(
|
||||
"the Domain coordinator is stale; remove no files manually and run Lazaret recovery",
|
||||
),
|
||||
Self::LostConnection => formatter.write_str(
|
||||
"the Domain coordinator connection was lost before the command completed",
|
||||
),
|
||||
Self::BootstrapOwnerDisconnected => formatter.write_str(
|
||||
"the Domain bootstrap owner disconnected before publication",
|
||||
),
|
||||
Self::WaitingDomainFailed => formatter.write_str(
|
||||
"the Domain failed while this Client was waiting; recovery is required before retry",
|
||||
),
|
||||
Self::EmptyCommand => formatter.write_str("a command is required after `--`"),
|
||||
Self::Backend(error) => error.fmt(formatter),
|
||||
Self::Io(error) => write!(formatter, "standard stream forwarding failed: {error}"),
|
||||
Self::Control(error) => error.fmt(formatter),
|
||||
Self::Protocol(message) => write!(formatter, "control protocol failed: {message}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: fmt::Debug + fmt::Display> std::error::Error for ClientError<E> {}
|
||||
|
||||
pub trait ClientBackend {
|
||||
type Endpoint;
|
||||
type Error;
|
||||
|
||||
/// # Errors
|
||||
/// Returns actionable host or configuration validation failures.
|
||||
fn validate(&mut self, command: &ClientCommand) -> Result<(), ClientError<Self::Error>>;
|
||||
/// # Errors
|
||||
/// Returns failures while inspecting current Domain coordination state.
|
||||
fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error>;
|
||||
/// # Errors
|
||||
/// Returns failures while attempting exclusive bootstrap leadership.
|
||||
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error>;
|
||||
/// # Errors
|
||||
/// Returns startup failure or loss of the bootstrap owner.
|
||||
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>>;
|
||||
/// # Errors
|
||||
/// Returns wait failure, stale publication, or bootstrap-owner loss.
|
||||
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>>;
|
||||
/// # Errors
|
||||
/// Returns session registration, stream forwarding, runtime, or connection failure.
|
||||
fn execute(
|
||||
&mut self,
|
||||
endpoint: Self::Endpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<Self::Error>>;
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns validation, coordination, bootstrap, connection, or command execution failures.
|
||||
pub fn run<B: ClientBackend>(
|
||||
backend: &mut B,
|
||||
command: &ClientCommand,
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<B::Error>> {
|
||||
if command.argv.is_empty() {
|
||||
return Err(ClientError::EmptyCommand);
|
||||
}
|
||||
backend.validate(command)?;
|
||||
let endpoint = match backend.observe().map_err(ClientError::Backend)? {
|
||||
DomainStatus::Ready(endpoint) => endpoint,
|
||||
DomainStatus::Starting => backend.wait_until_ready()?,
|
||||
DomainStatus::Stale => return Err(ClientError::StaleCoordinator),
|
||||
DomainStatus::Absent => match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
|
||||
BootstrapAcquisition::Leader => match backend.bootstrap() {
|
||||
Ok(endpoint) => endpoint,
|
||||
Err(ClientError::BootstrapOwnerDisconnected) => {
|
||||
return Err(ClientError::BootstrapOwnerDisconnected);
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
},
|
||||
BootstrapAcquisition::Waiter => backend.wait_until_ready()?,
|
||||
},
|
||||
};
|
||||
backend.execute(endpoint, &command.argv, stdout, stderr)
|
||||
}
|
||||
|
||||
pub struct ControlEndpoint {
|
||||
pub socket_path: PathBuf,
|
||||
pub publication: ProtocolPublicationIdentity,
|
||||
socket_authority: Option<File>,
|
||||
waiter_transfer: Mutex<Option<std::os::unix::net::UnixStream>>,
|
||||
}
|
||||
|
||||
impl ControlEndpoint {
|
||||
#[must_use]
|
||||
pub fn new(socket_path: PathBuf, publication: ProtocolPublicationIdentity) -> Self {
|
||||
Self {
|
||||
socket_path,
|
||||
publication,
|
||||
socket_authority: None,
|
||||
waiter_transfer: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn confined(
|
||||
socket_path: PathBuf,
|
||||
publication: ProtocolPublicationIdentity,
|
||||
authority: File,
|
||||
) -> Self {
|
||||
Self {
|
||||
socket_path,
|
||||
publication,
|
||||
socket_authority: Some(authority),
|
||||
waiter_transfer: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn retain_waiter_transfer(mut self, waiter: std::os::unix::net::UnixStream) -> Self {
|
||||
self.waiter_transfer = Mutex::new(Some(waiter));
|
||||
self
|
||||
}
|
||||
|
||||
fn acknowledge_waiter_transfer<E>(&self) -> Result<(), ClientError<E>> {
|
||||
self.waiter_transfer
|
||||
.lock()
|
||||
.map_err(|_| ClientError::Protocol("waiter transfer lock is poisoned"))?
|
||||
.take();
|
||||
let _ = &self.socket_authority;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ClientSessionIdentity {
|
||||
cli_id: ClientId,
|
||||
session_id: SessionId,
|
||||
anchor: SigningKey,
|
||||
}
|
||||
|
||||
impl ClientSessionIdentity {
|
||||
/// Creates one fresh CLI, Session, and host-private Session Anchor identity.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when operating-system randomness is unavailable.
|
||||
pub fn generate<E>() -> Result<Self, ClientError<E>> {
|
||||
Ok(Self {
|
||||
cli_id: ClientId::new(random_bytes()?),
|
||||
session_id: SessionId::new(random_bytes()?),
|
||||
anchor: SigningKey::from_bytes(&random_bytes()?),
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn cli_id(&self) -> ClientId {
|
||||
self.cli_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn session_id(&self) -> SessionId {
|
||||
self.session_id
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn session_anchor_id(&self) -> SessionAnchorId {
|
||||
SessionAnchorId::new(Sha256::digest(self.anchor.verifying_key().as_bytes()).into())
|
||||
}
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns enrollment, registration, framing, stream forwarding, or runtime-loss failures.
|
||||
pub fn execute_over_control_face(
|
||||
endpoint: &ControlEndpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<std::convert::Infallible>> {
|
||||
let identity = ClientSessionIdentity::generate()?;
|
||||
execute_over_control_face_as(endpoint, argv, stdout, stderr, &identity)
|
||||
}
|
||||
|
||||
/// Executes through the exact identity that owns cold-start leadership.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns enrollment, registration, framing, stream forwarding, or runtime-loss failures.
|
||||
pub fn execute_over_control_face_as<E>(
|
||||
endpoint: &ControlEndpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
identity: &ClientSessionIdentity,
|
||||
) -> Result<i32, ClientError<E>> {
|
||||
if argv.is_empty() {
|
||||
return Err(ClientError::EmptyCommand);
|
||||
}
|
||||
let generation = endpoint.publication.generation_identity.clone();
|
||||
let cli_id = identity.cli_id;
|
||||
let session_id = identity.session_id;
|
||||
let pending = PendingClientEnrollment::create(generation.clone(), cli_id)?;
|
||||
let anonymous = PublishedControlFace::client_config(&endpoint.publication, None)?;
|
||||
let mut enrollment =
|
||||
PublishedControlFace::connect(&endpoint.socket_path, anonymous, &endpoint.publication)?;
|
||||
PublishedControlFace::send(
|
||||
&mut enrollment,
|
||||
&ControlRequest::Enroll(pending.request().clone()),
|
||||
)?;
|
||||
let ControlResponse::Enrolled(ClientEnrollmentResponse { credential, .. }) =
|
||||
PublishedControlFace::receive(&mut enrollment)?
|
||||
else {
|
||||
return Err(ClientError::Protocol(
|
||||
"enrollment response was not Enrolled",
|
||||
));
|
||||
};
|
||||
if credential.generation_identity != generation
|
||||
|| credential.subject_id() != TlsSubjectId::ClientCli(cli_id)
|
||||
{
|
||||
return Err(ClientError::Protocol(
|
||||
"enrollment credential identity differs",
|
||||
));
|
||||
}
|
||||
drop(enrollment);
|
||||
|
||||
let binding = SessionBinding {
|
||||
generation_identity: generation,
|
||||
cli_id,
|
||||
session_id,
|
||||
session_anchor_public_key: identity.anchor.verifying_key().to_bytes().to_vec(),
|
||||
};
|
||||
let authenticated = PublishedControlFace::client_config(
|
||||
&endpoint.publication,
|
||||
Some((&credential, pending.private_key())),
|
||||
)?;
|
||||
let mut stream =
|
||||
PublishedControlFace::connect(&endpoint.socket_path, authenticated, &endpoint.publication)?;
|
||||
PublishedControlFace::send(
|
||||
&mut stream,
|
||||
&ControlRequest::Register(SessionRegistrationRequest {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
session_binding: binding.clone(),
|
||||
}),
|
||||
)?;
|
||||
let ControlResponse::Registered(registered) = PublishedControlFace::receive(&mut stream)?
|
||||
else {
|
||||
return Err(ClientError::Protocol(
|
||||
"registration response was not Registered",
|
||||
));
|
||||
};
|
||||
if registered.session_capability.session_binding != binding {
|
||||
return Err(ClientError::Protocol(
|
||||
"registration capability identity differs",
|
||||
));
|
||||
}
|
||||
endpoint.acknowledge_waiter_transfer()?;
|
||||
PublishedControlFace::send(
|
||||
&mut stream,
|
||||
&ControlRequest::Execute(NonInteractiveSessionRequest {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
session_id,
|
||||
argv: argv.to_vec(),
|
||||
}),
|
||||
)?;
|
||||
loop {
|
||||
let message = PublishedControlFace::receive(&mut stream).map_err(|error| match error {
|
||||
ControlError::Io(_) => ClientError::LostConnection,
|
||||
other => ClientError::Control(other),
|
||||
})?;
|
||||
match message {
|
||||
NonInteractiveSessionMessage::Stdout(output) => {
|
||||
require_session(output.session_id, session_id)?;
|
||||
stdout.write_all(&output.bytes)?;
|
||||
stdout.flush()?;
|
||||
}
|
||||
NonInteractiveSessionMessage::Stderr(output) => {
|
||||
require_session(output.session_id, session_id)?;
|
||||
stderr.write_all(&output.bytes)?;
|
||||
stderr.flush()?;
|
||||
}
|
||||
NonInteractiveSessionMessage::Exit(exit) => {
|
||||
require_session(exit.session_id(), session_id)?;
|
||||
return match exit.outcome() {
|
||||
SessionExitOutcome::Exited => exit
|
||||
.exit_code()
|
||||
.ok_or(ClientError::Protocol("Exited omitted its exit code")),
|
||||
SessionExitOutcome::Terminated | SessionExitOutcome::RuntimeLost => {
|
||||
Err(ClientError::LostConnection)
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn random_bytes<E>() -> Result<[u8; 32], ClientError<E>> {
|
||||
let mut bytes = [0_u8; 32];
|
||||
getrandom::fill(&mut bytes)
|
||||
.map_err(|_| ClientError::Protocol("operating-system randomness is unavailable"))?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn require_session<E>(actual: SessionId, expected: SessionId) -> Result<(), ClientError<E>> {
|
||||
if actual == expected {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(ClientError::Protocol("response Session ID differs"))
|
||||
}
|
||||
}
|
||||
|
||||
impl<E> From<io::Error> for ClientError<E> {
|
||||
fn from(error: io::Error) -> Self {
|
||||
Self::Io(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl<E> From<ControlError> for ClientError<E> {
|
||||
fn from(error: ControlError) -> Self {
|
||||
Self::Control(error)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,36 @@
|
|||
fn main() {
|
||||
let _ = lazaret_core::PROTOCOL_NAME;
|
||||
let current_workspace = match std::env::current_dir() {
|
||||
Ok(path) => path,
|
||||
Err(error) => {
|
||||
eprintln!("cannot determine current workspace: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let command =
|
||||
match lazaret_client::parse_command_line(std::env::args_os().skip(1), current_workspace) {
|
||||
Ok(command) => command,
|
||||
Err(error) => {
|
||||
eprintln!("invalid command line: {error}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
let mut backend = match lazaret_client::production::ProductionClientBackend::new() {
|
||||
Ok(backend) => backend,
|
||||
Err(error) => {
|
||||
eprintln!("Lazaret failed: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
match lazaret_client::run(
|
||||
&mut backend,
|
||||
&command,
|
||||
&mut std::io::stdout(),
|
||||
&mut std::io::stderr(),
|
||||
) {
|
||||
Ok(status) => std::process::exit(status),
|
||||
Err(error) => {
|
||||
eprintln!("Lazaret failed: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
650
crates/lazaret-client/src/production.rs
Normal file
650
crates/lazaret-client/src/production.rs
Normal file
|
|
@ -0,0 +1,650 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Write};
|
||||
use std::os::fd::{AsFd, AsRawFd};
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::PathBuf;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use lazaret_core::configuration::{
|
||||
ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
|
||||
};
|
||||
use lazaret_host_coordinator::client_bootstrap::PRIVATE_CLIENT_BOOTSTRAP_ROLE;
|
||||
use lazaret_host_coordinator::control::{ControlError, PublishedControlFace};
|
||||
use lazaret_host_coordinator::waiter::{
|
||||
BOOTSTRAP_WAITER_SOCKET_NAME, BootstrapWaitOutcome, BootstrapWaitRequest, read_frame,
|
||||
write_frame,
|
||||
};
|
||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||
use lazaret_linux::platform::{PlatformObservation, check_platform};
|
||||
use lazaret_linux::process::{BootstrapChild, TargetCommand, child_broker};
|
||||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
use rustix::pipe::{PipeFlags, pipe_with};
|
||||
|
||||
use crate::{
|
||||
BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, ClientSessionIdentity,
|
||||
ControlEndpoint, DomainStatus, execute_over_control_face_as,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ProductionClientError(String);
|
||||
|
||||
impl fmt::Display for ProductionClientError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for ProductionClientError {}
|
||||
|
||||
fn failure(error: impl fmt::Display) -> ProductionClientError {
|
||||
ProductionClientError(error.to_string())
|
||||
}
|
||||
|
||||
pub struct ProductionClientBackend {
|
||||
configuration: Option<ResolvedDomainConfiguration>,
|
||||
command: Option<ClientCommand>,
|
||||
identity: ClientSessionIdentity,
|
||||
coordinator: Option<BootstrapChild>,
|
||||
bootstrap_owner: Option<rustix::fd::OwnedFd>,
|
||||
}
|
||||
|
||||
impl ProductionClientBackend {
|
||||
/// Creates one production client invocation and its host-private session identity.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when operating-system randomness is unavailable.
|
||||
pub fn new() -> Result<Self, ClientError<ProductionClientError>> {
|
||||
Ok(Self {
|
||||
configuration: None,
|
||||
command: None,
|
||||
identity: ClientSessionIdentity::generate()?,
|
||||
coordinator: None,
|
||||
bootstrap_owner: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn configuration(&self) -> Result<&ResolvedDomainConfiguration, ProductionClientError> {
|
||||
self.configuration
|
||||
.as_ref()
|
||||
.ok_or_else(|| failure("client configuration was not validated"))
|
||||
}
|
||||
|
||||
fn discover(&self) -> Result<Discovery, ProductionClientError> {
|
||||
let configuration = self.configuration()?;
|
||||
match PublishedControlFace::discover_confined(
|
||||
configuration.runtime_domain_root(),
|
||||
rustix::process::getuid().as_raw(),
|
||||
) {
|
||||
Ok((authority, socket_path, publication))
|
||||
if publication_matches(configuration, &publication) =>
|
||||
{
|
||||
Ok(Discovery::Ready(Box::new(ControlEndpoint::confined(
|
||||
socket_path,
|
||||
publication,
|
||||
authority,
|
||||
))))
|
||||
}
|
||||
Err(ControlError::Io(error)) if error.kind() == io::ErrorKind::NotFound => {
|
||||
Ok(Discovery::Absent)
|
||||
}
|
||||
Ok(_) | Err(_) => Ok(Discovery::Stale),
|
||||
}
|
||||
}
|
||||
|
||||
fn domain_lock_is_available(&self) -> Result<bool, ProductionClientError> {
|
||||
let configuration = self.configuration()?;
|
||||
let root = configuration
|
||||
.persistent_domain_root()
|
||||
.open_authority()
|
||||
.map_err(failure)?;
|
||||
let file = open_private_domain_lock(&root)?;
|
||||
match try_exclusive_flock(file) {
|
||||
Ok(lock) => {
|
||||
drop(lock);
|
||||
Ok(true)
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(false),
|
||||
Err(error) => Err(failure(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn launch_coordinator(
|
||||
&mut self,
|
||||
command: &ClientCommand,
|
||||
) -> Result<BootstrapAcquisition, ProductionClientError> {
|
||||
let executable = std::env::current_exe().map_err(failure)?;
|
||||
let coordinator = executable
|
||||
.parent()
|
||||
.ok_or_else(|| failure("Lazaret executable has no installation directory"))?
|
||||
.join("lazaret-host-coordinator");
|
||||
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).map_err(failure)?;
|
||||
let mut target = TargetCommand::new(coordinator)
|
||||
.argument(PRIVATE_CLIENT_BOOTSTRAP_ROLE)
|
||||
.argument("--profile")
|
||||
.argument(&command.profile)
|
||||
.argument("--workspace")
|
||||
.argument(command.workspace.as_os_str())
|
||||
.with_lifetime_pipe_parent_edge();
|
||||
if let Some(configuration) = &command.configuration {
|
||||
target = target
|
||||
.argument("--config")
|
||||
.argument(configuration.as_os_str());
|
||||
}
|
||||
target = target
|
||||
.argument("--client-id")
|
||||
.argument(hex(self.identity.cli_id().as_bytes()))
|
||||
.argument("--session-id")
|
||||
.argument(hex(self.identity.session_id().as_bytes()))
|
||||
.argument("--anchor-id")
|
||||
.argument(hex(self.identity.session_anchor_id().as_bytes()))
|
||||
.argument("--domain-id")
|
||||
.argument(hex(self.configuration()?.domain_id().as_bytes()))
|
||||
.argument("--configuration-fingerprint")
|
||||
.argument(hex(self
|
||||
.configuration()?
|
||||
.configuration_fingerprint()
|
||||
.as_bytes()));
|
||||
let mut child = child_broker()
|
||||
.spawn_bootstrap(&target, File::from(reader))
|
||||
.map_err(failure)?;
|
||||
match child.acquisition_status().map_err(failure)? {
|
||||
b'L' => {
|
||||
self.coordinator = Some(child);
|
||||
self.bootstrap_owner = Some(writer);
|
||||
Ok(BootstrapAcquisition::Leader)
|
||||
}
|
||||
b'W' => {
|
||||
child.reap().map_err(failure)?;
|
||||
Ok(BootstrapAcquisition::Waiter)
|
||||
}
|
||||
b'E' => {
|
||||
let result = child.reap().map_err(failure)?;
|
||||
Err(failure(format!(
|
||||
"Host Coordinator refused bootstrap with {result:?}"
|
||||
)))
|
||||
}
|
||||
_ => Err(failure("Host Coordinator sent an invalid bootstrap status")),
|
||||
}
|
||||
}
|
||||
|
||||
fn await_publication(&mut self) -> Result<ControlEndpoint, ClientError<ProductionClientError>> {
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||
Discovery::Absent => {}
|
||||
}
|
||||
if self
|
||||
.coordinator
|
||||
.as_ref()
|
||||
.ok_or_else(|| ClientError::Backend(failure("Host Coordinator is absent")))?
|
||||
.has_exited()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?
|
||||
{
|
||||
let status = self
|
||||
.coordinator
|
||||
.take()
|
||||
.expect("checked coordinator remains present")
|
||||
.reap()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
return Err(ClientError::Backend(failure(format!(
|
||||
"Host Coordinator exited before publication with {status:?}"
|
||||
))));
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ClientBackend for ProductionClientBackend {
|
||||
type Endpoint = ControlEndpoint;
|
||||
type Error = ProductionClientError;
|
||||
|
||||
fn validate(&mut self, command: &ClientCommand) -> Result<(), ClientError<Self::Error>> {
|
||||
let kernel_release = std::fs::read_to_string("/proc/sys/kernel/osrelease")
|
||||
.map_err(|error| ClientError::UnsupportedHost(error.to_string()))?;
|
||||
check_platform(&PlatformObservation {
|
||||
operating_system: std::env::consts::OS,
|
||||
architecture: std::env::consts::ARCH,
|
||||
kernel_release: kernel_release.trim(),
|
||||
})
|
||||
.map_err(|error| ClientError::UnsupportedHost(error.to_string()))?;
|
||||
self.configuration = Some(
|
||||
load_configuration(command)
|
||||
.map_err(|error| ClientError::Configuration(error.to_string()))?,
|
||||
);
|
||||
self.command = Some(command.clone());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error> {
|
||||
match self.discover()? {
|
||||
Discovery::Ready(endpoint) => return Ok(DomainStatus::Ready(*endpoint)),
|
||||
Discovery::Stale => return Ok(DomainStatus::Stale),
|
||||
Discovery::Absent => {}
|
||||
}
|
||||
if self.domain_lock_is_available()? {
|
||||
Ok(DomainStatus::Absent)
|
||||
} else {
|
||||
Ok(DomainStatus::Starting)
|
||||
}
|
||||
}
|
||||
|
||||
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
|
||||
let command = self
|
||||
.command
|
||||
.clone()
|
||||
.ok_or_else(|| failure("client command was not validated"))?;
|
||||
self.launch_coordinator(&command)
|
||||
}
|
||||
|
||||
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
self.await_publication()
|
||||
}
|
||||
|
||||
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
let configuration = self.configuration().map_err(ClientError::Backend)?;
|
||||
let runtime_authority = configuration
|
||||
.runtime_domain_root()
|
||||
.open_authority()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
let socket = PathBuf::from(format!(
|
||||
"/proc/self/fd/{}/{}",
|
||||
runtime_authority.as_raw_fd(),
|
||||
BOOTSTRAP_WAITER_SOCKET_NAME
|
||||
));
|
||||
let request = BootstrapWaitRequest {
|
||||
protocol_version: lazaret_core::records::ProtocolVersion::V1,
|
||||
domain_id: configuration.domain_id(),
|
||||
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||
cli_id: self.identity.cli_id(),
|
||||
session_id: self.identity.session_id(),
|
||||
session_anchor_id: self.identity.session_anchor_id(),
|
||||
};
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||
Discovery::Absent => {}
|
||||
}
|
||||
match std::os::unix::net::UnixStream::connect(&socket) {
|
||||
Ok(mut stream) => {
|
||||
write_frame(&mut stream, &request).map_err(|error| {
|
||||
ClientError::Backend(failure(format!(
|
||||
"bootstrap waiter request failed: {error}"
|
||||
)))
|
||||
})?;
|
||||
let first: BootstrapWaitOutcome = read_frame(&mut stream).map_err(|error| {
|
||||
ClientError::Backend(failure(format!(
|
||||
"bootstrap waiter admission failed: {error}"
|
||||
)))
|
||||
})?;
|
||||
if first != BootstrapWaitOutcome::Waiting {
|
||||
return match first {
|
||||
BootstrapWaitOutcome::Refused => Err(ClientError::StaleCoordinator),
|
||||
BootstrapWaitOutcome::Failed => Err(ClientError::WaitingDomainFailed),
|
||||
BootstrapWaitOutcome::Published => {
|
||||
self.await_ready_publication(Some(stream))
|
||||
}
|
||||
BootstrapWaitOutcome::Waiting => unreachable!(),
|
||||
};
|
||||
}
|
||||
return match read_frame(&mut stream) {
|
||||
Ok(BootstrapWaitOutcome::Published) => {
|
||||
self.await_ready_publication(Some(stream))
|
||||
}
|
||||
Ok(BootstrapWaitOutcome::Failed) | Err(_) => {
|
||||
Err(ClientError::WaitingDomainFailed)
|
||||
}
|
||||
Ok(BootstrapWaitOutcome::Waiting | BootstrapWaitOutcome::Refused) => {
|
||||
Err(ClientError::Protocol(
|
||||
"bootstrap waiter sent an invalid terminal outcome",
|
||||
))
|
||||
}
|
||||
};
|
||||
}
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused
|
||||
) => {}
|
||||
Err(error) => return Err(ClientError::Backend(failure(error))),
|
||||
}
|
||||
if self
|
||||
.domain_lock_is_available()
|
||||
.map_err(ClientError::Backend)?
|
||||
{
|
||||
return Err(ClientError::WaitingDomainFailed);
|
||||
}
|
||||
thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&mut self,
|
||||
endpoint: Self::Endpoint,
|
||||
argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<Self::Error>> {
|
||||
let exit = execute_over_control_face_as(&endpoint, argv, stdout, stderr, &self.identity)?;
|
||||
if let Some(coordinator) = self.coordinator.take() {
|
||||
let status = coordinator
|
||||
.reap()
|
||||
.map_err(failure)
|
||||
.map_err(ClientError::Backend)?;
|
||||
if status.exit_code != Some(0) || status.terminating_signal.is_some() {
|
||||
return Err(ClientError::Backend(failure(format!(
|
||||
"Host Coordinator teardown failed with {status:?}"
|
||||
))));
|
||||
}
|
||||
}
|
||||
self.bootstrap_owner.take();
|
||||
Ok(exit)
|
||||
}
|
||||
}
|
||||
|
||||
impl ProductionClientBackend {
|
||||
fn await_ready_publication(
|
||||
&self,
|
||||
waiter: Option<std::os::unix::net::UnixStream>,
|
||||
) -> Result<ControlEndpoint, ClientError<ProductionClientError>> {
|
||||
loop {
|
||||
match self.discover().map_err(ClientError::Backend)? {
|
||||
Discovery::Ready(endpoint) => {
|
||||
let endpoint = *endpoint;
|
||||
return Ok(match waiter {
|
||||
Some(stream) => endpoint.retain_waiter_transfer(stream),
|
||||
None => endpoint,
|
||||
});
|
||||
}
|
||||
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||
Discovery::Absent => thread::sleep(Duration::from_millis(1)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn open_private_domain_lock(root: &impl AsFd) -> Result<File, ProductionClientError> {
|
||||
let file = File::from(
|
||||
openat(
|
||||
root,
|
||||
"domain.lock",
|
||||
OFlags::RDWR | OFlags::CREATE | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::RUSR | Mode::WUSR,
|
||||
)
|
||||
.map_err(failure)?,
|
||||
);
|
||||
let metadata = file.metadata().map_err(failure)?;
|
||||
if !metadata.file_type().is_file()
|
||||
|| metadata.uid() != rustix::process::getuid().as_raw()
|
||||
|| metadata.mode() & 0o7777 != 0o600
|
||||
|| metadata.nlink() != 1
|
||||
{
|
||||
return Err(failure(
|
||||
"Domain lock is not exact invoking-user mode-0600 authority",
|
||||
));
|
||||
}
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn publication_matches(
|
||||
configuration: &ResolvedDomainConfiguration,
|
||||
publication: &lazaret_core::records::ProtocolPublicationIdentity,
|
||||
) -> bool {
|
||||
publication.generation_identity.domain_id == configuration.domain_id()
|
||||
&& publication.generation_identity.configuration_fingerprint
|
||||
== configuration.configuration_fingerprint()
|
||||
}
|
||||
|
||||
fn load_configuration(
|
||||
command: &ClientCommand,
|
||||
) -> Result<ResolvedDomainConfiguration, Box<dyn std::error::Error>> {
|
||||
Ok(load_trusted_configuration(ConfigurationLoadRequest {
|
||||
explicit_configuration_path: command.configuration.clone(),
|
||||
selected_profile: command.profile.clone(),
|
||||
workspace: command.workspace.clone(),
|
||||
home: required_environment_path("HOME")?,
|
||||
xdg_config_home: optional_environment_path("XDG_CONFIG_HOME"),
|
||||
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
|
||||
xdg_runtime_dir: required_environment_path("XDG_RUNTIME_DIR")?,
|
||||
environment: environment(),
|
||||
sandbox_layout: production_sandbox_layout()?,
|
||||
})?)
|
||||
}
|
||||
|
||||
fn required_environment_path(name: &str) -> Result<PathBuf, ProductionClientError> {
|
||||
std::env::var_os(name)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.ok_or_else(|| failure(format!("{name} is not set")))
|
||||
}
|
||||
|
||||
fn optional_environment_path(name: &str) -> Option<PathBuf> {
|
||||
std::env::var_os(name)
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(PathBuf::from)
|
||||
}
|
||||
|
||||
fn environment() -> BTreeMap<String, OsString> {
|
||||
std::env::vars_os()
|
||||
.filter_map(|(name, value)| name.into_string().ok().map(|name| (name, value)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
use fmt::Write as _;
|
||||
|
||||
bytes.iter().fold(String::new(), |mut output, byte| {
|
||||
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
|
||||
output
|
||||
})
|
||||
}
|
||||
|
||||
enum Discovery {
|
||||
Absent,
|
||||
Ready(Box<ControlEndpoint>),
|
||||
Stale,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
|
||||
use super::*;
|
||||
use lazaret_core::codec::encode;
|
||||
use lazaret_core::identity::{
|
||||
CertificateDigest, CoordinatorId, GenerationId, SandboxId, TrustManifestDigest,
|
||||
};
|
||||
use lazaret_core::records::{GenerationIdentity, ProtocolPublicationIdentity, ProtocolVersion};
|
||||
|
||||
#[test]
|
||||
fn replaced_domain_roots_cannot_redirect_client_lock_or_control_authority() {
|
||||
let persistent_tree = tempfile::Builder::new()
|
||||
.prefix("lz-")
|
||||
.tempdir_in("/tmp")
|
||||
.unwrap();
|
||||
let persistent_configuration = configuration(persistent_tree.path());
|
||||
let persistent_path = path(
|
||||
persistent_configuration
|
||||
.persistent_domain_root()
|
||||
.canonical_label()
|
||||
.as_bytes(),
|
||||
);
|
||||
fs::rename(&persistent_path, persistent_path.with_extension("old")).unwrap();
|
||||
fs::create_dir(&persistent_path).unwrap();
|
||||
fs::set_permissions(&persistent_path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert!(
|
||||
persistent_configuration
|
||||
.persistent_domain_root()
|
||||
.open_authority()
|
||||
.is_err()
|
||||
);
|
||||
assert!(!persistent_path.join("domain.lock").exists());
|
||||
|
||||
let runtime_tree = tempfile::Builder::new()
|
||||
.prefix("lz-")
|
||||
.tempdir_in("/tmp")
|
||||
.unwrap();
|
||||
let runtime_configuration = configuration(runtime_tree.path());
|
||||
let runtime_path = path(
|
||||
runtime_configuration
|
||||
.runtime_domain_root()
|
||||
.canonical_label()
|
||||
.as_bytes(),
|
||||
);
|
||||
fs::rename(&runtime_path, runtime_path.with_extension("old")).unwrap();
|
||||
fs::create_dir(&runtime_path).unwrap();
|
||||
fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert!(
|
||||
PublishedControlFace::discover_confined(
|
||||
runtime_configuration.runtime_domain_root(),
|
||||
rustix::process::getuid().as_raw(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confined_control_discovery_connects_through_the_retained_root() {
|
||||
let tree = tempfile::Builder::new()
|
||||
.prefix("lz-")
|
||||
.tempdir_in("/tmp")
|
||||
.unwrap();
|
||||
let configuration = configuration(tree.path());
|
||||
let runtime_path = path(
|
||||
configuration
|
||||
.runtime_domain_root()
|
||||
.canonical_label()
|
||||
.as_bytes(),
|
||||
);
|
||||
let listener = UnixListener::bind(runtime_path.join("c.sock")).unwrap();
|
||||
fs::set_permissions(
|
||||
runtime_path.join("c.sock"),
|
||||
fs::Permissions::from_mode(0o600),
|
||||
)
|
||||
.unwrap();
|
||||
let publication = ProtocolPublicationIdentity {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
generation_identity: GenerationIdentity {
|
||||
agent_profile: "agent".into(),
|
||||
domain_id: configuration.domain_id(),
|
||||
generation_id: GenerationId::new(1),
|
||||
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||
coordinator_id: CoordinatorId::new([3; 32]),
|
||||
sandbox_id: SandboxId::new([4; 32]),
|
||||
},
|
||||
tls_ca_certificate_der: vec![1],
|
||||
trust_manifest_sha256: TrustManifestDigest::new([5; 32]),
|
||||
coordinator_tls_certificate_sha256: CertificateDigest::new([6; 32]),
|
||||
};
|
||||
let mut mismatch = publication.clone();
|
||||
mismatch.generation_identity.configuration_fingerprint =
|
||||
lazaret_core::identity::ConfigurationFingerprint::new([7; 32]);
|
||||
assert!(publication_matches(&configuration, &publication));
|
||||
assert!(!publication_matches(&configuration, &mismatch));
|
||||
fs::write(
|
||||
runtime_path.join("protocol-identity.postcard"),
|
||||
encode(&publication).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
fs::set_permissions(
|
||||
runtime_path.join("protocol-identity.postcard"),
|
||||
fs::Permissions::from_mode(0o600),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let (authority, socket, observed) = PublishedControlFace::discover_confined(
|
||||
configuration.runtime_domain_root(),
|
||||
rustix::process::getuid().as_raw(),
|
||||
)
|
||||
.unwrap();
|
||||
let client = UnixStream::connect(socket).unwrap();
|
||||
let (server, _) = listener.accept().unwrap();
|
||||
|
||||
assert_eq!(observed, publication);
|
||||
drop((authority, client, server));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiter_transfer_remains_live_until_control_registration_acknowledges_it() {
|
||||
let (client, mut coordinator) = UnixStream::pair().unwrap();
|
||||
coordinator.set_nonblocking(true).unwrap();
|
||||
let endpoint = ControlEndpoint::new(
|
||||
PathBuf::from("unused"),
|
||||
ProtocolPublicationIdentity {
|
||||
protocol_version: ProtocolVersion::V1,
|
||||
generation_identity: GenerationIdentity {
|
||||
agent_profile: "agent".into(),
|
||||
domain_id: lazaret_core::identity::DomainId::new([1; 32]),
|
||||
generation_id: GenerationId::new(1),
|
||||
configuration_fingerprint:
|
||||
lazaret_core::identity::ConfigurationFingerprint::new([2; 32]),
|
||||
coordinator_id: CoordinatorId::new([3; 32]),
|
||||
sandbox_id: SandboxId::new([4; 32]),
|
||||
},
|
||||
tls_ca_certificate_der: vec![1],
|
||||
trust_manifest_sha256: TrustManifestDigest::new([5; 32]),
|
||||
coordinator_tls_certificate_sha256: CertificateDigest::new([6; 32]),
|
||||
},
|
||||
)
|
||||
.retain_waiter_transfer(client);
|
||||
assert_eq!(
|
||||
coordinator.read(&mut [0_u8; 1]).unwrap_err().kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
|
||||
endpoint
|
||||
.acknowledge_waiter_transfer::<std::convert::Infallible>()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(coordinator.read(&mut [0_u8; 1]).unwrap(), 0);
|
||||
}
|
||||
|
||||
fn configuration(root: &std::path::Path) -> ResolvedDomainConfiguration {
|
||||
let configuration_directory = root.join("configuration");
|
||||
let workspace = root.join("workspace");
|
||||
let runtime = root.join("r");
|
||||
fs::create_dir(&configuration_directory).unwrap();
|
||||
fs::create_dir(&workspace).unwrap();
|
||||
fs::create_dir(&runtime).unwrap();
|
||||
let configuration_path = configuration_directory.join("lazaret.toml");
|
||||
fs::write(
|
||||
&configuration_path,
|
||||
format!(
|
||||
"schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.agent]\nimage = \"example.invalid/image:v1\"\n[profiles.agent.readiness]\ndenied = [{{ address = \"10.0.0.1\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
|
||||
root.join("data").display()
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
load_trusted_configuration(ConfigurationLoadRequest {
|
||||
explicit_configuration_path: Some(configuration_path),
|
||||
selected_profile: "agent".into(),
|
||||
workspace,
|
||||
home: root.join("home"),
|
||||
xdg_config_home: None,
|
||||
xdg_data_home: None,
|
||||
xdg_runtime_dir: runtime,
|
||||
environment: BTreeMap::new(),
|
||||
sandbox_layout: production_sandbox_layout().unwrap(),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn path(bytes: &[u8]) -> PathBuf {
|
||||
PathBuf::from(OsString::from_vec(bytes.to_vec()))
|
||||
}
|
||||
}
|
||||
255
crates/lazaret-client/tests/cli_contract.rs
Normal file
255
crates/lazaret-client/tests/cli_contract.rs
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
use std::collections::VecDeque;
|
||||
use std::convert::Infallible;
|
||||
use std::ffi::OsString;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lazaret_client::{
|
||||
BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, DomainStatus,
|
||||
parse_command_line, run,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum Endpoint {
|
||||
Existing,
|
||||
Started,
|
||||
Published,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeBackend {
|
||||
observations: VecDeque<DomainStatus<Endpoint>>,
|
||||
acquisition: Option<BootstrapAcquisition>,
|
||||
bootstrap: Option<Result<Endpoint, ClientError<Infallible>>>,
|
||||
waited: Option<Result<Endpoint, ClientError<Infallible>>>,
|
||||
execution: Option<Result<i32, ClientError<Infallible>>>,
|
||||
actions: Vec<&'static str>,
|
||||
validation: Option<ClientError<Infallible>>,
|
||||
}
|
||||
|
||||
impl ClientBackend for FakeBackend {
|
||||
type Endpoint = Endpoint;
|
||||
type Error = Infallible;
|
||||
|
||||
fn validate(&mut self, _command: &ClientCommand) -> Result<(), ClientError<Self::Error>> {
|
||||
self.actions.push("validate");
|
||||
self.validation.take().map_or(Ok(()), Err)
|
||||
}
|
||||
|
||||
fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error> {
|
||||
self.actions.push("observe");
|
||||
Ok(self.observations.pop_front().unwrap())
|
||||
}
|
||||
|
||||
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
|
||||
self.actions.push("acquire");
|
||||
Ok(self.acquisition.unwrap())
|
||||
}
|
||||
|
||||
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
self.actions.push("bootstrap");
|
||||
self.bootstrap.take().unwrap()
|
||||
}
|
||||
|
||||
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||
self.actions.push("wait");
|
||||
self.waited.take().unwrap()
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&mut self,
|
||||
_endpoint: Self::Endpoint,
|
||||
_argv: &[Vec<u8>],
|
||||
stdout: &mut dyn Write,
|
||||
stderr: &mut dyn Write,
|
||||
) -> Result<i32, ClientError<Self::Error>> {
|
||||
self.actions.push("execute");
|
||||
stdout.write_all(b"out").unwrap();
|
||||
stderr.write_all(b"err").unwrap();
|
||||
self.execution.take().unwrap_or(Ok(0))
|
||||
}
|
||||
}
|
||||
|
||||
fn command() -> ClientCommand {
|
||||
ClientCommand {
|
||||
profile: "agent".into(),
|
||||
workspace: PathBuf::from("/workspace"),
|
||||
configuration: None,
|
||||
argv: vec![b"sh".to_vec(), b"-c".to_vec(), b"exit 0".to_vec()],
|
||||
}
|
||||
}
|
||||
|
||||
fn invoke(backend: &mut FakeBackend) -> Result<(i32, Vec<u8>, Vec<u8>), String> {
|
||||
let mut stdout = Vec::new();
|
||||
let mut stderr = Vec::new();
|
||||
run(backend, &command(), &mut stdout, &mut stderr)
|
||||
.map(|status| (status, stdout, stderr))
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_start_leader_bootstraps_before_forwarding_streams() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Absent].into(),
|
||||
acquisition: Some(BootstrapAcquisition::Leader),
|
||||
bootstrap: Some(Ok(Endpoint::Started)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
invoke(&mut backend),
|
||||
Ok((0, b"out".to_vec(), b"err".to_vec()))
|
||||
);
|
||||
assert_eq!(
|
||||
backend.actions,
|
||||
["validate", "observe", "acquire", "bootstrap", "execute"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_domain_is_joined_without_bootstrap() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(invoke(&mut backend).unwrap().0, 0);
|
||||
assert_eq!(backend.actions, ["validate", "observe", "execute"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_owner_disconnect_fails_that_closed_session() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Absent].into(),
|
||||
acquisition: Some(BootstrapAcquisition::Leader),
|
||||
bootstrap: Some(Err(ClientError::BootstrapOwnerDisconnected)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
invoke(&mut backend)
|
||||
.unwrap_err()
|
||||
.contains("bootstrap owner disconnected")
|
||||
);
|
||||
assert_eq!(
|
||||
backend.actions,
|
||||
["validate", "observe", "acquire", "bootstrap"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_waiter_joins_after_publication() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Starting].into(),
|
||||
waited: Some(Ok(Endpoint::Published)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(invoke(&mut backend).unwrap().0, 0);
|
||||
assert_eq!(backend.actions, ["validate", "observe", "wait", "execute"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distinct_waiter_is_not_restarted_after_owner_loss() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Starting].into(),
|
||||
waited: Some(Err(ClientError::WaitingDomainFailed)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
invoke(&mut backend)
|
||||
.unwrap_err()
|
||||
.contains("failed while this Client was waiting")
|
||||
);
|
||||
assert_eq!(backend.actions, ["validate", "observe", "wait"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nonzero_guest_exit_is_the_cli_status() {
|
||||
let mut backend = FakeBackend {
|
||||
observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
|
||||
execution: Some(Ok(23)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
|
||||
assert_eq!(invoke(&mut backend).unwrap().0, 23);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_and_lost_coordinators_are_actionable_failures() {
|
||||
let mut stale = FakeBackend {
|
||||
observations: [DomainStatus::Stale].into(),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
assert!(invoke(&mut stale).unwrap_err().contains("stale"));
|
||||
|
||||
let mut lost = FakeBackend {
|
||||
observations: [DomainStatus::Ready(Endpoint::Existing)].into(),
|
||||
execution: Some(Err(ClientError::LostConnection)),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
assert!(
|
||||
invoke(&mut lost)
|
||||
.unwrap_err()
|
||||
.contains("connection was lost")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_host_and_configuration_errors_keep_their_cause() {
|
||||
for (error, expected) in [
|
||||
(
|
||||
ClientError::UnsupportedHost("Linux kernel 6.8 is below the required 6.9 floor".into()),
|
||||
"host is unsupported: Linux kernel 6.8 is below the required 6.9 floor",
|
||||
),
|
||||
(
|
||||
ClientError::Configuration("profile `agent` is not configured".into()),
|
||||
"configuration is invalid: profile `agent` is not configured",
|
||||
),
|
||||
] {
|
||||
let mut backend = FakeBackend {
|
||||
validation: Some(error),
|
||||
..FakeBackend::default()
|
||||
};
|
||||
assert_eq!(invoke(&mut backend).unwrap_err(), expected);
|
||||
assert_eq!(backend.actions, ["validate"]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_line_requires_an_explicit_profile_and_preserves_command_bytes() {
|
||||
let parsed = parse_command_line(
|
||||
[
|
||||
"--profile",
|
||||
"agent",
|
||||
"--workspace",
|
||||
"/project",
|
||||
"--config",
|
||||
"/host/config.toml",
|
||||
"--",
|
||||
"tool",
|
||||
"argument",
|
||||
]
|
||||
.map(OsString::from),
|
||||
PathBuf::from("/current"),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(parsed.profile, "agent");
|
||||
assert_eq!(parsed.workspace, PathBuf::from("/project"));
|
||||
assert_eq!(
|
||||
parsed.configuration,
|
||||
Some(PathBuf::from("/host/config.toml"))
|
||||
);
|
||||
assert_eq!(parsed.argv, [b"tool".to_vec(), b"argument".to_vec()]);
|
||||
|
||||
assert!(
|
||||
parse_command_line(
|
||||
["--", "tool"].map(OsString::from),
|
||||
PathBuf::from("/current")
|
||||
)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("profile")
|
||||
);
|
||||
}
|
||||
164
crates/lazaret-client/tests/control_face.rs
Normal file
164
crates/lazaret-client/tests/control_face.rs
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
use std::fs::OpenOptions;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use lazaret_client::{ControlEndpoint, execute_over_control_face};
|
||||
use lazaret_core::identity::{
|
||||
ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, HostPath, NetworkSidecarId,
|
||||
SandboxId,
|
||||
};
|
||||
use lazaret_core::records::{
|
||||
Domain, DurableDecision, Generation, GenerationIdentity, NonInteractiveOutput,
|
||||
NonInteractiveSessionMessage, NonInteractiveSessionRequest, SessionExit,
|
||||
};
|
||||
use lazaret_host_coordinator::control::{
|
||||
ControlError, NonInteractiveSessionExecutor, PublishedControlFace, TrustAnchor,
|
||||
};
|
||||
use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||
|
||||
struct TestDirectory(PathBuf);
|
||||
|
||||
impl TestDirectory {
|
||||
fn new() -> Self {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"lazaret-client-control-{}-{}",
|
||||
std::process::id(),
|
||||
random_suffix()
|
||||
));
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
Self(path)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TestDirectory {
|
||||
fn drop(&mut self) {
|
||||
std::fs::remove_dir_all(&self.0).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
struct Executor;
|
||||
|
||||
impl NonInteractiveSessionExecutor for Executor {
|
||||
fn execute(
|
||||
&self,
|
||||
request: &NonInteractiveSessionRequest,
|
||||
emit: &mut dyn FnMut(NonInteractiveSessionMessage) -> Result<(), ControlError>,
|
||||
) -> Result<SessionExit, ControlError> {
|
||||
assert_eq!(request.argv, [b"tool".to_vec(), b"argument".to_vec()]);
|
||||
emit(NonInteractiveSessionMessage::Stdout(NonInteractiveOutput {
|
||||
session_id: request.session_id,
|
||||
bytes: b"first".to_vec(),
|
||||
}))?;
|
||||
emit(NonInteractiveSessionMessage::Stderr(NonInteractiveOutput {
|
||||
session_id: request.session_id,
|
||||
bytes: b"second".to_vec(),
|
||||
}))?;
|
||||
Ok(SessionExit::exited(request.session_id, 19))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_enrolls_registers_forwards_both_streams_and_returns_guest_status() {
|
||||
let directory = TestDirectory::new();
|
||||
let identity = generation_identity();
|
||||
let store = open_store(&directory, &identity);
|
||||
let face = Arc::new(
|
||||
PublishedControlFace::publish_with_executor(
|
||||
&directory.0,
|
||||
rustix::process::getuid().as_raw(),
|
||||
TrustAnchor::create(identity).unwrap(),
|
||||
store,
|
||||
Arc::new(Executor),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let endpoint = ControlEndpoint::new(
|
||||
face.socket_path().to_path_buf(),
|
||||
face.publication_identity().clone(),
|
||||
);
|
||||
let server = {
|
||||
let face = Arc::clone(&face);
|
||||
std::thread::spawn(move || {
|
||||
face.serve_next().unwrap();
|
||||
face.serve_next().unwrap();
|
||||
})
|
||||
};
|
||||
let mut stdout = Vec::new();
|
||||
let mut stderr = Vec::new();
|
||||
|
||||
let status = execute_over_control_face(
|
||||
&endpoint,
|
||||
&[b"tool".to_vec(), b"argument".to_vec()],
|
||||
&mut stdout,
|
||||
&mut stderr,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
server.join().unwrap();
|
||||
assert_eq!(status, 19);
|
||||
assert_eq!(stdout, b"first");
|
||||
assert_eq!(stderr, b"second");
|
||||
}
|
||||
|
||||
fn generation_identity() -> GenerationIdentity {
|
||||
GenerationIdentity {
|
||||
agent_profile: "trusted".into(),
|
||||
domain_id: DomainId::new([1; 32]),
|
||||
generation_id: GenerationId::new(1),
|
||||
configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
|
||||
coordinator_id: CoordinatorId::new([3; 32]),
|
||||
sandbox_id: SandboxId::new([4; 32]),
|
||||
}
|
||||
}
|
||||
|
||||
fn open_store(
|
||||
directory: &TestDirectory,
|
||||
identity: &GenerationIdentity,
|
||||
) -> Arc<HostCoordinatorDomainStore> {
|
||||
let lock = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(directory.0.join("domain.lock"))
|
||||
.unwrap();
|
||||
let domain = Domain {
|
||||
domain_id: identity.domain_id,
|
||||
profile_name: identity.agent_profile.clone(),
|
||||
canonical_workspace: HostPath::new(b"/workspace/project".to_vec()),
|
||||
};
|
||||
let store = Arc::new(
|
||||
HostCoordinatorDomainStore::open(
|
||||
try_exclusive_flock(lock).unwrap(),
|
||||
&directory.0.join("domain.redb"),
|
||||
&domain,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
for durable_decision in [
|
||||
DurableDecision::None,
|
||||
DurableDecision::Preparing,
|
||||
DurableDecision::Published,
|
||||
] {
|
||||
store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(Generation {
|
||||
domain_id: identity.domain_id,
|
||||
generation_id: identity.generation_id,
|
||||
configuration_fingerprint: identity.configuration_fingerprint,
|
||||
coordinator_id: identity.coordinator_id,
|
||||
network_sidecar_id: NetworkSidecarId::new([5; 32]),
|
||||
sandbox_id: identity.sandbox_id,
|
||||
durable_decision,
|
||||
})])
|
||||
.unwrap();
|
||||
}
|
||||
store
|
||||
}
|
||||
|
||||
fn random_suffix() -> u64 {
|
||||
let mut bytes = [0_u8; 8];
|
||||
getrandom::fill(&mut bytes).unwrap();
|
||||
u64::from_ne_bytes(bytes)
|
||||
}
|
||||
Loading…
Reference in a new issue