Remove phase vocabulary from Lazaret interfaces
This commit is contained in:
parent
3f9b1b3543
commit
e7dd95aea6
32 changed files with 547 additions and 579 deletions
667
crates/lazaret-client/src/coordinator_backend.rs
Normal file
667
crates/lazaret-client/src/coordinator_backend.rs
Normal file
|
|
@ -0,0 +1,667 @@
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::ffi::OsString;
|
||||||
|
use std::fmt;
|
||||||
|
use std::fs::File;
|
||||||
|
use std::io::{self, IsTerminal, Write};
|
||||||
|
use std::os::fd::{AsFd, AsRawFd};
|
||||||
|
use std::os::unix::fs::MetadataExt;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use lazaret_core::configuration::{
|
||||||
|
ConfigurationLoadRequest, ResolvedDomainConfiguration, load_trusted_configuration,
|
||||||
|
};
|
||||||
|
use lazaret_host_coordinator::client_bootstrap::PRIVATE_CLIENT_BOOTSTRAP_ROLE;
|
||||||
|
use lazaret_host_coordinator::control::{ControlError, PublishedControlFace};
|
||||||
|
use lazaret_host_coordinator::waiter::{
|
||||||
|
BOOTSTRAP_WAITER_SOCKET_NAME, BootstrapWaitOutcome, BootstrapWaitRequest, read_frame,
|
||||||
|
write_frame,
|
||||||
|
};
|
||||||
|
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||||
|
use lazaret_linux::platform::{PlatformObservation, check_platform};
|
||||||
|
use lazaret_linux::process::{BootstrapChild, TargetCommand, child_broker};
|
||||||
|
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
|
||||||
|
use rustix::fs::{Mode, OFlags, openat};
|
||||||
|
use rustix::pipe::{PipeFlags, pipe_with};
|
||||||
|
|
||||||
|
use crate::{
|
||||||
|
BootstrapAcquisition, ClientBackend, ClientCommand, ClientError, ClientSessionIdentity,
|
||||||
|
ControlEndpoint, DomainStatus, execute_interactive_over_control_face_as,
|
||||||
|
execute_over_control_face_as,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct CoordinatorClientError(String);
|
||||||
|
|
||||||
|
impl fmt::Display for CoordinatorClientError {
|
||||||
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
formatter.write_str(&self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for CoordinatorClientError {}
|
||||||
|
|
||||||
|
fn failure(error: impl fmt::Display) -> CoordinatorClientError {
|
||||||
|
CoordinatorClientError(error.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct CoordinatorClientBackend {
|
||||||
|
configuration: Option<ResolvedDomainConfiguration>,
|
||||||
|
command: Option<ClientCommand>,
|
||||||
|
identity: ClientSessionIdentity,
|
||||||
|
coordinator: Option<BootstrapChild>,
|
||||||
|
bootstrap_owner: Option<rustix::fd::OwnedFd>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CoordinatorClientBackend {
|
||||||
|
/// Creates one client invocation and its host-private session identity.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Returns an error when operating-system randomness is unavailable.
|
||||||
|
pub fn new() -> Result<Self, ClientError<CoordinatorClientError>> {
|
||||||
|
Ok(Self {
|
||||||
|
configuration: None,
|
||||||
|
command: None,
|
||||||
|
identity: ClientSessionIdentity::generate()?,
|
||||||
|
coordinator: None,
|
||||||
|
bootstrap_owner: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn configuration(&self) -> Result<&ResolvedDomainConfiguration, CoordinatorClientError> {
|
||||||
|
self.configuration
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| failure("client configuration was not validated"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn discover(&self) -> Result<Discovery, CoordinatorClientError> {
|
||||||
|
let configuration = self.configuration()?;
|
||||||
|
match PublishedControlFace::discover_confined(
|
||||||
|
configuration.runtime_domain_root(),
|
||||||
|
rustix::process::getuid().as_raw(),
|
||||||
|
) {
|
||||||
|
Ok((authority, socket_path, publication))
|
||||||
|
if publication_matches(configuration, &publication) =>
|
||||||
|
{
|
||||||
|
if !control_socket_is_live(&socket_path) {
|
||||||
|
return Ok(Discovery::Stale);
|
||||||
|
}
|
||||||
|
Ok(Discovery::Ready(Box::new(ControlEndpoint::confined(
|
||||||
|
socket_path,
|
||||||
|
publication,
|
||||||
|
authority,
|
||||||
|
))))
|
||||||
|
}
|
||||||
|
Err(ControlError::Io(error)) if error.kind() == io::ErrorKind::NotFound => {
|
||||||
|
Ok(Discovery::Absent)
|
||||||
|
}
|
||||||
|
Ok(_) | Err(_) => Ok(Discovery::Stale),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn domain_lock_is_available(&self) -> Result<bool, CoordinatorClientError> {
|
||||||
|
let configuration = self.configuration()?;
|
||||||
|
let root = configuration
|
||||||
|
.persistent_domain_root()
|
||||||
|
.open_authority()
|
||||||
|
.map_err(failure)?;
|
||||||
|
let file = open_private_domain_lock(&root)?;
|
||||||
|
match try_exclusive_flock(file) {
|
||||||
|
Ok(lock) => {
|
||||||
|
drop(lock);
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(false),
|
||||||
|
Err(error) => Err(failure(error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn launch_coordinator(
|
||||||
|
&mut self,
|
||||||
|
command: &ClientCommand,
|
||||||
|
) -> Result<BootstrapAcquisition, CoordinatorClientError> {
|
||||||
|
let executable = std::env::current_exe().map_err(failure)?;
|
||||||
|
let coordinator = executable
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| failure("Lazaret executable has no installation directory"))?
|
||||||
|
.join("lazaret-host-coordinator");
|
||||||
|
let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).map_err(failure)?;
|
||||||
|
let mut target = TargetCommand::new(coordinator)
|
||||||
|
.argument(PRIVATE_CLIENT_BOOTSTRAP_ROLE)
|
||||||
|
.argument("--profile")
|
||||||
|
.argument(&command.profile)
|
||||||
|
.argument("--workspace")
|
||||||
|
.argument(command.workspace.as_os_str())
|
||||||
|
.with_lifetime_pipe_parent_edge();
|
||||||
|
if let Some(configuration) = &command.configuration {
|
||||||
|
target = target
|
||||||
|
.argument("--config")
|
||||||
|
.argument(configuration.as_os_str());
|
||||||
|
}
|
||||||
|
target = target
|
||||||
|
.argument("--client-id")
|
||||||
|
.argument(hex(self.identity.cli_id().as_bytes()))
|
||||||
|
.argument("--session-id")
|
||||||
|
.argument(hex(self.identity.session_id().as_bytes()))
|
||||||
|
.argument("--anchor-id")
|
||||||
|
.argument(hex(self.identity.session_anchor_id().as_bytes()))
|
||||||
|
.argument("--domain-id")
|
||||||
|
.argument(hex(self.configuration()?.domain_id().as_bytes()))
|
||||||
|
.argument("--configuration-fingerprint")
|
||||||
|
.argument(hex(self
|
||||||
|
.configuration()?
|
||||||
|
.configuration_fingerprint()
|
||||||
|
.as_bytes()));
|
||||||
|
let mut child = child_broker()
|
||||||
|
.spawn_bootstrap(&target, File::from(reader))
|
||||||
|
.map_err(failure)?;
|
||||||
|
match child.acquisition_status().map_err(failure)? {
|
||||||
|
b'L' => {
|
||||||
|
self.coordinator = Some(child);
|
||||||
|
self.bootstrap_owner = Some(writer);
|
||||||
|
Ok(BootstrapAcquisition::Leader)
|
||||||
|
}
|
||||||
|
b'W' => {
|
||||||
|
child.reap().map_err(failure)?;
|
||||||
|
Ok(BootstrapAcquisition::Waiter)
|
||||||
|
}
|
||||||
|
b'E' => {
|
||||||
|
let result = child.reap().map_err(failure)?;
|
||||||
|
Err(failure(format!(
|
||||||
|
"Host Coordinator refused bootstrap with {result:?}"
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
_ => Err(failure("Host Coordinator sent an invalid bootstrap status")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn await_publication(
|
||||||
|
&mut self,
|
||||||
|
) -> Result<ControlEndpoint, ClientError<CoordinatorClientError>> {
|
||||||
|
loop {
|
||||||
|
match self.discover().map_err(ClientError::Backend)? {
|
||||||
|
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||||
|
Discovery::Stale | Discovery::Absent => {}
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.coordinator
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| ClientError::Backend(failure("Host Coordinator is absent")))?
|
||||||
|
.has_exited()
|
||||||
|
.map_err(failure)
|
||||||
|
.map_err(ClientError::Backend)?
|
||||||
|
{
|
||||||
|
let status = self
|
||||||
|
.coordinator
|
||||||
|
.take()
|
||||||
|
.expect("checked coordinator remains present")
|
||||||
|
.reap()
|
||||||
|
.map_err(failure)
|
||||||
|
.map_err(ClientError::Backend)?;
|
||||||
|
return Err(ClientError::Backend(failure(format!(
|
||||||
|
"Host Coordinator exited before publication with {status:?}"
|
||||||
|
))));
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClientBackend for CoordinatorClientBackend {
|
||||||
|
type Endpoint = ControlEndpoint;
|
||||||
|
type Error = CoordinatorClientError;
|
||||||
|
|
||||||
|
fn validate(&mut self, command: &ClientCommand) -> Result<(), ClientError<Self::Error>> {
|
||||||
|
let kernel_release = std::fs::read_to_string("/proc/sys/kernel/osrelease")
|
||||||
|
.map_err(|error| ClientError::UnsupportedHost(error.to_string()))?;
|
||||||
|
check_platform(&PlatformObservation {
|
||||||
|
operating_system: std::env::consts::OS,
|
||||||
|
architecture: std::env::consts::ARCH,
|
||||||
|
kernel_release: kernel_release.trim(),
|
||||||
|
})
|
||||||
|
.map_err(|error| ClientError::UnsupportedHost(error.to_string()))?;
|
||||||
|
self.configuration = Some(
|
||||||
|
load_configuration(command)
|
||||||
|
.map_err(|error| ClientError::Configuration(error.to_string()))?,
|
||||||
|
);
|
||||||
|
self.command = Some(command.clone());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn observe(&mut self) -> Result<DomainStatus<Self::Endpoint>, Self::Error> {
|
||||||
|
match self.discover()? {
|
||||||
|
Discovery::Ready(endpoint) => return Ok(DomainStatus::Ready(*endpoint)),
|
||||||
|
Discovery::Stale => return Ok(DomainStatus::Stale),
|
||||||
|
Discovery::Absent => {}
|
||||||
|
}
|
||||||
|
Ok(DomainStatus::Absent)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
|
||||||
|
let command = self
|
||||||
|
.command
|
||||||
|
.clone()
|
||||||
|
.ok_or_else(|| failure("client command was not validated"))?;
|
||||||
|
self.launch_coordinator(&command)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bootstrap(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||||
|
self.await_publication()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_until_ready(&mut self) -> Result<Self::Endpoint, ClientError<Self::Error>> {
|
||||||
|
let configuration = self.configuration().map_err(ClientError::Backend)?;
|
||||||
|
let runtime_authority = configuration
|
||||||
|
.runtime_domain_root()
|
||||||
|
.open_authority()
|
||||||
|
.map_err(failure)
|
||||||
|
.map_err(ClientError::Backend)?;
|
||||||
|
let socket = PathBuf::from(format!(
|
||||||
|
"/proc/self/fd/{}/{}",
|
||||||
|
runtime_authority.as_raw_fd(),
|
||||||
|
BOOTSTRAP_WAITER_SOCKET_NAME
|
||||||
|
));
|
||||||
|
let request = BootstrapWaitRequest {
|
||||||
|
protocol_version: lazaret_core::records::ProtocolVersion::V1,
|
||||||
|
domain_id: configuration.domain_id(),
|
||||||
|
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||||
|
cli_id: self.identity.cli_id(),
|
||||||
|
session_id: self.identity.session_id(),
|
||||||
|
session_anchor_id: self.identity.session_anchor_id(),
|
||||||
|
};
|
||||||
|
loop {
|
||||||
|
match self.discover().map_err(ClientError::Backend)? {
|
||||||
|
Discovery::Ready(endpoint) => return Ok(*endpoint),
|
||||||
|
Discovery::Stale | Discovery::Absent => {}
|
||||||
|
}
|
||||||
|
match std::os::unix::net::UnixStream::connect(&socket) {
|
||||||
|
Ok(mut stream) => {
|
||||||
|
write_frame(&mut stream, &request).map_err(|error| {
|
||||||
|
ClientError::Backend(failure(format!(
|
||||||
|
"bootstrap waiter request failed: {error}"
|
||||||
|
)))
|
||||||
|
})?;
|
||||||
|
let first: BootstrapWaitOutcome = read_frame(&mut stream).map_err(|error| {
|
||||||
|
ClientError::Backend(failure(format!(
|
||||||
|
"bootstrap waiter admission failed: {error}"
|
||||||
|
)))
|
||||||
|
})?;
|
||||||
|
if first != BootstrapWaitOutcome::Waiting {
|
||||||
|
return match first {
|
||||||
|
BootstrapWaitOutcome::Refused => Err(ClientError::StaleCoordinator),
|
||||||
|
BootstrapWaitOutcome::Failed => Err(ClientError::WaitingDomainFailed),
|
||||||
|
BootstrapWaitOutcome::Published => {
|
||||||
|
self.await_ready_publication(Some(stream))
|
||||||
|
}
|
||||||
|
BootstrapWaitOutcome::Waiting => unreachable!(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return match read_frame(&mut stream) {
|
||||||
|
Ok(BootstrapWaitOutcome::Published) => {
|
||||||
|
self.await_ready_publication(Some(stream))
|
||||||
|
}
|
||||||
|
Ok(BootstrapWaitOutcome::Failed) | Err(_) => {
|
||||||
|
Err(ClientError::WaitingDomainFailed)
|
||||||
|
}
|
||||||
|
Ok(BootstrapWaitOutcome::Waiting | BootstrapWaitOutcome::Refused) => {
|
||||||
|
Err(ClientError::Protocol(
|
||||||
|
"bootstrap waiter sent an invalid terminal outcome",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
Err(error)
|
||||||
|
if matches!(
|
||||||
|
error.kind(),
|
||||||
|
io::ErrorKind::NotFound | io::ErrorKind::ConnectionRefused
|
||||||
|
) => {}
|
||||||
|
Err(error) => return Err(ClientError::Backend(failure(error))),
|
||||||
|
}
|
||||||
|
if self
|
||||||
|
.domain_lock_is_available()
|
||||||
|
.map_err(ClientError::Backend)?
|
||||||
|
{
|
||||||
|
return Err(ClientError::WaitingDomainFailed);
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn execute(
|
||||||
|
&mut self,
|
||||||
|
endpoint: Self::Endpoint,
|
||||||
|
argv: &[Vec<u8>],
|
||||||
|
stdout: &mut dyn Write,
|
||||||
|
stderr: &mut dyn Write,
|
||||||
|
) -> Result<i32, ClientError<Self::Error>> {
|
||||||
|
let exit = if std::io::stdin().is_terminal() && std::io::stdout().is_terminal() {
|
||||||
|
let mut terminal = crate::terminal::SystemTerminal::open()?;
|
||||||
|
execute_interactive_over_control_face_as(
|
||||||
|
&endpoint,
|
||||||
|
argv,
|
||||||
|
&mut terminal,
|
||||||
|
&self.identity,
|
||||||
|
)?
|
||||||
|
} else {
|
||||||
|
execute_over_control_face_as(&endpoint, argv, stdout, stderr, &self.identity)?
|
||||||
|
};
|
||||||
|
if let Some(coordinator) = self.coordinator.take() {
|
||||||
|
let status = coordinator
|
||||||
|
.reap()
|
||||||
|
.map_err(failure)
|
||||||
|
.map_err(ClientError::Backend)?;
|
||||||
|
if status.exit_code != Some(0) || status.terminating_signal.is_some() {
|
||||||
|
return Err(ClientError::Backend(failure(format!(
|
||||||
|
"Host Coordinator teardown failed with {status:?}"
|
||||||
|
))));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.bootstrap_owner.take();
|
||||||
|
Ok(exit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CoordinatorClientBackend {
|
||||||
|
fn await_ready_publication(
|
||||||
|
&self,
|
||||||
|
waiter: Option<std::os::unix::net::UnixStream>,
|
||||||
|
) -> Result<ControlEndpoint, ClientError<CoordinatorClientError>> {
|
||||||
|
loop {
|
||||||
|
match self.discover().map_err(ClientError::Backend)? {
|
||||||
|
Discovery::Ready(endpoint) => {
|
||||||
|
let endpoint = *endpoint;
|
||||||
|
return Ok(match waiter {
|
||||||
|
Some(stream) => endpoint.retain_waiter_transfer(stream),
|
||||||
|
None => endpoint,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Discovery::Stale => return Err(ClientError::StaleCoordinator),
|
||||||
|
Discovery::Absent => thread::sleep(Duration::from_millis(1)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn open_private_domain_lock(root: &impl AsFd) -> Result<File, CoordinatorClientError> {
|
||||||
|
let file = File::from(
|
||||||
|
openat(
|
||||||
|
root,
|
||||||
|
"domain.lock",
|
||||||
|
OFlags::RDWR | OFlags::CREATE | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||||
|
Mode::RUSR | Mode::WUSR,
|
||||||
|
)
|
||||||
|
.map_err(failure)?,
|
||||||
|
);
|
||||||
|
let metadata = file.metadata().map_err(failure)?;
|
||||||
|
if !metadata.file_type().is_file()
|
||||||
|
|| metadata.uid() != rustix::process::getuid().as_raw()
|
||||||
|
|| metadata.mode() & 0o7777 != 0o600
|
||||||
|
|| metadata.nlink() != 1
|
||||||
|
{
|
||||||
|
return Err(failure(
|
||||||
|
"Domain lock is not exact invoking-user mode-0600 authority",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn control_socket_is_live(socket_path: &std::path::Path) -> bool {
|
||||||
|
std::os::unix::net::UnixStream::connect(socket_path).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn publication_matches(
|
||||||
|
configuration: &ResolvedDomainConfiguration,
|
||||||
|
publication: &lazaret_core::records::ProtocolPublicationIdentity,
|
||||||
|
) -> bool {
|
||||||
|
publication.generation_identity.domain_id == configuration.domain_id()
|
||||||
|
&& publication.generation_identity.configuration_fingerprint
|
||||||
|
== configuration.configuration_fingerprint()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn load_configuration(
|
||||||
|
command: &ClientCommand,
|
||||||
|
) -> Result<ResolvedDomainConfiguration, Box<dyn std::error::Error>> {
|
||||||
|
Ok(load_trusted_configuration(ConfigurationLoadRequest {
|
||||||
|
explicit_configuration_path: command.configuration.clone(),
|
||||||
|
selected_profile: command.profile.clone(),
|
||||||
|
workspace: command.workspace.clone(),
|
||||||
|
home: required_environment_path("HOME")?,
|
||||||
|
xdg_config_home: optional_environment_path("XDG_CONFIG_HOME"),
|
||||||
|
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
|
||||||
|
xdg_runtime_dir: required_environment_path("XDG_RUNTIME_DIR")?,
|
||||||
|
environment: environment(),
|
||||||
|
sandbox_layout: default_sandbox_layout()?,
|
||||||
|
})?)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn required_environment_path(name: &str) -> Result<PathBuf, CoordinatorClientError> {
|
||||||
|
std::env::var_os(name)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(PathBuf::from)
|
||||||
|
.ok_or_else(|| failure(format!("{name} is not set")))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn optional_environment_path(name: &str) -> Option<PathBuf> {
|
||||||
|
std::env::var_os(name)
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(PathBuf::from)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn environment() -> BTreeMap<String, OsString> {
|
||||||
|
std::env::vars_os()
|
||||||
|
.filter_map(|(name, value)| name.into_string().ok().map(|name| (name, value)))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex(bytes: &[u8]) -> String {
|
||||||
|
use fmt::Write as _;
|
||||||
|
|
||||||
|
bytes.iter().fold(String::new(), |mut output, byte| {
|
||||||
|
write!(output, "{byte:02x}").expect("writing to a String cannot fail");
|
||||||
|
output
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Discovery {
|
||||||
|
Absent,
|
||||||
|
Ready(Box<ControlEndpoint>),
|
||||||
|
Stale,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fs;
|
||||||
|
use std::io::Read;
|
||||||
|
use std::os::unix::ffi::OsStringExt;
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
use std::os::unix::net::{UnixListener, UnixStream};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use lazaret_core::codec::encode;
|
||||||
|
use lazaret_core::identity::{
|
||||||
|
CertificateDigest, CoordinatorId, GenerationId, SandboxId, TrustManifestDigest,
|
||||||
|
};
|
||||||
|
use lazaret_core::records::{GenerationIdentity, ProtocolPublicationIdentity, ProtocolVersion};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn replaced_domain_roots_cannot_redirect_client_lock_or_control_authority() {
|
||||||
|
let persistent_tree = tempfile::Builder::new()
|
||||||
|
.prefix("lz-")
|
||||||
|
.tempdir_in("/tmp")
|
||||||
|
.unwrap();
|
||||||
|
let persistent_configuration = configuration(persistent_tree.path());
|
||||||
|
let persistent_path = path(
|
||||||
|
persistent_configuration
|
||||||
|
.persistent_domain_root()
|
||||||
|
.canonical_label()
|
||||||
|
.as_bytes(),
|
||||||
|
);
|
||||||
|
fs::rename(&persistent_path, persistent_path.with_extension("old")).unwrap();
|
||||||
|
fs::create_dir(&persistent_path).unwrap();
|
||||||
|
fs::set_permissions(&persistent_path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
assert!(
|
||||||
|
persistent_configuration
|
||||||
|
.persistent_domain_root()
|
||||||
|
.open_authority()
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(!persistent_path.join("domain.lock").exists());
|
||||||
|
|
||||||
|
let runtime_tree = tempfile::Builder::new()
|
||||||
|
.prefix("lz-")
|
||||||
|
.tempdir_in("/tmp")
|
||||||
|
.unwrap();
|
||||||
|
let runtime_configuration = configuration(runtime_tree.path());
|
||||||
|
let runtime_path = path(
|
||||||
|
runtime_configuration
|
||||||
|
.runtime_domain_root()
|
||||||
|
.canonical_label()
|
||||||
|
.as_bytes(),
|
||||||
|
);
|
||||||
|
fs::rename(&runtime_path, runtime_path.with_extension("old")).unwrap();
|
||||||
|
fs::create_dir(&runtime_path).unwrap();
|
||||||
|
fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
assert!(
|
||||||
|
PublishedControlFace::discover_confined(
|
||||||
|
runtime_configuration.runtime_domain_root(),
|
||||||
|
rustix::process::getuid().as_raw(),
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn confined_control_discovery_connects_through_the_retained_root() {
|
||||||
|
let tree = tempfile::Builder::new()
|
||||||
|
.prefix("lz-")
|
||||||
|
.tempdir_in("/tmp")
|
||||||
|
.unwrap();
|
||||||
|
let configuration = configuration(tree.path());
|
||||||
|
let runtime_path = path(
|
||||||
|
configuration
|
||||||
|
.runtime_domain_root()
|
||||||
|
.canonical_label()
|
||||||
|
.as_bytes(),
|
||||||
|
);
|
||||||
|
let listener = UnixListener::bind(runtime_path.join("c.sock")).unwrap();
|
||||||
|
fs::set_permissions(
|
||||||
|
runtime_path.join("c.sock"),
|
||||||
|
fs::Permissions::from_mode(0o600),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let publication = ProtocolPublicationIdentity {
|
||||||
|
protocol_version: ProtocolVersion::V1,
|
||||||
|
generation_identity: GenerationIdentity {
|
||||||
|
agent_profile: "agent".into(),
|
||||||
|
domain_id: configuration.domain_id(),
|
||||||
|
generation_id: GenerationId::new(1),
|
||||||
|
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||||
|
coordinator_id: CoordinatorId::new([3; 32]),
|
||||||
|
sandbox_id: SandboxId::new([4; 32]),
|
||||||
|
},
|
||||||
|
tls_ca_certificate_der: vec![1],
|
||||||
|
trust_manifest_sha256: TrustManifestDigest::new([5; 32]),
|
||||||
|
coordinator_tls_certificate_sha256: CertificateDigest::new([6; 32]),
|
||||||
|
};
|
||||||
|
let mut mismatch = publication.clone();
|
||||||
|
mismatch.generation_identity.configuration_fingerprint =
|
||||||
|
lazaret_core::identity::ConfigurationFingerprint::new([7; 32]);
|
||||||
|
assert!(publication_matches(&configuration, &publication));
|
||||||
|
assert!(!publication_matches(&configuration, &mismatch));
|
||||||
|
fs::write(
|
||||||
|
runtime_path.join("protocol-identity.postcard"),
|
||||||
|
encode(&publication).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::set_permissions(
|
||||||
|
runtime_path.join("protocol-identity.postcard"),
|
||||||
|
fs::Permissions::from_mode(0o600),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let (authority, socket, observed) = PublishedControlFace::discover_confined(
|
||||||
|
configuration.runtime_domain_root(),
|
||||||
|
rustix::process::getuid().as_raw(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let client = UnixStream::connect(socket).unwrap();
|
||||||
|
let (server, _) = listener.accept().unwrap();
|
||||||
|
|
||||||
|
assert_eq!(observed, publication);
|
||||||
|
assert!(control_socket_is_live(&runtime_path.join("c.sock")));
|
||||||
|
let (probe, _) = listener.accept().unwrap();
|
||||||
|
drop((authority, client, server, probe, listener));
|
||||||
|
assert!(!control_socket_is_live(&runtime_path.join("c.sock")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn waiter_transfer_remains_live_until_control_registration_acknowledges_it() {
|
||||||
|
let (client, mut coordinator) = UnixStream::pair().unwrap();
|
||||||
|
coordinator.set_nonblocking(true).unwrap();
|
||||||
|
let endpoint = ControlEndpoint::new(
|
||||||
|
PathBuf::from("unused"),
|
||||||
|
ProtocolPublicationIdentity {
|
||||||
|
protocol_version: ProtocolVersion::V1,
|
||||||
|
generation_identity: GenerationIdentity {
|
||||||
|
agent_profile: "agent".into(),
|
||||||
|
domain_id: lazaret_core::identity::DomainId::new([1; 32]),
|
||||||
|
generation_id: GenerationId::new(1),
|
||||||
|
configuration_fingerprint:
|
||||||
|
lazaret_core::identity::ConfigurationFingerprint::new([2; 32]),
|
||||||
|
coordinator_id: CoordinatorId::new([3; 32]),
|
||||||
|
sandbox_id: SandboxId::new([4; 32]),
|
||||||
|
},
|
||||||
|
tls_ca_certificate_der: vec![1],
|
||||||
|
trust_manifest_sha256: TrustManifestDigest::new([5; 32]),
|
||||||
|
coordinator_tls_certificate_sha256: CertificateDigest::new([6; 32]),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.retain_waiter_transfer(client);
|
||||||
|
assert_eq!(
|
||||||
|
coordinator.read(&mut [0_u8; 1]).unwrap_err().kind(),
|
||||||
|
io::ErrorKind::WouldBlock
|
||||||
|
);
|
||||||
|
|
||||||
|
endpoint
|
||||||
|
.acknowledge_waiter_transfer::<std::convert::Infallible>()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(coordinator.read(&mut [0_u8; 1]).unwrap(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn configuration(root: &std::path::Path) -> ResolvedDomainConfiguration {
|
||||||
|
let configuration_directory = root.join("configuration");
|
||||||
|
let workspace = root.join("workspace");
|
||||||
|
let runtime = root.join("r");
|
||||||
|
fs::create_dir(&configuration_directory).unwrap();
|
||||||
|
fs::create_dir(&workspace).unwrap();
|
||||||
|
fs::create_dir(&runtime).unwrap();
|
||||||
|
let configuration_path = configuration_directory.join("lazaret.toml");
|
||||||
|
fs::write(
|
||||||
|
&configuration_path,
|
||||||
|
format!(
|
||||||
|
"schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.agent]\nimage = \"example.invalid/image:v1\"\n[profiles.agent.readiness]\ndenied = [{{ address = \"10.0.0.1\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
|
||||||
|
root.join("data").display()
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
load_trusted_configuration(ConfigurationLoadRequest {
|
||||||
|
explicit_configuration_path: Some(configuration_path),
|
||||||
|
selected_profile: "agent".into(),
|
||||||
|
workspace,
|
||||||
|
home: root.join("home"),
|
||||||
|
xdg_config_home: None,
|
||||||
|
xdg_data_home: None,
|
||||||
|
xdg_runtime_dir: runtime,
|
||||||
|
environment: BTreeMap::new(),
|
||||||
|
sandbox_layout: default_sandbox_layout().unwrap(),
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn path(bytes: &[u8]) -> PathBuf {
|
||||||
|
PathBuf::from(OsString::from_vec(bytes.to_vec()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -22,7 +22,7 @@ use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
use crate::terminal::{HostTerminal, InteractiveSessionTransport, run_interactive_session};
|
use crate::terminal::{HostTerminal, InteractiveSessionTransport, run_interactive_session};
|
||||||
|
|
||||||
pub mod production;
|
pub mod coordinator_backend;
|
||||||
pub mod state;
|
pub mod state;
|
||||||
pub mod terminal;
|
pub mod terminal;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@ fn main() {
|
||||||
std::process::exit(2);
|
std::process::exit(2);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let mut backend = match lazaret_client::production::ProductionClientBackend::new() {
|
let mut backend = match lazaret_client::coordinator_backend::CoordinatorClientBackend::new() {
|
||||||
Ok(backend) => backend,
|
Ok(backend) => backend,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
eprintln!("Lazaret failed: {error}");
|
eprintln!("Lazaret failed: {error}");
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ use rustix::fs::{Mode, OFlags, ResolveFlags, openat2};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
use crate::ClientCommand;
|
use crate::ClientCommand;
|
||||||
use crate::production::{load_configuration, open_private_domain_lock};
|
use crate::coordinator_backend::{load_configuration, open_private_domain_lock};
|
||||||
|
|
||||||
const OUTPUT_SCHEMA: &str = "lazaret-state-output-v1";
|
const OUTPUT_SCHEMA: &str = "lazaret-state-output-v1";
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -128,12 +128,12 @@ impl fmt::Debug for SandboxLayoutV1 {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SandboxLayoutV1 {
|
impl SandboxLayoutV1 {
|
||||||
/// Builds the closed destination set supplied by production sandbox-layout construction.
|
/// Builds the closed destination set supplied by default sandbox-layout construction.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Every destination must be a normalized absolute non-root sandbox path and the set must
|
/// Every destination must be a normalized absolute non-root sandbox path and the set must
|
||||||
/// include `/workspace`.
|
/// include `/workspace`.
|
||||||
pub fn from_production_layout(
|
pub fn from_protected_destinations(
|
||||||
destinations: impl IntoIterator<Item = String>,
|
destinations: impl IntoIterator<Item = String>,
|
||||||
) -> Result<Self, ConfigurationError> {
|
) -> Result<Self, ConfigurationError> {
|
||||||
let mut protected_destinations = BTreeSet::new();
|
let mut protected_destinations = BTreeSet::new();
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
#![doc = "Shared production interfaces for Lazaret processes."]
|
#![doc = "Shared interfaces for Lazaret processes."]
|
||||||
|
|
||||||
pub mod codec;
|
pub mod codec;
|
||||||
pub mod configuration;
|
pub mod configuration;
|
||||||
|
|
|
||||||
|
|
@ -58,7 +58,7 @@ fn write_configuration(tree: &TestTree, body: &str) -> PathBuf {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn layout() -> SandboxLayoutV1 {
|
fn layout() -> SandboxLayoutV1 {
|
||||||
SandboxLayoutV1::from_production_layout([
|
SandboxLayoutV1::from_protected_destinations([
|
||||||
"/workspace".to_owned(),
|
"/workspace".to_owned(),
|
||||||
"/proc".to_owned(),
|
"/proc".to_owned(),
|
||||||
"/sys".to_owned(),
|
"/sys".to_owned(),
|
||||||
|
|
@ -187,14 +187,14 @@ volumes = [{ host = "/source", container = "/root/source", kind = "socket" }]
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sandbox_layout_is_closed_normalized_and_requires_workspace() {
|
fn sandbox_layout_is_closed_normalized_and_requires_workspace() {
|
||||||
assert!(SandboxLayoutV1::from_production_layout(["/proc".into()]).is_err());
|
assert!(SandboxLayoutV1::from_protected_destinations(["/proc".into()]).is_err());
|
||||||
assert!(
|
assert!(
|
||||||
SandboxLayoutV1::from_production_layout(["/workspace".into(), "/workspace".into()])
|
SandboxLayoutV1::from_protected_destinations(["/workspace".into(), "/workspace".into()])
|
||||||
.is_err()
|
.is_err()
|
||||||
);
|
);
|
||||||
for invalid in ["workspace", "/", "/workspace/", "/workspace/../control"] {
|
for invalid in ["workspace", "/", "/workspace/", "/workspace/../control"] {
|
||||||
assert!(
|
assert!(
|
||||||
SandboxLayoutV1::from_production_layout(["/workspace".into(), invalid.into(),])
|
SandboxLayoutV1::from_protected_destinations(["/workspace".into(), invalid.into(),])
|
||||||
.is_err()
|
.is_err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -49,8 +49,8 @@ path = "tests/escalation.rs"
|
||||||
harness = false
|
harness = false
|
||||||
|
|
||||||
[[test]]
|
[[test]]
|
||||||
name = "production"
|
name = "cold-start"
|
||||||
path = "tests/production.rs"
|
path = "tests/cold_start.rs"
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|
|
||||||
|
|
@ -25,9 +25,7 @@ use crate::control::{
|
||||||
TrustAnchor,
|
TrustAnchor,
|
||||||
};
|
};
|
||||||
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
||||||
use crate::escalation::{
|
use crate::escalation::{EscalationService, EscalationServiceHandle, EscalationServiceInputs};
|
||||||
ProductionEscalation, ProductionEscalationHandle, ProductionEscalationInputs,
|
|
||||||
};
|
|
||||||
use crate::git_routing::GitRoutingCoordinator;
|
use crate::git_routing::GitRoutingCoordinator;
|
||||||
use crate::lifecycle::{
|
use crate::lifecycle::{
|
||||||
HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, SessionAdmissionIdentity,
|
HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, SessionAdmissionIdentity,
|
||||||
|
|
@ -42,27 +40,27 @@ use crate::state_anchor::{StateAnchorCoordinator, StateAnchorReleaseAuthority};
|
||||||
use crate::terminal::SessionPromptTransport;
|
use crate::terminal::SessionPromptTransport;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ProductionLifecycleError(String);
|
pub struct LifecycleServiceError(String);
|
||||||
|
|
||||||
impl fmt::Display for ProductionLifecycleError {
|
impl fmt::Display for LifecycleServiceError {
|
||||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
formatter.write_str(&self.0)
|
formatter.write_str(&self.0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::error::Error for ProductionLifecycleError {}
|
impl std::error::Error for LifecycleServiceError {}
|
||||||
|
|
||||||
impl ProductionLifecycleError {
|
impl LifecycleServiceError {
|
||||||
pub(crate) fn service(error: impl fmt::Display) -> Self {
|
pub(crate) fn service(error: impl fmt::Display) -> Self {
|
||||||
Self(error.to_string())
|
Self(error.to_string())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn failure(error: impl fmt::Display) -> ProductionLifecycleError {
|
fn failure(error: impl fmt::Display) -> LifecycleServiceError {
|
||||||
ProductionLifecycleError(error.to_string())
|
LifecycleServiceError(error.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ProductionBootstrapPlan<'a> {
|
pub struct BootstrapPlan<'a> {
|
||||||
pub store: Arc<HostCoordinatorDomainStore>,
|
pub store: Arc<HostCoordinatorDomainStore>,
|
||||||
pub catalog: &'a Catalog,
|
pub catalog: &'a Catalog,
|
||||||
pub configuration: &'a ResolvedDomainConfiguration,
|
pub configuration: &'a ResolvedDomainConfiguration,
|
||||||
|
|
@ -81,24 +79,24 @@ pub struct ProductionBootstrapPlan<'a> {
|
||||||
pub non_interactive_executor: Arc<dyn NonInteractiveSessionExecutor>,
|
pub non_interactive_executor: Arc<dyn NonInteractiveSessionExecutor>,
|
||||||
pub interactive_executor: Arc<dyn InteractiveSessionExecutor>,
|
pub interactive_executor: Arc<dyn InteractiveSessionExecutor>,
|
||||||
pub prompt_transport: Arc<SessionPromptTransport>,
|
pub prompt_transport: Arc<SessionPromptTransport>,
|
||||||
pub(crate) runtime_staging: &'a crate::production::RuntimeStaging,
|
pub(crate) runtime_staging: &'a crate::cold_start::RuntimeStaging,
|
||||||
pub git_routing: Option<ProductionGitRouting>,
|
pub git_routing: Option<BootstrapGitRouting>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ProductionGitRouting {
|
pub struct BootstrapGitRouting {
|
||||||
pub mountpoint: HostPath,
|
pub mountpoint: HostPath,
|
||||||
pub host_root: HostPath,
|
pub host_root: HostPath,
|
||||||
pub isolated_root: HostPath,
|
pub isolated_root: HostPath,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct ProductionLifecycleAdapter<'a> {
|
pub struct CoordinatorLifecycleAdapter<'a> {
|
||||||
plan: ProductionBootstrapPlan<'a>,
|
plan: BootstrapPlan<'a>,
|
||||||
registry: Arc<SessionRegistry>,
|
registry: Arc<SessionRegistry>,
|
||||||
trust_anchor: Option<TrustAnchor>,
|
trust_anchor: Option<TrustAnchor>,
|
||||||
staged_control: Option<StagedControlFace>,
|
staged_control: Option<StagedControlFace>,
|
||||||
published_control: Option<Arc<PublishedControlFace>>,
|
published_control: Option<Arc<PublishedControlFace>>,
|
||||||
escalation: Option<ProductionEscalation<'a>>,
|
escalation: Option<EscalationService<'a>>,
|
||||||
escalation_handle: Option<ProductionEscalationHandle>,
|
escalation_handle: Option<EscalationServiceHandle>,
|
||||||
awaiting_runtime: Option<AwaitingAttachedRunsc>,
|
awaiting_runtime: Option<AwaitingAttachedRunsc>,
|
||||||
runtime: Option<CommittedSandboxRuntime>,
|
runtime: Option<CommittedSandboxRuntime>,
|
||||||
state_anchor_mount: Option<MountIdentity>,
|
state_anchor_mount: Option<MountIdentity>,
|
||||||
|
|
@ -107,43 +105,43 @@ pub struct ProductionLifecycleAdapter<'a> {
|
||||||
state_anchor_absence: Option<StateAnchorAbsenceObservation>,
|
state_anchor_absence: Option<StateAnchorAbsenceObservation>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub type ProductionHostCoordinatorLifecycle<'store, 'resources> = HostCoordinatorLifecycle<
|
pub type CoordinatorLifecycle<'store, 'resources> = HostCoordinatorLifecycle<
|
||||||
'store,
|
'store,
|
||||||
ProductionLifecycleAdapter<'resources>,
|
CoordinatorLifecycleAdapter<'resources>,
|
||||||
ProductionSessionRegistry,
|
CoordinatorSessionRegistry,
|
||||||
>;
|
>;
|
||||||
|
|
||||||
/// Composes the cold-start lifecycle with the exact shared production session registry.
|
/// Composes the cold-start lifecycle with the exact shared session registry.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Refuses a plan whose Domain Store is not the caller-retained lock-owning store.
|
/// Refuses a plan whose Domain Store is not the caller-retained lock-owning store.
|
||||||
pub fn production_lifecycle<'store, 'resources>(
|
pub fn coordinator_lifecycle<'store, 'resources>(
|
||||||
store: &'store Arc<HostCoordinatorDomainStore>,
|
store: &'store Arc<HostCoordinatorDomainStore>,
|
||||||
plan: ProductionBootstrapPlan<'resources>,
|
plan: BootstrapPlan<'resources>,
|
||||||
) -> Result<ProductionHostCoordinatorLifecycle<'store, 'resources>, ProductionLifecycleError> {
|
) -> Result<CoordinatorLifecycle<'store, 'resources>, LifecycleServiceError> {
|
||||||
if !Arc::ptr_eq(store, &plan.store) {
|
if !Arc::ptr_eq(store, &plan.store) {
|
||||||
return Err(ProductionLifecycleError(
|
return Err(LifecycleServiceError(
|
||||||
"production bootstrap Domain Store authority differs".into(),
|
"bootstrap Domain Store authority differs".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let (adapter, registry) = ProductionLifecycleAdapter::new(plan);
|
let (adapter, registry) = CoordinatorLifecycleAdapter::new(plan);
|
||||||
Ok(HostCoordinatorLifecycle::new(store, adapter, registry))
|
Ok(HostCoordinatorLifecycle::new(store, adapter, registry))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct ProductionSessionRegistry {
|
pub struct CoordinatorSessionRegistry {
|
||||||
registry: Arc<SessionRegistry>,
|
registry: Arc<SessionRegistry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProductionSessionRegistry {
|
impl CoordinatorSessionRegistry {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn registry(&self) -> Arc<SessionRegistry> {
|
pub fn registry(&self) -> Arc<SessionRegistry> {
|
||||||
Arc::clone(&self.registry)
|
Arc::clone(&self.registry)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SessionLifecycleAdapter for ProductionSessionRegistry {
|
impl SessionLifecycleAdapter for CoordinatorSessionRegistry {
|
||||||
type Error = ProductionLifecycleError;
|
type Error = LifecycleServiceError;
|
||||||
type Admission = ControlSessionAdmission;
|
type Admission = ControlSessionAdmission;
|
||||||
|
|
||||||
fn admission_identity(&self, admission: &Self::Admission) -> SessionAdmissionIdentity {
|
fn admission_identity(&self, admission: &Self::Admission) -> SessionAdmissionIdentity {
|
||||||
|
|
@ -182,9 +180,9 @@ impl SessionLifecycleAdapter for ProductionSessionRegistry {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> ProductionLifecycleAdapter<'a> {
|
impl<'a> CoordinatorLifecycleAdapter<'a> {
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn new(plan: ProductionBootstrapPlan<'a>) -> (Self, ProductionSessionRegistry) {
|
pub fn new(plan: BootstrapPlan<'a>) -> (Self, CoordinatorSessionRegistry) {
|
||||||
let registry = Arc::new(SessionRegistry::new(
|
let registry = Arc::new(SessionRegistry::new(
|
||||||
Arc::clone(&plan.store),
|
Arc::clone(&plan.store),
|
||||||
plan.generation_identity.clone(),
|
plan.generation_identity.clone(),
|
||||||
|
|
@ -205,7 +203,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
state_anchor_release: None,
|
state_anchor_release: None,
|
||||||
state_anchor_absence: None,
|
state_anchor_absence: None,
|
||||||
},
|
},
|
||||||
ProductionSessionRegistry { registry },
|
CoordinatorSessionRegistry { registry },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -216,11 +214,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
|
|
||||||
pub(crate) fn take_escalation(
|
pub(crate) fn take_escalation(
|
||||||
&mut self,
|
&mut self,
|
||||||
) -> Option<(ProductionEscalation<'a>, ProductionEscalationHandle)> {
|
) -> Option<(EscalationService<'a>, EscalationServiceHandle)> {
|
||||||
Some((self.escalation.take()?, self.escalation_handle.take()?))
|
Some((self.escalation.take()?, self.escalation_handle.take()?))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn require_generation(&self, generation: &Generation) -> Result<(), ProductionLifecycleError> {
|
fn require_generation(&self, generation: &Generation) -> Result<(), LifecycleServiceError> {
|
||||||
if generation.domain_id == self.plan.generation_identity.domain_id
|
if generation.domain_id == self.plan.generation_identity.domain_id
|
||||||
&& generation.generation_id == self.plan.generation_identity.generation_id
|
&& generation.generation_id == self.plan.generation_identity.generation_id
|
||||||
&& generation.configuration_fingerprint
|
&& generation.configuration_fingerprint
|
||||||
|
|
@ -230,8 +228,8 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
{
|
{
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
Err(ProductionLifecycleError(
|
Err(LifecycleServiceError(
|
||||||
"production bootstrap generation identity changed".into(),
|
"bootstrap generation identity changed".into(),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -252,18 +250,16 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
StateAnchorCoordinator::new(self.store(), self.plan.catalog)
|
StateAnchorCoordinator::new(self.store(), self.plan.catalog)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn create_trust_anchor(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn create_trust_anchor(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
if self.trust_anchor.is_some() {
|
if self.trust_anchor.is_some() {
|
||||||
return Err(ProductionLifecycleError(
|
return Err(LifecycleServiceError("Trust Anchor already exists".into()));
|
||||||
"Trust Anchor already exists".into(),
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
self.trust_anchor =
|
self.trust_anchor =
|
||||||
Some(TrustAnchor::create(self.plan.generation_identity.clone()).map_err(failure)?);
|
Some(TrustAnchor::create(self.plan.generation_identity.clone()).map_err(failure)?);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn start_runtime_supervisor(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn start_runtime_supervisor(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let sandbox = self.sandbox();
|
let sandbox = self.sandbox();
|
||||||
sandbox
|
sandbox
|
||||||
.begin_intent(
|
.begin_intent(
|
||||||
|
|
@ -286,7 +282,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn lease_prepared_lower(&self) -> Result<(), ProductionLifecycleError> {
|
fn lease_prepared_lower(&self) -> Result<(), LifecycleServiceError> {
|
||||||
self.state_anchor()
|
self.state_anchor()
|
||||||
.begin_acquisition(
|
.begin_acquisition(
|
||||||
self.plan.generation_identity.generation_id,
|
self.plan.generation_identity.generation_id,
|
||||||
|
|
@ -297,13 +293,13 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
.map_err(failure)
|
.map_err(failure)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mount_state_anchor(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn mount_state_anchor(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let store = Arc::clone(&self.plan.store);
|
let store = Arc::clone(&self.plan.store);
|
||||||
let coordinator = StateAnchorCoordinator::new(&store, self.plan.catalog);
|
let coordinator = StateAnchorCoordinator::new(&store, self.plan.catalog);
|
||||||
let mount = self
|
let mount = self
|
||||||
.awaiting_runtime
|
.awaiting_runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
|
||||||
.mount_state_anchor(
|
.mount_state_anchor(
|
||||||
&coordinator,
|
&coordinator,
|
||||||
self.plan.prepared_artifact.prepared_artifact_id,
|
self.plan.prepared_artifact.prepared_artifact_id,
|
||||||
|
|
@ -323,7 +319,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
let git_mount = self
|
let git_mount = self
|
||||||
.awaiting_runtime
|
.awaiting_runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
|
||||||
.start_git_routing(&git_routing)
|
.start_git_routing(&git_routing)
|
||||||
.map_err(|error| failure(format!("starting Git routing: {error}")))?;
|
.map_err(|error| failure(format!("starting Git routing: {error}")))?;
|
||||||
self.git_routing_mount = Some(git_mount);
|
self.git_routing_mount = Some(git_mount);
|
||||||
|
|
@ -331,32 +327,32 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn start_connectivity_helper(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn start_connectivity_helper(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let store = Arc::clone(&self.plan.store);
|
let store = Arc::clone(&self.plan.store);
|
||||||
let network = NetworkCoordinator::new(&store);
|
let network = NetworkCoordinator::new(&store);
|
||||||
self.awaiting_runtime
|
self.awaiting_runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
|
||||||
.receive_helper_staged(&network)
|
.receive_helper_staged(&network)
|
||||||
.map(|_| ())
|
.map(|_| ())
|
||||||
.map_err(|error| failure(format!("starting connectivity helper: {error}")))
|
.map_err(|error| failure(format!("starting connectivity helper: {error}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn start_sidecar(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn start_sidecar(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
self.awaiting_runtime
|
self.awaiting_runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
|
||||||
.start_sidecar()
|
.start_sidecar()
|
||||||
.map_err(|error| failure(format!("starting sidecar: {error}")))
|
.map_err(|error| failure(format!("starting sidecar: {error}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn mark_network_ready(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn mark_network_ready(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let store = Arc::clone(&self.plan.store);
|
let store = Arc::clone(&self.plan.store);
|
||||||
let network = NetworkCoordinator::new(&store);
|
let network = NetworkCoordinator::new(&store);
|
||||||
let mut probe = self.plan.network_probe.clone();
|
let mut probe = self.plan.network_probe.clone();
|
||||||
self.awaiting_runtime
|
self.awaiting_runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
|
||||||
.receive_network_readiness(
|
.receive_network_readiness(
|
||||||
&network,
|
&network,
|
||||||
self.plan.network_policy,
|
self.plan.network_policy,
|
||||||
|
|
@ -370,13 +366,13 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
.map_err(|error| failure(format!("marking network ready: {error}")))
|
.map_err(|error| failure(format!("marking network ready: {error}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn start_sandbox(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn start_sandbox(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let store = Arc::clone(&self.plan.store);
|
let store = Arc::clone(&self.plan.store);
|
||||||
let sandbox = SandboxCoordinator::new(&store);
|
let sandbox = SandboxCoordinator::new(&store);
|
||||||
let mut pending = self
|
let mut pending = self
|
||||||
.awaiting_runtime
|
.awaiting_runtime
|
||||||
.take()
|
.take()
|
||||||
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?;
|
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?;
|
||||||
let staged = pending
|
let staged = pending
|
||||||
.receive_staged(&sandbox)
|
.receive_staged(&sandbox)
|
||||||
.map_err(|error| failure(format!("receiving attached runsc staging: {error}")))?;
|
.map_err(|error| failure(format!("receiving attached runsc staging: {error}")))?;
|
||||||
|
|
@ -394,11 +390,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stage_publication(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn stage_publication(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let mut anchor = self
|
let mut anchor = self
|
||||||
.trust_anchor
|
.trust_anchor
|
||||||
.take()
|
.take()
|
||||||
.ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?;
|
.ok_or_else(|| LifecycleServiceError("Trust Anchor is absent".into()))?;
|
||||||
if self
|
if self
|
||||||
.plan
|
.plan
|
||||||
.configuration
|
.configuration
|
||||||
|
|
@ -421,7 +417,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
lazaret_core::identity::RequestId::new([0; 32]),
|
lazaret_core::identity::RequestId::new([0; 32]),
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
let (escalation, handle) = ProductionEscalation::new(ProductionEscalationInputs {
|
let (escalation, handle) = EscalationService::new(EscalationServiceInputs {
|
||||||
configuration: self.plan.configuration,
|
configuration: self.plan.configuration,
|
||||||
publication,
|
publication,
|
||||||
credentials,
|
credentials,
|
||||||
|
|
@ -476,11 +472,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn expose_publication(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn expose_publication(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let published = self
|
let published = self
|
||||||
.staged_control
|
.staged_control
|
||||||
.take()
|
.take()
|
||||||
.ok_or_else(|| ProductionLifecycleError("staged control face is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("staged control face is absent".into()))?
|
||||||
.expose()
|
.expose()
|
||||||
.map_err(failure)?;
|
.map_err(failure)?;
|
||||||
let ownership = published.recovery_ownership().map_err(failure)?;
|
let ownership = published.recovery_ownership().map_err(failure)?;
|
||||||
|
|
@ -500,7 +496,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn clean_publication_after_withdrawal(&self) -> Result<(), ProductionLifecycleError> {
|
fn clean_publication_after_withdrawal(&self) -> Result<(), LifecycleServiceError> {
|
||||||
let generation_id = self.plan.generation_identity.generation_id;
|
let generation_id = self.plan.generation_identity.generation_id;
|
||||||
let current = self
|
let current = self
|
||||||
.plan
|
.plan
|
||||||
|
|
@ -513,7 +509,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
participant.generation_id() == generation_id
|
participant.generation_id() == generation_id
|
||||||
&& participant.role() == ParticipantRole::Publication
|
&& participant.role() == ParticipantRole::Publication
|
||||||
})
|
})
|
||||||
.ok_or_else(|| ProductionLifecycleError("Publication participant is absent".into()))?;
|
.ok_or_else(|| LifecycleServiceError("Publication participant is absent".into()))?;
|
||||||
let release = Participant::snapshot(
|
let release = Participant::snapshot(
|
||||||
generation_id,
|
generation_id,
|
||||||
ParticipantRole::Publication,
|
ParticipantRole::Publication,
|
||||||
|
|
@ -537,19 +533,19 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
.map_err(failure)
|
.map_err(failure)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stop_sandbox(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn stop_sandbox(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let store = Arc::clone(&self.plan.store);
|
let store = Arc::clone(&self.plan.store);
|
||||||
let sandbox = SandboxCoordinator::new(&store);
|
let sandbox = SandboxCoordinator::new(&store);
|
||||||
self.runtime
|
self.runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
|
||||||
.stop_sandbox(&sandbox)
|
.stop_sandbox(&sandbox)
|
||||||
.map_err(failure)?;
|
.map_err(failure)?;
|
||||||
if self.plan.git_routing.is_some() {
|
if self.plan.git_routing.is_some() {
|
||||||
let git_routing = GitRoutingCoordinator::new(&store);
|
let git_routing = GitRoutingCoordinator::new(&store);
|
||||||
self.runtime
|
self.runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
|
||||||
.stop_git_routing(&git_routing)
|
.stop_git_routing(&git_routing)
|
||||||
.map_err(failure)?;
|
.map_err(failure)?;
|
||||||
self.git_routing_mount = None;
|
self.git_routing_mount = None;
|
||||||
|
|
@ -557,7 +553,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
self.plan.runtime_staging.cleanup_overlay().map_err(failure)
|
self.plan.runtime_staging.cleanup_overlay().map_err(failure)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn stop_sidecar(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn stop_sidecar(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let store = Arc::clone(&self.plan.store);
|
let store = Arc::clone(&self.plan.store);
|
||||||
let network = NetworkCoordinator::new(&store);
|
let network = NetworkCoordinator::new(&store);
|
||||||
network
|
network
|
||||||
|
|
@ -565,7 +561,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
.map_err(failure)?;
|
.map_err(failure)?;
|
||||||
self.runtime
|
self.runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
|
||||||
.stop_sidecar()
|
.stop_sidecar()
|
||||||
.map_err(failure)?;
|
.map_err(failure)?;
|
||||||
network
|
network
|
||||||
|
|
@ -573,7 +569,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
.map_err(failure)
|
.map_err(failure)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn unmount_state_anchor(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn unmount_state_anchor(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let coordinator = self.state_anchor();
|
let coordinator = self.state_anchor();
|
||||||
self.state_anchor_release = Some(
|
self.state_anchor_release = Some(
|
||||||
coordinator
|
coordinator
|
||||||
|
|
@ -586,11 +582,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
let runtime = self
|
let runtime = self
|
||||||
.runtime
|
.runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?;
|
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?;
|
||||||
let release = self
|
let release = self
|
||||||
.state_anchor_release
|
.state_anchor_release
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.ok_or_else(|| ProductionLifecycleError("State Anchor release is absent".into()))?;
|
.ok_or_else(|| LifecycleServiceError("State Anchor release is absent".into()))?;
|
||||||
runtime
|
runtime
|
||||||
.unmount_state_anchor(release.mount())
|
.unmount_state_anchor(release.mount())
|
||||||
.map_err(failure)?;
|
.map_err(failure)?;
|
||||||
|
|
@ -607,18 +603,18 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn release_prepared_lower(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn release_prepared_lower(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
let release = self
|
let release = self
|
||||||
.state_anchor_release
|
.state_anchor_release
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.ok_or_else(|| ProductionLifecycleError("State Anchor release is absent".into()))?;
|
.ok_or_else(|| LifecycleServiceError("State Anchor release is absent".into()))?;
|
||||||
let first_observation = self.state_anchor_absence.ok_or_else(|| {
|
let first_observation = self.state_anchor_absence.ok_or_else(|| {
|
||||||
ProductionLifecycleError("State Anchor absence evidence is absent".into())
|
LifecycleServiceError("State Anchor absence evidence is absent".into())
|
||||||
})?;
|
})?;
|
||||||
let runtime = self
|
let runtime = self
|
||||||
.runtime
|
.runtime
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?;
|
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?;
|
||||||
let coordinator = StateAnchorCoordinator::new(&self.plan.store, self.plan.catalog);
|
let coordinator = StateAnchorCoordinator::new(&self.plan.store, self.plan.catalog);
|
||||||
coordinator
|
coordinator
|
||||||
.complete_release_after_unmount(
|
.complete_release_after_unmount(
|
||||||
|
|
@ -650,17 +646,17 @@ impl<'a> ProductionLifecycleAdapter<'a> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn graceful_exit(&mut self) -> Result<(), ProductionLifecycleError> {
|
fn graceful_exit(&mut self) -> Result<(), LifecycleServiceError> {
|
||||||
self.runtime
|
self.runtime
|
||||||
.take()
|
.take()
|
||||||
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
|
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
|
||||||
.graceful_exit()
|
.graceful_exit()
|
||||||
.map_err(failure)
|
.map_err(failure)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LifecycleAdapter for ProductionLifecycleAdapter<'_> {
|
impl LifecycleAdapter for CoordinatorLifecycleAdapter<'_> {
|
||||||
type Error = ProductionLifecycleError;
|
type Error = LifecycleServiceError;
|
||||||
|
|
||||||
fn execute(
|
fn execute(
|
||||||
&mut self,
|
&mut self,
|
||||||
|
|
@ -721,13 +717,11 @@ impl LifecycleAdapter for ProductionLifecycleAdapter<'_> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Serves one production control connection through the exact retained lifecycle owner.
|
/// Serves one control connection through the exact retained lifecycle owner.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Refuses absent publication, poisoned lifecycle ownership, or control/session failure.
|
/// Refuses absent publication, poisoned lifecycle ownership, or control/session failure.
|
||||||
pub fn serve_production_control(
|
pub fn serve_control(lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>) -> Result<(), ControlError> {
|
||||||
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
|
|
||||||
) -> Result<(), ControlError> {
|
|
||||||
let face = lifecycle
|
let face = lifecycle
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| ControlError::LockPoisoned)?
|
.map_err(|_| ControlError::LockPoisoned)?
|
||||||
|
|
@ -735,22 +729,22 @@ pub fn serve_production_control(
|
||||||
.published_control()
|
.published_control()
|
||||||
.ok_or(ControlError::StaleGeneration)?;
|
.ok_or(ControlError::StaleGeneration)?;
|
||||||
let connection = face.accept_connection()?;
|
let connection = face.accept_connection()?;
|
||||||
serve_production_control_connection(lifecycle, &face, connection)
|
serve_control_connection(lifecycle, &face, connection)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn serve_production_control_connection(
|
pub(crate) fn serve_control_connection(
|
||||||
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
|
lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>,
|
||||||
face: &PublishedControlFace,
|
face: &PublishedControlFace,
|
||||||
connection: AcceptedControlConnection,
|
connection: AcceptedControlConnection,
|
||||||
) -> Result<(), ControlError> {
|
) -> Result<(), ControlError> {
|
||||||
serve_production_control_connection_with_escalation(lifecycle, face, connection, None)
|
serve_control_connection_with_escalation(lifecycle, face, connection, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn serve_production_control_connection_with_escalation(
|
pub(crate) fn serve_control_connection_with_escalation(
|
||||||
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
|
lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>,
|
||||||
face: &PublishedControlFace,
|
face: &PublishedControlFace,
|
||||||
connection: AcceptedControlConnection,
|
connection: AcceptedControlConnection,
|
||||||
escalation: Option<&ProductionEscalationHandle>,
|
escalation: Option<&EscalationServiceHandle>,
|
||||||
) -> Result<(), ControlError> {
|
) -> Result<(), ControlError> {
|
||||||
face.serve_connection_coordinated(
|
face.serve_connection_coordinated(
|
||||||
connection,
|
connection,
|
||||||
|
|
|
||||||
|
|
@ -17,17 +17,15 @@ use lazaret_core::identity::{
|
||||||
};
|
};
|
||||||
use lazaret_core::records::{Domain, Session, SessionPhase};
|
use lazaret_core::records::{Domain, Session, SessionPhase};
|
||||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
|
||||||
use rustix::fs::{Mode, OFlags, openat};
|
use rustix::fs::{Mode, OFlags, openat};
|
||||||
|
|
||||||
use crate::bootstrap::{
|
use crate::bootstrap::{CoordinatorLifecycle, serve_control_connection_with_escalation};
|
||||||
ProductionHostCoordinatorLifecycle, serve_production_control_connection_with_escalation,
|
use crate::cold_start::{BootstrapOwnerGuard, ColdStart, ColdStartError};
|
||||||
};
|
|
||||||
use crate::control::ControlError;
|
use crate::control::ControlError;
|
||||||
use crate::domain_store::HostCoordinatorDomainStore;
|
use crate::domain_store::HostCoordinatorDomainStore;
|
||||||
use crate::escalation::{ProductionEscalation, ProductionEscalationHandle};
|
use crate::escalation::{EscalationService, EscalationServiceHandle};
|
||||||
use crate::production::{BootstrapOwnerGuard, ProductionColdStart, ProductionColdStartError};
|
use crate::runtime_selector::{InstalledProductPaths, select_runtime};
|
||||||
use crate::production_selector::{InstalledProductPaths, select_production_cold_start};
|
|
||||||
use crate::waiter::BootstrapWaiterFace;
|
use crate::waiter::BootstrapWaiterFace;
|
||||||
|
|
||||||
pub const PRIVATE_CLIENT_BOOTSTRAP_ROLE: &str = "--lazaret-private-client-bootstrap-v1";
|
pub const PRIVATE_CLIENT_BOOTSTRAP_ROLE: &str = "--lazaret-private-client-bootstrap-v1";
|
||||||
|
|
@ -135,7 +133,7 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
|
||||||
.build()
|
.build()
|
||||||
.map_err(failure)?;
|
.map_err(failure)?;
|
||||||
let selection = runtime
|
let selection = runtime
|
||||||
.block_on(select_production_cold_start(
|
.block_on(select_runtime(
|
||||||
&configuration,
|
&configuration,
|
||||||
Arc::clone(&store),
|
Arc::clone(&store),
|
||||||
InstalledProductPaths {
|
InstalledProductPaths {
|
||||||
|
|
@ -154,18 +152,18 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
|
||||||
session_anchor_id: parsed.session_anchor_id,
|
session_anchor_id: parsed.session_anchor_id,
|
||||||
session_phase: SessionPhase::New,
|
session_phase: SessionPhase::New,
|
||||||
};
|
};
|
||||||
ProductionColdStart::stage(inputs)
|
ColdStart::stage(inputs)
|
||||||
.map_err(failure)?
|
.map_err(failure)?
|
||||||
.run(bootstrap_owner, &owner, serve_published)
|
.run(bootstrap_owner, &owner, serve_published)
|
||||||
.map_err(failure)
|
.map_err(failure)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn serve_published(
|
fn serve_published(
|
||||||
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
|
lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>,
|
||||||
waiters: &RefCell<BootstrapWaiterFace>,
|
waiters: &RefCell<BootstrapWaiterFace>,
|
||||||
bootstrap_owner: &BootstrapOwnerGuard,
|
bootstrap_owner: &BootstrapOwnerGuard,
|
||||||
bootstrap_owner_session_id: SessionId,
|
bootstrap_owner_session_id: SessionId,
|
||||||
) -> Result<(), ProductionColdStartError> {
|
) -> Result<(), ColdStartError> {
|
||||||
let escalation = take_escalation(lifecycle)?;
|
let escalation = take_escalation(lifecycle)?;
|
||||||
std::thread::scope(|scope| {
|
std::thread::scope(|scope| {
|
||||||
let escalation_enabled = escalation.is_some();
|
let escalation_enabled = escalation.is_some();
|
||||||
|
|
@ -187,9 +185,9 @@ fn serve_published(
|
||||||
}
|
}
|
||||||
require_escalation_running(&escalation_completions, escalation_enabled)?;
|
require_escalation_running(&escalation_completions, escalation_enabled)?;
|
||||||
let face = {
|
let face = {
|
||||||
let mut lifecycle = lifecycle.lock().map_err(|_| {
|
let mut lifecycle = lifecycle
|
||||||
ProductionColdStartError::service("production lifecycle lock is poisoned")
|
.lock()
|
||||||
})?;
|
.map_err(|_| ColdStartError::service("lifecycle lock is poisoned"))?;
|
||||||
let Some(face) = lifecycle.adapter().published_control() else {
|
let Some(face) = lifecycle.adapter().published_control() else {
|
||||||
for socket in active.values() {
|
for socket in active.values() {
|
||||||
let _ = socket.shutdown(std::net::Shutdown::Both);
|
let _ = socket.shutdown(std::net::Shutdown::Both);
|
||||||
|
|
@ -202,23 +200,21 @@ fn serve_published(
|
||||||
escalation.stop();
|
escalation.stop();
|
||||||
escalation_completions
|
escalation_completions
|
||||||
.recv()
|
.recv()
|
||||||
.map_err(ProductionColdStartError::service)?
|
.map_err(ColdStartError::service)?
|
||||||
.map_err(ProductionColdStartError::service)?;
|
.map_err(ColdStartError::service)?;
|
||||||
}
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
lifecycle
|
lifecycle
|
||||||
.reconcile_published_bootstrap_owner(
|
.reconcile_published_bootstrap_owner(
|
||||||
bootstrap_owner_session_id,
|
bootstrap_owner_session_id,
|
||||||
bootstrap_owner
|
bootstrap_owner.is_live().map_err(ColdStartError::service)?,
|
||||||
.is_live()
|
|
||||||
.map_err(ProductionColdStartError::service)?,
|
|
||||||
)
|
)
|
||||||
.map_err(ProductionColdStartError::service)?;
|
.map_err(ColdStartError::service)?;
|
||||||
waiters
|
waiters
|
||||||
.borrow_mut()
|
.borrow_mut()
|
||||||
.reconcile_published(&mut lifecycle)
|
.reconcile_published(&mut lifecycle)
|
||||||
.map_err(ProductionColdStartError::service)?;
|
.map_err(ColdStartError::service)?;
|
||||||
face
|
face
|
||||||
};
|
};
|
||||||
if active.len() == MAX_CONCURRENT_CONNECTIONS {
|
if active.len() == MAX_CONCURRENT_CONNECTIONS {
|
||||||
|
|
@ -229,16 +225,16 @@ fn serve_published(
|
||||||
Ok(connection) => {
|
Ok(connection) => {
|
||||||
let cancellation = connection
|
let cancellation = connection
|
||||||
.cancellation_handle()
|
.cancellation_handle()
|
||||||
.map_err(ProductionColdStartError::service)?;
|
.map_err(ColdStartError::service)?;
|
||||||
let connection_id = next_connection;
|
let connection_id = next_connection;
|
||||||
next_connection = next_connection.checked_add(1).ok_or_else(|| {
|
next_connection = next_connection.checked_add(1).ok_or_else(|| {
|
||||||
ProductionColdStartError::service("control connection counter overflow")
|
ColdStartError::service("control connection counter overflow")
|
||||||
})?;
|
})?;
|
||||||
active.insert(connection_id, cancellation);
|
active.insert(connection_id, cancellation);
|
||||||
let completed = completed.clone();
|
let completed = completed.clone();
|
||||||
let escalation = escalation_handle.clone();
|
let escalation = escalation_handle.clone();
|
||||||
scope.spawn(move || {
|
scope.spawn(move || {
|
||||||
let result = serve_production_control_connection_with_escalation(
|
let result = serve_control_connection_with_escalation(
|
||||||
lifecycle,
|
lifecycle,
|
||||||
&face,
|
&face,
|
||||||
connection,
|
connection,
|
||||||
|
|
@ -252,44 +248,41 @@ fn serve_published(
|
||||||
{
|
{
|
||||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||||
}
|
}
|
||||||
Err(error) => return Err(ProductionColdStartError::service(error)),
|
Err(error) => return Err(ColdStartError::service(error)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn take_escalation<'resources>(
|
fn take_escalation<'resources>(
|
||||||
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, 'resources>>,
|
lifecycle: &Mutex<CoordinatorLifecycle<'_, 'resources>>,
|
||||||
) -> Result<
|
) -> Result<Option<(EscalationService<'resources>, EscalationServiceHandle)>, ColdStartError> {
|
||||||
Option<(ProductionEscalation<'resources>, ProductionEscalationHandle)>,
|
|
||||||
ProductionColdStartError,
|
|
||||||
> {
|
|
||||||
let mut lifecycle = lifecycle
|
let mut lifecycle = lifecycle
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| ProductionColdStartError::service("production lifecycle lock is poisoned"))?;
|
.map_err(|_| ColdStartError::service("lifecycle lock is poisoned"))?;
|
||||||
lifecycle
|
lifecycle
|
||||||
.adapter()
|
.adapter()
|
||||||
.published_control()
|
.published_control()
|
||||||
.ok_or_else(|| ProductionColdStartError::service("control face is absent"))?
|
.ok_or_else(|| ColdStartError::service("control face is absent"))?
|
||||||
.set_nonblocking(true)
|
.set_nonblocking(true)
|
||||||
.map_err(ProductionColdStartError::service)?;
|
.map_err(ColdStartError::service)?;
|
||||||
Ok(lifecycle.adapter_mut().take_escalation())
|
Ok(lifecycle.adapter_mut().take_escalation())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn require_escalation_running(
|
fn require_escalation_running(
|
||||||
completions: &std::sync::mpsc::Receiver<Result<(), crate::escalation::EscalationError>>,
|
completions: &std::sync::mpsc::Receiver<Result<(), crate::escalation::EscalationError>>,
|
||||||
enabled: bool,
|
enabled: bool,
|
||||||
) -> Result<(), ProductionColdStartError> {
|
) -> Result<(), ColdStartError> {
|
||||||
match completions.try_recv() {
|
match completions.try_recv() {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
result.map_err(ProductionColdStartError::service)?;
|
result.map_err(ColdStartError::service)?;
|
||||||
Err(ProductionColdStartError::service(
|
Err(ColdStartError::service(
|
||||||
"production escalation service stopped before teardown",
|
"escalation service stopped before teardown",
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
Err(std::sync::mpsc::TryRecvError::Disconnected) if enabled => Err(
|
Err(std::sync::mpsc::TryRecvError::Disconnected) if enabled => {
|
||||||
ProductionColdStartError::service("production escalation service was abandoned"),
|
Err(ColdStartError::service("escalation service was abandoned"))
|
||||||
),
|
}
|
||||||
Err(std::sync::mpsc::TryRecvError::Empty | std::sync::mpsc::TryRecvError::Disconnected) => {
|
Err(std::sync::mpsc::TryRecvError::Empty | std::sync::mpsc::TryRecvError::Disconnected) => {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -299,15 +292,13 @@ fn require_escalation_running(
|
||||||
fn wait_for_connection_completion(
|
fn wait_for_connection_completion(
|
||||||
completions: &std::sync::mpsc::Receiver<(u64, Result<(), ControlError>)>,
|
completions: &std::sync::mpsc::Receiver<(u64, Result<(), ControlError>)>,
|
||||||
active: &mut BTreeMap<u64, std::os::unix::net::UnixStream>,
|
active: &mut BTreeMap<u64, std::os::unix::net::UnixStream>,
|
||||||
) -> Result<(), ProductionColdStartError> {
|
) -> Result<(), ColdStartError> {
|
||||||
let (connection_id, result) = completions
|
let (connection_id, result) = completions.recv().map_err(ColdStartError::service)?;
|
||||||
.recv()
|
|
||||||
.map_err(ProductionColdStartError::service)?;
|
|
||||||
active.remove(&connection_id);
|
active.remove(&connection_id);
|
||||||
connection_completion(result)
|
connection_completion(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn connection_completion(result: Result<(), ControlError>) -> Result<(), ProductionColdStartError> {
|
fn connection_completion(result: Result<(), ControlError>) -> Result<(), ColdStartError> {
|
||||||
match result {
|
match result {
|
||||||
Ok(()) => Ok(()),
|
Ok(()) => Ok(()),
|
||||||
Err(ControlError::Io(error))
|
Err(ControlError::Io(error))
|
||||||
|
|
@ -322,7 +313,7 @@ fn connection_completion(result: Result<(), ControlError>) -> Result<(), Product
|
||||||
{
|
{
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
Err(error) => Err(ProductionColdStartError::service(error)),
|
Err(error) => Err(ColdStartError::service(error)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -470,7 +461,7 @@ fn load_configuration(
|
||||||
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
|
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
|
||||||
xdg_runtime_dir: runtime,
|
xdg_runtime_dir: runtime,
|
||||||
environment: environment(),
|
environment: environment(),
|
||||||
sandbox_layout: production_sandbox_layout()?,
|
sandbox_layout: default_sandbox_layout()?,
|
||||||
})?)
|
})?)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
1091
crates/lazaret-host-coordinator/src/cold_start.rs
Normal file
1091
crates/lazaret-host-coordinator/src/cold_start.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -993,7 +993,7 @@ struct SessionAuthorityFiles {
|
||||||
registration_identity: MailboxEntryIdentity,
|
registration_identity: MailboxEntryIdentity,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct ProductionSessionAuthority {
|
struct SessionEscalationAuthority {
|
||||||
session: VerifiedSession,
|
session: VerifiedSession,
|
||||||
files: SessionAuthorityFiles,
|
files: SessionAuthorityFiles,
|
||||||
}
|
}
|
||||||
|
|
@ -1004,7 +1004,7 @@ struct RegistrationCommand {
|
||||||
completed: mpsc::SyncSender<Result<(), String>>,
|
completed: mpsc::SyncSender<Result<(), String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
enum ProductionEscalationCommand {
|
enum EscalationServiceCommand {
|
||||||
Register(Box<RegistrationCommand>),
|
Register(Box<RegistrationCommand>),
|
||||||
Disconnect {
|
Disconnect {
|
||||||
session_id: SessionId,
|
session_id: SessionId,
|
||||||
|
|
@ -1014,11 +1014,11 @@ enum ProductionEscalationCommand {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub(crate) struct ProductionEscalationHandle {
|
pub(crate) struct EscalationServiceHandle {
|
||||||
commands: mpsc::Sender<ProductionEscalationCommand>,
|
commands: mpsc::Sender<EscalationServiceCommand>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProductionEscalationHandle {
|
impl EscalationServiceHandle {
|
||||||
pub(crate) fn register(
|
pub(crate) fn register(
|
||||||
&self,
|
&self,
|
||||||
binding: SessionBinding,
|
binding: SessionBinding,
|
||||||
|
|
@ -1026,40 +1026,40 @@ impl ProductionEscalationHandle {
|
||||||
) -> Result<(), EscalationError> {
|
) -> Result<(), EscalationError> {
|
||||||
let (completed, result) = mpsc::sync_channel(1);
|
let (completed, result) = mpsc::sync_channel(1);
|
||||||
self.commands
|
self.commands
|
||||||
.send(ProductionEscalationCommand::Register(Box::new(
|
.send(EscalationServiceCommand::Register(Box::new(
|
||||||
RegistrationCommand {
|
RegistrationCommand {
|
||||||
binding,
|
binding,
|
||||||
registration,
|
registration,
|
||||||
completed,
|
completed,
|
||||||
},
|
},
|
||||||
)))
|
)))
|
||||||
.map_err(|_| io::Error::other("production escalation service stopped"))?;
|
.map_err(|_| io::Error::other("escalation service stopped"))?;
|
||||||
result
|
result
|
||||||
.recv()
|
.recv()
|
||||||
.map_err(|_| io::Error::other("production escalation registration was abandoned"))?
|
.map_err(|_| io::Error::other("escalation registration was abandoned"))?
|
||||||
.map_err(|error| io::Error::other(error).into())
|
.map_err(|error| io::Error::other(error).into())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn disconnect(&self, session_id: SessionId) -> Result<(), EscalationError> {
|
pub(crate) fn disconnect(&self, session_id: SessionId) -> Result<(), EscalationError> {
|
||||||
let (completed, result) = mpsc::sync_channel(1);
|
let (completed, result) = mpsc::sync_channel(1);
|
||||||
self.commands
|
self.commands
|
||||||
.send(ProductionEscalationCommand::Disconnect {
|
.send(EscalationServiceCommand::Disconnect {
|
||||||
session_id,
|
session_id,
|
||||||
completed,
|
completed,
|
||||||
})
|
})
|
||||||
.map_err(|_| io::Error::other("production escalation service stopped"))?;
|
.map_err(|_| io::Error::other("escalation service stopped"))?;
|
||||||
result
|
result
|
||||||
.recv()
|
.recv()
|
||||||
.map_err(|_| io::Error::other("production escalation disconnect was abandoned"))?
|
.map_err(|_| io::Error::other("escalation disconnect was abandoned"))?
|
||||||
.map_err(|error| io::Error::other(error).into())
|
.map_err(|error| io::Error::other(error).into())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn stop(&self) {
|
pub(crate) fn stop(&self) {
|
||||||
let _ = self.commands.send(ProductionEscalationCommand::Stop);
|
let _ = self.commands.send(EscalationServiceCommand::Stop);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct ProductionEscalation<'a> {
|
pub(crate) struct EscalationService<'a> {
|
||||||
configuration: &'a ResolvedDomainConfiguration,
|
configuration: &'a ResolvedDomainConfiguration,
|
||||||
publication: ProtocolPublicationIdentity,
|
publication: ProtocolPublicationIdentity,
|
||||||
credentials: SandboxCredentialBundle,
|
credentials: SandboxCredentialBundle,
|
||||||
|
|
@ -1068,12 +1068,12 @@ pub(crate) struct ProductionEscalation<'a> {
|
||||||
prompt_transport: Arc<SessionPromptTransport>,
|
prompt_transport: Arc<SessionPromptTransport>,
|
||||||
coordinator: EscalationCoordinator,
|
coordinator: EscalationCoordinator,
|
||||||
ownership: LocalExecutionOwnership,
|
ownership: LocalExecutionOwnership,
|
||||||
sessions: BTreeMap<SessionId, ProductionSessionAuthority>,
|
sessions: BTreeMap<SessionId, SessionEscalationAuthority>,
|
||||||
active: BTreeMap<RequestId, SessionId>,
|
active: BTreeMap<RequestId, SessionId>,
|
||||||
commands: mpsc::Receiver<ProductionEscalationCommand>,
|
commands: mpsc::Receiver<EscalationServiceCommand>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct ProductionEscalationInputs<'a> {
|
pub(crate) struct EscalationServiceInputs<'a> {
|
||||||
pub configuration: &'a ResolvedDomainConfiguration,
|
pub configuration: &'a ResolvedDomainConfiguration,
|
||||||
pub publication: ProtocolPublicationIdentity,
|
pub publication: ProtocolPublicationIdentity,
|
||||||
pub credentials: SandboxCredentialBundle,
|
pub credentials: SandboxCredentialBundle,
|
||||||
|
|
@ -1086,10 +1086,10 @@ pub(crate) struct ProductionEscalationInputs<'a> {
|
||||||
pub invoking_uid: u32,
|
pub invoking_uid: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> ProductionEscalation<'a> {
|
impl<'a> EscalationService<'a> {
|
||||||
pub(crate) fn new(
|
pub(crate) fn new(
|
||||||
inputs: ProductionEscalationInputs<'a>,
|
inputs: EscalationServiceInputs<'a>,
|
||||||
) -> Result<(Self, ProductionEscalationHandle), EscalationError> {
|
) -> Result<(Self, EscalationServiceHandle), EscalationError> {
|
||||||
inputs.credential_files.publish(
|
inputs.credential_files.publish(
|
||||||
OsStr::new("publication.postcard"),
|
OsStr::new("publication.postcard"),
|
||||||
&lazaret_core::codec::encode(&inputs.publication)
|
&lazaret_core::codec::encode(&inputs.publication)
|
||||||
|
|
@ -1122,7 +1122,7 @@ impl<'a> ProductionEscalation<'a> {
|
||||||
active: BTreeMap::new(),
|
active: BTreeMap::new(),
|
||||||
commands: receiver,
|
commands: receiver,
|
||||||
},
|
},
|
||||||
ProductionEscalationHandle { commands },
|
EscalationServiceHandle { commands },
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1141,10 +1141,10 @@ impl<'a> ProductionEscalation<'a> {
|
||||||
|
|
||||||
fn apply_command(
|
fn apply_command(
|
||||||
&mut self,
|
&mut self,
|
||||||
command: ProductionEscalationCommand,
|
command: EscalationServiceCommand,
|
||||||
) -> Result<bool, EscalationError> {
|
) -> Result<bool, EscalationError> {
|
||||||
match command {
|
match command {
|
||||||
ProductionEscalationCommand::Register(command) => {
|
EscalationServiceCommand::Register(command) => {
|
||||||
let RegistrationCommand {
|
let RegistrationCommand {
|
||||||
binding,
|
binding,
|
||||||
registration,
|
registration,
|
||||||
|
|
@ -1160,7 +1160,7 @@ impl<'a> ProductionEscalation<'a> {
|
||||||
}
|
}
|
||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
ProductionEscalationCommand::Disconnect {
|
EscalationServiceCommand::Disconnect {
|
||||||
session_id,
|
session_id,
|
||||||
completed,
|
completed,
|
||||||
} => {
|
} => {
|
||||||
|
|
@ -1174,7 +1174,7 @@ impl<'a> ProductionEscalation<'a> {
|
||||||
}
|
}
|
||||||
Ok(false)
|
Ok(false)
|
||||||
}
|
}
|
||||||
ProductionEscalationCommand::Stop => {
|
EscalationServiceCommand::Stop => {
|
||||||
let sessions = self.sessions.keys().copied().collect::<Vec<_>>();
|
let sessions = self.sessions.keys().copied().collect::<Vec<_>>();
|
||||||
for session_id in sessions {
|
for session_id in sessions {
|
||||||
self.disconnect(session_id)?;
|
self.disconnect(session_id)?;
|
||||||
|
|
@ -1226,7 +1226,7 @@ impl<'a> ProductionEscalation<'a> {
|
||||||
};
|
};
|
||||||
self.sessions.insert(
|
self.sessions.insert(
|
||||||
session_id,
|
session_id,
|
||||||
ProductionSessionAuthority {
|
SessionEscalationAuthority {
|
||||||
session,
|
session,
|
||||||
files: SessionAuthorityFiles {
|
files: SessionAuthorityFiles {
|
||||||
binding_name,
|
binding_name,
|
||||||
|
|
|
||||||
|
|
@ -2,16 +2,16 @@
|
||||||
|
|
||||||
pub mod bootstrap;
|
pub mod bootstrap;
|
||||||
pub mod client_bootstrap;
|
pub mod client_bootstrap;
|
||||||
|
pub mod cold_start;
|
||||||
pub mod control;
|
pub mod control;
|
||||||
pub mod domain_store;
|
pub mod domain_store;
|
||||||
pub mod escalation;
|
pub mod escalation;
|
||||||
pub mod git_routing;
|
pub mod git_routing;
|
||||||
pub mod lifecycle;
|
pub mod lifecycle;
|
||||||
pub mod network;
|
pub mod network;
|
||||||
pub mod production;
|
|
||||||
pub mod production_selector;
|
|
||||||
pub mod recovery;
|
pub mod recovery;
|
||||||
pub mod runtime;
|
pub mod runtime;
|
||||||
|
pub mod runtime_selector;
|
||||||
pub mod sandbox_init;
|
pub mod sandbox_init;
|
||||||
pub mod state_anchor;
|
pub mod state_anchor;
|
||||||
pub mod state_maintenance;
|
pub mod state_maintenance;
|
||||||
|
|
|
||||||
|
|
@ -276,7 +276,7 @@ where
|
||||||
/// Drives one cold-start leader through the complete normal publication boundary.
|
/// Drives one cold-start leader through the complete normal publication boundary.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Returns the first invalid transition, durable update, or production adapter failure.
|
/// Returns the first invalid transition, durable update, or lifecycle adapter failure.
|
||||||
pub fn bootstrap_new_domain(
|
pub fn bootstrap_new_domain(
|
||||||
&mut self,
|
&mut self,
|
||||||
generation: Generation,
|
generation: Generation,
|
||||||
|
|
@ -291,7 +291,7 @@ where
|
||||||
/// Returns `false` after applying the modeled bootstrap-owner disconnect transition.
|
/// Returns `false` after applying the modeled bootstrap-owner disconnect transition.
|
||||||
///
|
///
|
||||||
/// # Errors
|
/// # Errors
|
||||||
/// Returns the first invalid transition, durable update, or production adapter failure.
|
/// Returns the first invalid transition, durable update, or lifecycle adapter failure.
|
||||||
pub fn bootstrap_new_domain_while_owner_live(
|
pub fn bootstrap_new_domain_while_owner_live(
|
||||||
&mut self,
|
&mut self,
|
||||||
generation: Generation,
|
generation: Generation,
|
||||||
|
|
|
||||||
|
|
@ -565,7 +565,7 @@ fn require_mount_absent(
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub(crate) enum ProductionRecoveryBoundary {
|
pub(crate) enum RecoveryBoundary {
|
||||||
Snapshot,
|
Snapshot,
|
||||||
Publication,
|
Publication,
|
||||||
Sandbox,
|
Sandbox,
|
||||||
|
|
@ -578,22 +578,16 @@ pub(crate) enum ProductionRecoveryBoundary {
|
||||||
FinishGeneration,
|
FinishGeneration,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn recover_production_before_cold_start(
|
pub(crate) fn recover_before_cold_start(
|
||||||
configuration: &ResolvedDomainConfiguration,
|
configuration: &ResolvedDomainConfiguration,
|
||||||
store: &HostCoordinatorDomainStore,
|
store: &HostCoordinatorDomainStore,
|
||||||
catalog: &Catalog,
|
catalog: &Catalog,
|
||||||
prepared: &PreparedLowerStore,
|
prepared: &PreparedLowerStore,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
recover_production_before_cold_start_with(
|
recover_before_cold_start_with(configuration, store, catalog, prepared, |_, _| Ok(()))
|
||||||
configuration,
|
|
||||||
store,
|
|
||||||
catalog,
|
|
||||||
prepared,
|
|
||||||
|_, _| Ok(()),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn recover_production_before_cold_start_with<Failpoint>(
|
fn recover_before_cold_start_with<Failpoint>(
|
||||||
configuration: &ResolvedDomainConfiguration,
|
configuration: &ResolvedDomainConfiguration,
|
||||||
store: &HostCoordinatorDomainStore,
|
store: &HostCoordinatorDomainStore,
|
||||||
catalog: &Catalog,
|
catalog: &Catalog,
|
||||||
|
|
@ -601,9 +595,9 @@ fn recover_production_before_cold_start_with<Failpoint>(
|
||||||
failpoint: Failpoint,
|
failpoint: Failpoint,
|
||||||
) -> Result<(), String>
|
) -> Result<(), String>
|
||||||
where
|
where
|
||||||
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
|
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
|
||||||
{
|
{
|
||||||
recover_production_before_cold_start_with_opener(
|
recover_before_cold_start_with_opener(
|
||||||
configuration,
|
configuration,
|
||||||
store,
|
store,
|
||||||
catalog,
|
catalog,
|
||||||
|
|
@ -614,7 +608,7 @@ where
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn recover_production_before_cold_start_with_opener<Failpoint, Open, Delete>(
|
fn recover_before_cold_start_with_opener<Failpoint, Open, Delete>(
|
||||||
configuration: &ResolvedDomainConfiguration,
|
configuration: &ResolvedDomainConfiguration,
|
||||||
store: &HostCoordinatorDomainStore,
|
store: &HostCoordinatorDomainStore,
|
||||||
catalog: &Catalog,
|
catalog: &Catalog,
|
||||||
|
|
@ -624,12 +618,12 @@ fn recover_production_before_cold_start_with_opener<Failpoint, Open, Delete>(
|
||||||
mut delete: Delete,
|
mut delete: Delete,
|
||||||
) -> Result<(), String>
|
) -> Result<(), String>
|
||||||
where
|
where
|
||||||
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
|
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
|
||||||
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
|
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
|
||||||
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
|
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
|
||||||
{
|
{
|
||||||
recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
|
recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
|
||||||
failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
|
failpoint(RecoveryBoundary::Snapshot, GenerationId::new(0))?;
|
||||||
let snapshot = store
|
let snapshot = store
|
||||||
.snapshot()
|
.snapshot()
|
||||||
.map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
|
.map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
|
||||||
|
|
@ -661,14 +655,11 @@ where
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
failpoint(
|
failpoint(RecoveryBoundary::PrepareOperations, GenerationId::new(0))?;
|
||||||
ProductionRecoveryBoundary::PrepareOperations,
|
|
||||||
GenerationId::new(0),
|
|
||||||
)?;
|
|
||||||
prepared
|
prepared
|
||||||
.recover_preparations()
|
.recover_preparations()
|
||||||
.map_err(|error| error.to_string())?;
|
.map_err(|error| error.to_string())?;
|
||||||
failpoint(ProductionRecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
|
failpoint(RecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
|
||||||
let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
|
let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
|
||||||
let selected = catalog_snapshot
|
let selected = catalog_snapshot
|
||||||
.prepared_artifacts
|
.prepared_artifacts
|
||||||
|
|
@ -697,7 +688,7 @@ fn recover_generation<Failpoint, Open, Delete>(
|
||||||
delete: &mut Delete,
|
delete: &mut Delete,
|
||||||
) -> Result<(), String>
|
) -> Result<(), String>
|
||||||
where
|
where
|
||||||
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
|
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
|
||||||
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
|
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
|
||||||
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
|
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
|
||||||
{
|
{
|
||||||
|
|
@ -744,11 +735,11 @@ where
|
||||||
let mut owners = seal_runtime_owners(&ownership, open)
|
let mut owners = seal_runtime_owners(&ownership, open)
|
||||||
.map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
|
.map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
|
||||||
|
|
||||||
failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
|
failpoint(RecoveryBoundary::Publication, generation_id)?;
|
||||||
recover_publication(configuration, store, generation_id)
|
recover_publication(configuration, store, generation_id)
|
||||||
.map_err(|error| format!("Publication recovery failed: {error}"))?;
|
.map_err(|error| format!("Publication recovery failed: {error}"))?;
|
||||||
|
|
||||||
failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
|
failpoint(RecoveryBoundary::Sandbox, generation_id)?;
|
||||||
reconcile_runtime_before_state_anchor(&mut owners)
|
reconcile_runtime_before_state_anchor(&mut owners)
|
||||||
.map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
|
.map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
|
||||||
require_sandbox_absent(configuration, store, generation_id, delete)
|
require_sandbox_absent(configuration, store, generation_id, delete)
|
||||||
|
|
@ -756,13 +747,13 @@ where
|
||||||
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
|
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
|
||||||
.map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
|
.map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
|
||||||
|
|
||||||
failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
|
failpoint(RecoveryBoundary::HelperAndNetwork, generation_id)?;
|
||||||
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
|
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
|
||||||
.map_err(|error| format!("Helper cleanup failed: {error}"))?;
|
.map_err(|error| format!("Helper cleanup failed: {error}"))?;
|
||||||
clean_role_after_absence(store, generation_id, ParticipantRole::Network)
|
clean_role_after_absence(store, generation_id, ParticipantRole::Network)
|
||||||
.map_err(|error| format!("Network cleanup failed: {error}"))?;
|
.map_err(|error| format!("Network cleanup failed: {error}"))?;
|
||||||
|
|
||||||
failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
|
failpoint(RecoveryBoundary::GitRouting, generation_id)?;
|
||||||
require_mount_absent(
|
require_mount_absent(
|
||||||
ownership.git_routing_path.as_ref(),
|
ownership.git_routing_path.as_ref(),
|
||||||
ownership.git_routing_mount,
|
ownership.git_routing_mount,
|
||||||
|
|
@ -772,11 +763,11 @@ where
|
||||||
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
|
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
|
||||||
.map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
|
.map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
|
||||||
|
|
||||||
failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
|
failpoint(RecoveryBoundary::StateAnchor, generation_id)?;
|
||||||
recover_state_anchor(configuration, store, catalog, generation, &mut owners)
|
recover_state_anchor(configuration, store, catalog, generation, &mut owners)
|
||||||
.map_err(|error| format!("State Anchor recovery failed: {error}"))?;
|
.map_err(|error| format!("State Anchor recovery failed: {error}"))?;
|
||||||
|
|
||||||
failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
|
failpoint(RecoveryBoundary::PreparedLower, generation_id)?;
|
||||||
recover_partial_prepared_lower(configuration, store, catalog, generation_id)
|
recover_partial_prepared_lower(configuration, store, catalog, generation_id)
|
||||||
.map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
|
.map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
|
||||||
|
|
||||||
|
|
@ -787,7 +778,7 @@ where
|
||||||
}) {
|
}) {
|
||||||
return Err("generation retains non-Clean participants after recovery".into());
|
return Err("generation retains non-Clean participants after recovery".into());
|
||||||
}
|
}
|
||||||
failpoint(ProductionRecoveryBoundary::FinishGeneration, generation_id)?;
|
failpoint(RecoveryBoundary::FinishGeneration, generation_id)?;
|
||||||
let mut clean = generation.clone();
|
let mut clean = generation.clone();
|
||||||
clean.durable_decision = DurableDecision::Clean;
|
clean.durable_decision = DurableDecision::Clean;
|
||||||
store
|
store
|
||||||
|
|
@ -1341,7 +1332,7 @@ mod tests {
|
||||||
SandboxOwnership, StateAnchorOwnership,
|
SandboxOwnership, StateAnchorOwnership,
|
||||||
};
|
};
|
||||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
|
@ -1504,8 +1495,8 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn production_preflight_recovers_full_runtime_then_is_idempotent() {
|
fn recovery_preflight_recovers_full_runtime_then_is_idempotent() {
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let mut boundaries = Vec::new();
|
let mut boundaries = Vec::new();
|
||||||
fixture
|
fixture
|
||||||
.recover_with(|boundary, generation| {
|
.recover_with(|boundary, generation| {
|
||||||
|
|
@ -1516,7 +1507,7 @@ mod tests {
|
||||||
assert!(
|
assert!(
|
||||||
boundaries
|
boundaries
|
||||||
.iter()
|
.iter()
|
||||||
.any(|(boundary, _)| { *boundary == ProductionRecoveryBoundary::StateAnchor })
|
.any(|(boundary, _)| { *boundary == RecoveryBoundary::StateAnchor })
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
fixture.store.snapshot().unwrap().generations[0].durable_decision,
|
fixture.store.snapshot().unwrap().generations[0].durable_decision,
|
||||||
|
|
@ -1552,7 +1543,7 @@ mod tests {
|
||||||
.exists()
|
.exists()
|
||||||
);
|
);
|
||||||
|
|
||||||
recover_production_before_cold_start(
|
recover_before_cold_start(
|
||||||
&fixture.configuration,
|
&fixture.configuration,
|
||||||
&fixture.store,
|
&fixture.store,
|
||||||
&fixture.catalog,
|
&fixture.catalog,
|
||||||
|
|
@ -1570,12 +1561,12 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bare_publication_intent_requires_every_fixed_label_to_be_absent() {
|
fn bare_publication_intent_requires_every_fixed_label_to_be_absent() {
|
||||||
let absent = ProductionRecoveryFixture::new();
|
let absent = RecoveryFixture::new();
|
||||||
absent.prepare_bare_publication(PublicationFixtureBoundary::TrulyAbsent);
|
absent.prepare_bare_publication(PublicationFixtureBoundary::TrulyAbsent);
|
||||||
recover_publication(&absent.configuration, &absent.store, GenerationId::new(2)).unwrap();
|
recover_publication(&absent.configuration, &absent.store, GenerationId::new(2)).unwrap();
|
||||||
assert_eq!(absent.publication(2).state(), ParticipantState::Clean);
|
assert_eq!(absent.publication(2).state(), ParticipantState::Clean);
|
||||||
|
|
||||||
let identity = ProductionRecoveryFixture::new();
|
let identity = RecoveryFixture::new();
|
||||||
identity.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedIdentity);
|
identity.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedIdentity);
|
||||||
assert!(
|
assert!(
|
||||||
recover_publication(
|
recover_publication(
|
||||||
|
|
@ -1594,7 +1585,7 @@ mod tests {
|
||||||
assert_eq!(identity.publication(2).state(), ParticipantState::Intent);
|
assert_eq!(identity.publication(2).state(), ParticipantState::Intent);
|
||||||
assert!(identity.publication(2).ownership().is_none());
|
assert!(identity.publication(2).ownership().is_none());
|
||||||
|
|
||||||
let socket = ProductionRecoveryFixture::new();
|
let socket = RecoveryFixture::new();
|
||||||
socket.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedSocket);
|
socket.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedSocket);
|
||||||
assert!(
|
assert!(
|
||||||
recover_publication(&socket.configuration, &socket.store, GenerationId::new(2))
|
recover_publication(&socket.configuration, &socket.store, GenerationId::new(2))
|
||||||
|
|
@ -1617,27 +1608,27 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn production_preflight_retries_every_recovery_boundary() {
|
fn recovery_preflight_retries_every_recovery_boundary() {
|
||||||
for failed in [
|
for failed in [
|
||||||
ProductionRecoveryBoundary::Snapshot,
|
RecoveryBoundary::Snapshot,
|
||||||
ProductionRecoveryBoundary::Publication,
|
RecoveryBoundary::Publication,
|
||||||
ProductionRecoveryBoundary::Sandbox,
|
RecoveryBoundary::Sandbox,
|
||||||
ProductionRecoveryBoundary::HelperAndNetwork,
|
RecoveryBoundary::HelperAndNetwork,
|
||||||
ProductionRecoveryBoundary::GitRouting,
|
RecoveryBoundary::GitRouting,
|
||||||
ProductionRecoveryBoundary::StateAnchor,
|
RecoveryBoundary::StateAnchor,
|
||||||
ProductionRecoveryBoundary::PreparedLower,
|
RecoveryBoundary::PreparedLower,
|
||||||
ProductionRecoveryBoundary::PrepareOperations,
|
RecoveryBoundary::PrepareOperations,
|
||||||
ProductionRecoveryBoundary::ArtifactGc,
|
RecoveryBoundary::ArtifactGc,
|
||||||
ProductionRecoveryBoundary::FinishGeneration,
|
RecoveryBoundary::FinishGeneration,
|
||||||
] {
|
] {
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let mut injected = false;
|
let mut injected = false;
|
||||||
assert!(
|
assert!(
|
||||||
fixture
|
fixture
|
||||||
.recover_with(|boundary, _| {
|
.recover_with(|boundary, _| {
|
||||||
if boundary == failed && !injected {
|
if boundary == failed && !injected {
|
||||||
injected = true;
|
injected = true;
|
||||||
Err("injected production recovery boundary".into())
|
Err("injected recovery boundary".into())
|
||||||
} else {
|
} else {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -1654,10 +1645,10 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn production_preflight_opens_every_exact_owner_once() {
|
fn recovery_preflight_opens_every_exact_owner_once() {
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let mut opened = BTreeMap::new();
|
let mut opened = BTreeMap::new();
|
||||||
recover_production_before_cold_start_with_opener(
|
recover_before_cold_start_with_opener(
|
||||||
&fixture.configuration,
|
&fixture.configuration,
|
||||||
&fixture.store,
|
&fixture.store,
|
||||||
&fixture.catalog,
|
&fixture.catalog,
|
||||||
|
|
@ -1678,7 +1669,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn publication_replacement_and_runsc_root_replacement_refuse_untouched() {
|
fn publication_replacement_and_runsc_root_replacement_refuse_untouched() {
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
||||||
fixture
|
fixture
|
||||||
.configuration
|
.configuration
|
||||||
|
|
@ -1701,7 +1692,7 @@ mod tests {
|
||||||
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
|
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
|
||||||
assert!(socket.exists());
|
assert!(socket.exists());
|
||||||
|
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
||||||
fixture
|
fixture
|
||||||
.configuration
|
.configuration
|
||||||
|
|
@ -1720,7 +1711,7 @@ mod tests {
|
||||||
.is_symlink()
|
.is_symlink()
|
||||||
);
|
);
|
||||||
|
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
||||||
fixture
|
fixture
|
||||||
.configuration
|
.configuration
|
||||||
|
|
@ -1734,7 +1725,7 @@ mod tests {
|
||||||
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
|
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
|
||||||
assert_eq!(fs::read(&identity).unwrap(), b"replacement");
|
assert_eq!(fs::read(&identity).unwrap(), b"replacement");
|
||||||
|
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
||||||
fixture
|
fixture
|
||||||
.configuration
|
.configuration
|
||||||
|
|
@ -1747,7 +1738,7 @@ mod tests {
|
||||||
fs::rename(&runsc, &old_runsc).unwrap();
|
fs::rename(&runsc, &old_runsc).unwrap();
|
||||||
fs::create_dir(&runsc).unwrap();
|
fs::create_dir(&runsc).unwrap();
|
||||||
let mut deletes = 0;
|
let mut deletes = 0;
|
||||||
let result = recover_production_before_cold_start_with_opener(
|
let result = recover_before_cold_start_with_opener(
|
||||||
&fixture.configuration,
|
&fixture.configuration,
|
||||||
&fixture.store,
|
&fixture.store,
|
||||||
&fixture.catalog,
|
&fixture.catalog,
|
||||||
|
|
@ -1763,7 +1754,7 @@ mod tests {
|
||||||
assert_eq!(deletes, 0);
|
assert_eq!(deletes, 0);
|
||||||
assert!(runsc.exists());
|
assert!(runsc.exists());
|
||||||
|
|
||||||
let fixture = ProductionRecoveryFixture::new();
|
let fixture = RecoveryFixture::new();
|
||||||
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
|
||||||
fixture
|
fixture
|
||||||
.configuration
|
.configuration
|
||||||
|
|
@ -1783,7 +1774,7 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct ProductionRecoveryFixture {
|
struct RecoveryFixture {
|
||||||
_tree: tempfile::TempDir,
|
_tree: tempfile::TempDir,
|
||||||
configuration: ResolvedDomainConfiguration,
|
configuration: ResolvedDomainConfiguration,
|
||||||
store: HostCoordinatorDomainStore,
|
store: HostCoordinatorDomainStore,
|
||||||
|
|
@ -1791,12 +1782,12 @@ mod tests {
|
||||||
prepared: PreparedLowerStore,
|
prepared: PreparedLowerStore,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProductionRecoveryFixture {
|
impl RecoveryFixture {
|
||||||
fn recover_with<Failpoint>(&self, failpoint: Failpoint) -> Result<(), String>
|
fn recover_with<Failpoint>(&self, failpoint: Failpoint) -> Result<(), String>
|
||||||
where
|
where
|
||||||
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
|
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
|
||||||
{
|
{
|
||||||
recover_production_before_cold_start_with_opener(
|
recover_before_cold_start_with_opener(
|
||||||
&self.configuration,
|
&self.configuration,
|
||||||
&self.store,
|
&self.store,
|
||||||
&self.catalog,
|
&self.catalog,
|
||||||
|
|
@ -2221,7 +2212,7 @@ mod tests {
|
||||||
xdg_data_home: None,
|
xdg_data_home: None,
|
||||||
xdg_runtime_dir: runtime,
|
xdg_runtime_dir: runtime,
|
||||||
environment: BTreeMap::new(),
|
environment: BTreeMap::new(),
|
||||||
sandbox_layout: production_sandbox_layout().unwrap(),
|
sandbox_layout: default_sandbox_layout().unwrap(),
|
||||||
})
|
})
|
||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
629
crates/lazaret-host-coordinator/src/runtime_selector.rs
Normal file
629
crates/lazaret-host-coordinator/src/runtime_selector.rs
Normal file
|
|
@ -0,0 +1,629 @@
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::fmt;
|
||||||
|
use std::fs::File;
|
||||||
|
use std::os::unix::ffi::OsStringExt;
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use lazaret_core::configuration::ResolvedDomainConfiguration;
|
||||||
|
use lazaret_core::identity::{
|
||||||
|
ConfigurationFingerprint, ContainerId, CoordinatorId, DomainId, GenerationId, NetworkSidecarId,
|
||||||
|
PrepareOperationId, SandboxId,
|
||||||
|
};
|
||||||
|
use lazaret_core::network::{DnsQueryExpectation, ReachabilityExpectation};
|
||||||
|
use lazaret_core::records::{DurableDecision, Generation, IpAddress, PreparedArtifact, VolumeKind};
|
||||||
|
use lazaret_linux::filesystem::namespace_identity;
|
||||||
|
use lazaret_linux::platform::{PlatformObservation, check_platform};
|
||||||
|
use lazaret_prepared_lower::{
|
||||||
|
IdMap, IdMapRange, OciRegistryTransport, Platform, PreparedDestination,
|
||||||
|
PreparedDestinationKind, PreparedLowerStore, RegistryCredentials,
|
||||||
|
};
|
||||||
|
use lazaret_runtime_supervisor::network::{DnsProbeTarget, NetworkProbeRequest, TcpProbeTarget};
|
||||||
|
use lazaret_runtime_supervisor::sandbox::{
|
||||||
|
CREDENTIALS_DESTINATION, ESCALATION_CLIENT_DESTINATION, MAILBOX_REQUESTS_DESTINATION,
|
||||||
|
MAILBOX_RESULTS_DESTINATION, RootProcess, SANDBOX_INIT_DESTINATION, TERMINFO_ROOT_DESTINATION,
|
||||||
|
WORKSPACE_DESTINATION,
|
||||||
|
};
|
||||||
|
use lazaret_storage_catalog::{Catalog, CatalogError};
|
||||||
|
use rustix::fs::{Mode, OFlags, ResolveFlags, openat2};
|
||||||
|
|
||||||
|
use crate::cold_start::ColdStartInputs;
|
||||||
|
use crate::domain_store::HostCoordinatorDomainStore;
|
||||||
|
use crate::recovery::recover_before_cold_start;
|
||||||
|
use crate::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE;
|
||||||
|
|
||||||
|
const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct RuntimeSelectionError(String);
|
||||||
|
|
||||||
|
impl fmt::Display for RuntimeSelectionError {
|
||||||
|
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
formatter.write_str(&self.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for RuntimeSelectionError {}
|
||||||
|
|
||||||
|
fn failure(error: impl fmt::Display) -> RuntimeSelectionError {
|
||||||
|
RuntimeSelectionError(error.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct InstalledProductPaths<'a> {
|
||||||
|
pub escalation_client: &'a Path,
|
||||||
|
pub sandbox_init: &'a Path,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct RuntimeSelection {
|
||||||
|
store: Arc<HostCoordinatorDomainStore>,
|
||||||
|
domain_id: DomainId,
|
||||||
|
configuration_fingerprint: ConfigurationFingerprint,
|
||||||
|
catalog: Catalog,
|
||||||
|
artifact: PreparedArtifact,
|
||||||
|
artifact_authority: File,
|
||||||
|
artifact_label: PathBuf,
|
||||||
|
generation: Generation,
|
||||||
|
resolver_addresses: Vec<IpAddress>,
|
||||||
|
network_probe: NetworkProbeRequest,
|
||||||
|
reachability: ReachabilityExpectation,
|
||||||
|
container_id: ContainerId,
|
||||||
|
escalation_client: File,
|
||||||
|
sandbox_init: File,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RuntimeSelection {
|
||||||
|
/// Borrows the selected resources as the existing cold-start input.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Refuses a configuration other than the exact one used for selection.
|
||||||
|
pub fn cold_start_inputs<'a>(
|
||||||
|
&'a self,
|
||||||
|
configuration: &'a ResolvedDomainConfiguration,
|
||||||
|
) -> Result<ColdStartInputs<'a>, RuntimeSelectionError> {
|
||||||
|
if configuration.domain_id() != self.domain_id
|
||||||
|
|| configuration.configuration_fingerprint() != self.configuration_fingerprint
|
||||||
|
{
|
||||||
|
return Err(failure("selected cold start does not match configuration"));
|
||||||
|
}
|
||||||
|
Ok(ColdStartInputs {
|
||||||
|
store: Arc::clone(&self.store),
|
||||||
|
catalog: &self.catalog,
|
||||||
|
configuration,
|
||||||
|
generation: self.generation.clone(),
|
||||||
|
prepared_artifact: &self.artifact,
|
||||||
|
prepared_artifact_authority: &self.artifact_authority,
|
||||||
|
prepared_artifact_label: &self.artifact_label,
|
||||||
|
root_process: RootProcess::new(
|
||||||
|
SANDBOX_INIT_DESTINATION,
|
||||||
|
[PRIVATE_SANDBOX_INIT_ROLE.to_owned()],
|
||||||
|
)
|
||||||
|
.map_err(failure)?,
|
||||||
|
resolver_addresses: self.resolver_addresses.clone(),
|
||||||
|
network_probe: self.network_probe.clone(),
|
||||||
|
reachability: self.reachability.clone(),
|
||||||
|
container_id: self.container_id,
|
||||||
|
invoking_uid: rustix::process::getuid().as_raw(),
|
||||||
|
escalation_client: &self.escalation_client,
|
||||||
|
sandbox_init: &self.sandbox_init,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Selects and prepares every Host Coordinator-owned cold-start input.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
/// Refuses invalid installed paths, unsupported architectures, registry or image failures,
|
||||||
|
/// changed Storage Root authority, exhausted identities, and configuration-policy mismatch.
|
||||||
|
pub async fn select_runtime(
|
||||||
|
configuration: &ResolvedDomainConfiguration,
|
||||||
|
store: Arc<HostCoordinatorDomainStore>,
|
||||||
|
product_paths: InstalledProductPaths<'_>,
|
||||||
|
) -> Result<RuntimeSelection, RuntimeSelectionError> {
|
||||||
|
let kernel_release = std::fs::read_to_string("/proc/sys/kernel/osrelease").map_err(failure)?;
|
||||||
|
select_runtime_for_platform(
|
||||||
|
configuration,
|
||||||
|
store,
|
||||||
|
product_paths,
|
||||||
|
&PlatformObservation {
|
||||||
|
operating_system: std::env::consts::OS,
|
||||||
|
architecture: std::env::consts::ARCH,
|
||||||
|
kernel_release: kernel_release.trim(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn select_runtime_for_platform(
|
||||||
|
configuration: &ResolvedDomainConfiguration,
|
||||||
|
store: Arc<HostCoordinatorDomainStore>,
|
||||||
|
product_paths: InstalledProductPaths<'_>,
|
||||||
|
observation: &PlatformObservation<'_>,
|
||||||
|
) -> Result<RuntimeSelection, RuntimeSelectionError> {
|
||||||
|
check_platform(observation).map_err(failure)?;
|
||||||
|
let escalation_client = open_installed_executable(product_paths.escalation_client)?;
|
||||||
|
let sandbox_init = open_installed_executable(product_paths.sandbox_init)?;
|
||||||
|
let catalog = Catalog::under_storage_root(configuration.storage_root()).map_err(failure)?;
|
||||||
|
match catalog.initialize() {
|
||||||
|
Ok(()) | Err(CatalogError::AlreadyInitialized) => {}
|
||||||
|
Err(error) => return Err(failure(error)),
|
||||||
|
}
|
||||||
|
let prepared =
|
||||||
|
PreparedLowerStore::under_storage_root(configuration.storage_root(), catalog.clone())
|
||||||
|
.map_err(failure)?;
|
||||||
|
recover_before_cold_start(configuration, &store, &catalog, &prepared).map_err(failure)?;
|
||||||
|
let image = OciRegistryTransport
|
||||||
|
.pull_confined(
|
||||||
|
&configuration.fingerprint_record().image,
|
||||||
|
&RegistryCredentials::Anonymous,
|
||||||
|
&Platform::linux("amd64"),
|
||||||
|
configuration.storage_root(),
|
||||||
|
random_id()?,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| failure(format!("pulling configured anonymous OCI image: {error:#}")))?;
|
||||||
|
let artifact = prepared
|
||||||
|
.prepare_transported(
|
||||||
|
PrepareOperationId::new(random_id()?),
|
||||||
|
&image,
|
||||||
|
rootless_id_map(),
|
||||||
|
prepared_destinations(configuration),
|
||||||
|
)
|
||||||
|
.map_err(failure)?;
|
||||||
|
let artifact_authority = prepared
|
||||||
|
.open_artifact_authority(&artifact)
|
||||||
|
.map_err(failure)?;
|
||||||
|
let artifact_label = PathBuf::from(std::ffi::OsString::from_vec(
|
||||||
|
configuration
|
||||||
|
.storage_root()
|
||||||
|
.canonical_label()
|
||||||
|
.as_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
))
|
||||||
|
.join("prepared-lower")
|
||||||
|
.join(PathBuf::from(std::ffi::OsString::from_vec(
|
||||||
|
artifact.path.as_bytes().to_vec(),
|
||||||
|
)));
|
||||||
|
image.cleanup().map_err(failure)?;
|
||||||
|
|
||||||
|
let next_generation = next_generation(&store)?;
|
||||||
|
let (coordinator_id, network_sidecar_id, sandbox_id) = distinct_generation_ids()?;
|
||||||
|
let generation = Generation {
|
||||||
|
domain_id: configuration.domain_id(),
|
||||||
|
generation_id: GenerationId::new(next_generation),
|
||||||
|
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||||
|
coordinator_id,
|
||||||
|
network_sidecar_id,
|
||||||
|
sandbox_id,
|
||||||
|
durable_decision: DurableDecision::None,
|
||||||
|
};
|
||||||
|
let namespace = namespace_identity(Path::new(NETWORK_NAMESPACE_PATH)).map_err(failure)?;
|
||||||
|
let (resolver_addresses, network_probe, reachability) =
|
||||||
|
select_network_readiness(configuration, &generation, namespace);
|
||||||
|
Ok(RuntimeSelection {
|
||||||
|
store,
|
||||||
|
domain_id: configuration.domain_id(),
|
||||||
|
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||||
|
catalog,
|
||||||
|
artifact,
|
||||||
|
artifact_authority,
|
||||||
|
artifact_label,
|
||||||
|
generation,
|
||||||
|
resolver_addresses,
|
||||||
|
network_probe,
|
||||||
|
reachability,
|
||||||
|
container_id: ContainerId::new(random_id()?),
|
||||||
|
escalation_client,
|
||||||
|
sandbox_init,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prepared_destinations(configuration: &ResolvedDomainConfiguration) -> Vec<PreparedDestination> {
|
||||||
|
let mut destinations = [
|
||||||
|
"/dev",
|
||||||
|
"/proc",
|
||||||
|
"/sys",
|
||||||
|
"/tmp",
|
||||||
|
"/etc",
|
||||||
|
"/home",
|
||||||
|
"/run",
|
||||||
|
WORKSPACE_DESTINATION,
|
||||||
|
CREDENTIALS_DESTINATION,
|
||||||
|
MAILBOX_REQUESTS_DESTINATION,
|
||||||
|
MAILBOX_RESULTS_DESTINATION,
|
||||||
|
TERMINFO_ROOT_DESTINATION,
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(|path| PreparedDestination {
|
||||||
|
path: path.into(),
|
||||||
|
kind: PreparedDestinationKind::Directory,
|
||||||
|
})
|
||||||
|
.chain(
|
||||||
|
[
|
||||||
|
"/etc/resolv.conf",
|
||||||
|
"/etc/hosts",
|
||||||
|
"/etc/hostname",
|
||||||
|
ESCALATION_CLIENT_DESTINATION,
|
||||||
|
SANDBOX_INIT_DESTINATION,
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(|path| PreparedDestination {
|
||||||
|
path: path.into(),
|
||||||
|
kind: PreparedDestinationKind::RegularFile,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
destinations.extend(
|
||||||
|
configuration
|
||||||
|
.fingerprint_record()
|
||||||
|
.persistent_dirs
|
||||||
|
.iter()
|
||||||
|
.map(|path| PreparedDestination {
|
||||||
|
path: path.into(),
|
||||||
|
kind: PreparedDestinationKind::Directory,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
destinations.extend(
|
||||||
|
configuration
|
||||||
|
.volumes()
|
||||||
|
.iter()
|
||||||
|
.map(|volume| PreparedDestination {
|
||||||
|
path: volume.record().container_path.as_str().into(),
|
||||||
|
kind: match volume.record().kind {
|
||||||
|
VolumeKind::Directory => PreparedDestinationKind::Directory,
|
||||||
|
VolumeKind::File => PreparedDestinationKind::RegularFile,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
destinations
|
||||||
|
}
|
||||||
|
|
||||||
|
fn next_generation(store: &HostCoordinatorDomainStore) -> Result<u64, RuntimeSelectionError> {
|
||||||
|
store
|
||||||
|
.snapshot()
|
||||||
|
.map_err(failure)?
|
||||||
|
.generations
|
||||||
|
.iter()
|
||||||
|
.map(|generation| generation.generation_id.get())
|
||||||
|
.max()
|
||||||
|
.unwrap_or(0)
|
||||||
|
.checked_add(1)
|
||||||
|
.ok_or_else(|| failure("generation identity exhausted"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn select_network_readiness(
|
||||||
|
configuration: &ResolvedDomainConfiguration,
|
||||||
|
generation: &Generation,
|
||||||
|
enforcement_namespace: lazaret_core::identity::NamespaceIdentity,
|
||||||
|
) -> (Vec<IpAddress>, NetworkProbeRequest, ReachabilityExpectation) {
|
||||||
|
let readiness = configuration.network_readiness();
|
||||||
|
let denied = readiness
|
||||||
|
.denied
|
||||||
|
.iter()
|
||||||
|
.map(|target| TcpProbeTarget {
|
||||||
|
address: IpAddress::V4(target.address),
|
||||||
|
port: target.port,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let permitted = readiness
|
||||||
|
.permitted
|
||||||
|
.iter()
|
||||||
|
.map(|target| TcpProbeTarget {
|
||||||
|
address: IpAddress::V4(target.address),
|
||||||
|
port: target.port,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let dns = readiness
|
||||||
|
.dns
|
||||||
|
.iter()
|
||||||
|
.map(|target| DnsProbeTarget {
|
||||||
|
resolver: IpAddress::V4(target.resolver),
|
||||||
|
query_name: target.query.clone(),
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let reachability = ReachabilityExpectation {
|
||||||
|
denied_destinations: denied.iter().map(|target| target.address).collect(),
|
||||||
|
permitted_destinations: permitted.iter().map(|target| target.address).collect(),
|
||||||
|
dns_queries: dns
|
||||||
|
.iter()
|
||||||
|
.map(|target| DnsQueryExpectation {
|
||||||
|
resolver: target.resolver,
|
||||||
|
query_name: target.query_name.clone(),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
};
|
||||||
|
let resolvers = dns
|
||||||
|
.iter()
|
||||||
|
.map(|target| target.resolver)
|
||||||
|
.collect::<BTreeSet<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.collect();
|
||||||
|
(
|
||||||
|
resolvers,
|
||||||
|
NetworkProbeRequest {
|
||||||
|
domain_id: generation.domain_id,
|
||||||
|
generation_id: generation.generation_id,
|
||||||
|
enforcement_namespace,
|
||||||
|
denied,
|
||||||
|
permitted,
|
||||||
|
dns,
|
||||||
|
},
|
||||||
|
reachability,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_installed_executable(path: &Path) -> Result<File, RuntimeSelectionError> {
|
||||||
|
let descriptor = openat2(
|
||||||
|
rustix::fs::CWD,
|
||||||
|
path,
|
||||||
|
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||||
|
Mode::empty(),
|
||||||
|
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
|
||||||
|
)
|
||||||
|
.map_err(failure)?;
|
||||||
|
let file = File::from(descriptor);
|
||||||
|
let metadata = file.metadata().map_err(failure)?;
|
||||||
|
if !metadata.is_file() || metadata.permissions().mode() & 0o111 == 0 {
|
||||||
|
return Err(failure(
|
||||||
|
"installed product path is not an executable regular file",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rootless_id_map() -> IdMap {
|
||||||
|
IdMap {
|
||||||
|
uid: vec![IdMapRange {
|
||||||
|
container_id: 0,
|
||||||
|
namespace_id: rustix::process::getuid().as_raw(),
|
||||||
|
size: 1,
|
||||||
|
}],
|
||||||
|
gid: vec![IdMapRange {
|
||||||
|
container_id: 0,
|
||||||
|
namespace_id: rustix::process::getgid().as_raw(),
|
||||||
|
size: 1,
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn distinct_generation_ids()
|
||||||
|
-> Result<(CoordinatorId, NetworkSidecarId, SandboxId), RuntimeSelectionError> {
|
||||||
|
for _ in 0..8 {
|
||||||
|
let coordinator = CoordinatorId::new(random_id()?);
|
||||||
|
let network = NetworkSidecarId::new(random_id()?);
|
||||||
|
let sandbox = SandboxId::new(random_id()?);
|
||||||
|
if coordinator.as_bytes() != network.as_bytes()
|
||||||
|
&& coordinator.as_bytes() != sandbox.as_bytes()
|
||||||
|
&& network.as_bytes() != sandbox.as_bytes()
|
||||||
|
{
|
||||||
|
return Ok((coordinator, network, sandbox));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(failure(
|
||||||
|
"fresh generation role identities repeatedly collided",
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn random_id() -> Result<[u8; 32], RuntimeSelectionError> {
|
||||||
|
let mut bytes = [0; 32];
|
||||||
|
getrandom::fill(&mut bytes).map_err(failure)?;
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fs::{self, OpenOptions};
|
||||||
|
use std::future::Future;
|
||||||
|
use std::os::unix::ffi::OsStringExt;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::task::{Context, Poll, Waker};
|
||||||
|
|
||||||
|
use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration};
|
||||||
|
use lazaret_core::identity::{DeviceId, InodeId, NamespaceIdentity};
|
||||||
|
use lazaret_core::records::Domain;
|
||||||
|
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||||
|
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
|
||||||
|
|
||||||
|
use crate::domain_store::DomainStoreMutation;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolved_readiness_maps_exactly_and_rootless_id_map_matches_builder_policy() {
|
||||||
|
let tree = tempfile::tempdir().unwrap();
|
||||||
|
let configuration = configuration(tree.path());
|
||||||
|
let generation = Generation {
|
||||||
|
domain_id: configuration.domain_id(),
|
||||||
|
generation_id: GenerationId::new(9),
|
||||||
|
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||||
|
coordinator_id: CoordinatorId::new([1; 32]),
|
||||||
|
network_sidecar_id: NetworkSidecarId::new([2; 32]),
|
||||||
|
sandbox_id: SandboxId::new([3; 32]),
|
||||||
|
durable_decision: DurableDecision::None,
|
||||||
|
};
|
||||||
|
let namespace = NamespaceIdentity {
|
||||||
|
device: DeviceId::new(4),
|
||||||
|
inode: InodeId::new(5),
|
||||||
|
};
|
||||||
|
let (resolvers, request, expectation) =
|
||||||
|
select_network_readiness(&configuration, &generation, namespace);
|
||||||
|
assert_eq!(resolvers, [IpAddress::V4([198, 51, 100, 53])]);
|
||||||
|
assert_eq!(request.denied[0].port, 8443);
|
||||||
|
assert_eq!(request.permitted[0].port, 443);
|
||||||
|
assert_eq!(request.dns[0].query_name, "example.invalid");
|
||||||
|
assert_eq!(expectation.denied_destinations, [request.denied[0].address]);
|
||||||
|
assert_eq!(
|
||||||
|
expectation.permitted_destinations,
|
||||||
|
[request.permitted[0].address]
|
||||||
|
);
|
||||||
|
assert_eq!(expectation.dns_queries[0].resolver, request.dns[0].resolver);
|
||||||
|
|
||||||
|
let id_map = rootless_id_map();
|
||||||
|
assert_eq!(id_map.uid[0].container_id, 0);
|
||||||
|
assert_eq!(
|
||||||
|
id_map.uid[0].namespace_id,
|
||||||
|
rustix::process::getuid().as_raw()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
id_map.gid[0].namespace_id,
|
||||||
|
rustix::process::getgid().as_raw()
|
||||||
|
);
|
||||||
|
let destinations = prepared_destinations(&configuration);
|
||||||
|
assert!(destinations.iter().any(|destination| {
|
||||||
|
destination.path == Path::new(WORKSPACE_DESTINATION)
|
||||||
|
&& destination.kind == PreparedDestinationKind::Directory
|
||||||
|
}));
|
||||||
|
assert!(destinations.iter().any(|destination| {
|
||||||
|
destination.path == Path::new(SANDBOX_INIT_DESTINATION)
|
||||||
|
&& destination.kind == PreparedDestinationKind::RegularFile
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn installed_product_selection_requires_exact_executable_and_fresh_roles() {
|
||||||
|
let tree = tempfile::tempdir().unwrap();
|
||||||
|
let executable = tree.path().join("product");
|
||||||
|
fs::copy("/bin/true", &executable).unwrap();
|
||||||
|
fs::set_permissions(&executable, fs::Permissions::from_mode(0o700)).unwrap();
|
||||||
|
assert!(open_installed_executable(&executable).is_ok());
|
||||||
|
fs::set_permissions(&executable, fs::Permissions::from_mode(0o600)).unwrap();
|
||||||
|
assert!(open_installed_executable(&executable).is_err());
|
||||||
|
let (coordinator, network, sandbox) = distinct_generation_ids().unwrap();
|
||||||
|
assert_ne!(coordinator.as_bytes(), network.as_bytes());
|
||||||
|
assert_ne!(coordinator.as_bytes(), sandbox.as_bytes());
|
||||||
|
assert_ne!(network.as_bytes(), sandbox.as_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unsupported_platforms_refuse_before_installed_or_storage_side_effects() {
|
||||||
|
for observation in [
|
||||||
|
PlatformObservation {
|
||||||
|
operating_system: "linux",
|
||||||
|
architecture: "x86_64",
|
||||||
|
kernel_release: "6.8.14",
|
||||||
|
},
|
||||||
|
PlatformObservation {
|
||||||
|
operating_system: "linux",
|
||||||
|
architecture: "aarch64",
|
||||||
|
kernel_release: "6.9.0",
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
let tree = tempfile::tempdir().unwrap();
|
||||||
|
let configuration = configuration(tree.path());
|
||||||
|
let store = domain_store(&configuration);
|
||||||
|
let mut future = Box::pin(select_runtime_for_platform(
|
||||||
|
&configuration,
|
||||||
|
store,
|
||||||
|
InstalledProductPaths {
|
||||||
|
escalation_client: Path::new("/missing-escalation-client"),
|
||||||
|
sandbox_init: Path::new("/missing-sandbox-init"),
|
||||||
|
},
|
||||||
|
&observation,
|
||||||
|
));
|
||||||
|
let mut context = Context::from_waker(Waker::noop());
|
||||||
|
let Poll::Ready(result) = future.as_mut().poll(&mut context) else {
|
||||||
|
panic!("platform refusal performed asynchronous work");
|
||||||
|
};
|
||||||
|
assert!(result.is_err());
|
||||||
|
let storage = tree.path().join("data");
|
||||||
|
assert!(!storage.join("storage-catalog.lock").exists());
|
||||||
|
assert!(!storage.join("catalog.redb").exists());
|
||||||
|
assert!(!storage.join("prepared-lower").exists());
|
||||||
|
assert!(!storage.join("image-transports").exists());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn generation_selection_and_cold_start_retain_one_exact_domain_store() {
|
||||||
|
let tree = tempfile::tempdir().unwrap();
|
||||||
|
let configuration = configuration(tree.path());
|
||||||
|
let first_root = tree.path().join("store-a");
|
||||||
|
let second_root = tree.path().join("store-b");
|
||||||
|
fs::create_dir(&first_root).unwrap();
|
||||||
|
fs::create_dir(&second_root).unwrap();
|
||||||
|
let first = domain_store_at(&configuration, &first_root);
|
||||||
|
let second = domain_store_at(&configuration, &second_root);
|
||||||
|
second
|
||||||
|
.apply(&[DomainStoreMutation::PutGeneration(Generation {
|
||||||
|
domain_id: configuration.domain_id(),
|
||||||
|
generation_id: GenerationId::new(1),
|
||||||
|
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||||
|
coordinator_id: CoordinatorId::new([1; 32]),
|
||||||
|
network_sidecar_id: NetworkSidecarId::new([2; 32]),
|
||||||
|
sandbox_id: SandboxId::new([3; 32]),
|
||||||
|
durable_decision: DurableDecision::None,
|
||||||
|
})])
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(next_generation(&first).unwrap(), 1);
|
||||||
|
assert_eq!(next_generation(&second).unwrap(), 2);
|
||||||
|
assert!(!Arc::ptr_eq(&first, &second));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn domain_store(
|
||||||
|
configuration: &ResolvedDomainConfiguration,
|
||||||
|
) -> Arc<HostCoordinatorDomainStore> {
|
||||||
|
let root = PathBuf::from(std::ffi::OsString::from_vec(
|
||||||
|
configuration
|
||||||
|
.persistent_domain_root()
|
||||||
|
.canonical_label()
|
||||||
|
.as_bytes()
|
||||||
|
.to_vec(),
|
||||||
|
));
|
||||||
|
domain_store_at(configuration, &root)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn domain_store_at(
|
||||||
|
configuration: &ResolvedDomainConfiguration,
|
||||||
|
root: &Path,
|
||||||
|
) -> Arc<HostCoordinatorDomainStore> {
|
||||||
|
let lock = OpenOptions::new()
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(root.join("domain.lock"))
|
||||||
|
.unwrap();
|
||||||
|
Arc::new(
|
||||||
|
HostCoordinatorDomainStore::open(
|
||||||
|
try_exclusive_flock(lock).unwrap(),
|
||||||
|
&root.join("domain.redb"),
|
||||||
|
&Domain {
|
||||||
|
domain_id: configuration.domain_id(),
|
||||||
|
profile_name: configuration.selected_profile().to_owned(),
|
||||||
|
canonical_workspace: configuration.workspace().canonical_label().clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn configuration(root: &Path) -> ResolvedDomainConfiguration {
|
||||||
|
let configuration_directory = root.join("configuration");
|
||||||
|
let workspace = root.join("workspace");
|
||||||
|
let runtime = root.join("runtime");
|
||||||
|
fs::create_dir(&configuration_directory).unwrap();
|
||||||
|
fs::create_dir(&workspace).unwrap();
|
||||||
|
fs::create_dir(&runtime).unwrap();
|
||||||
|
let configuration_path = configuration_directory.join("lazaret.toml");
|
||||||
|
fs::write(
|
||||||
|
&configuration_path,
|
||||||
|
format!(
|
||||||
|
"schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 8443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"Example.Invalid.\" }}]\n",
|
||||||
|
root.join("data").display()
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
load_trusted_configuration(ConfigurationLoadRequest {
|
||||||
|
explicit_configuration_path: Some(configuration_path),
|
||||||
|
selected_profile: "default".into(),
|
||||||
|
workspace,
|
||||||
|
home: root.join("home"),
|
||||||
|
xdg_config_home: None,
|
||||||
|
xdg_data_home: None,
|
||||||
|
xdg_runtime_dir: runtime,
|
||||||
|
environment: BTreeMap::new(),
|
||||||
|
sandbox_layout: default_sandbox_layout().unwrap(),
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -700,7 +700,7 @@ fn configuration(tree: &TestTree) -> lazaret_core::configuration::ResolvedDomain
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let layout = SandboxLayoutV1::from_production_layout([
|
let layout = SandboxLayoutV1::from_protected_destinations([
|
||||||
"/workspace".to_owned(),
|
"/workspace".to_owned(),
|
||||||
"/proc".to_owned(),
|
"/proc".to_owned(),
|
||||||
"/sys".to_owned(),
|
"/sys".to_owned(),
|
||||||
|
|
|
||||||
|
|
@ -270,7 +270,7 @@ mod tests {
|
||||||
};
|
};
|
||||||
use lazaret_core::records::{Domain, Generation};
|
use lazaret_core::records::{Domain, Generation};
|
||||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
|
||||||
use rustix::fs::{FileType, Mode, XattrFlags, fsetxattr, makedev, mknodat};
|
use rustix::fs::{FileType, Mode, XattrFlags, fsetxattr, makedev, mknodat};
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
@ -491,7 +491,7 @@ mod tests {
|
||||||
xdg_data_home: None,
|
xdg_data_home: None,
|
||||||
xdg_runtime_dir: runtime,
|
xdg_runtime_dir: runtime,
|
||||||
environment: BTreeMap::new(),
|
environment: BTreeMap::new(),
|
||||||
sandbox_layout: production_sandbox_layout().unwrap(),
|
sandbox_layout: default_sandbox_layout().unwrap(),
|
||||||
})
|
})
|
||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -544,7 +544,7 @@ fn engine_for(fixture: &Fixture) -> (Arc<MockFactory>, SessionTerminalEngine) {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[allow(clippy::too_many_lines)]
|
#[allow(clippy::too_many_lines)]
|
||||||
fn production_prompt_adapter_retains_resume_resize_before_authenticated_denial() {
|
fn prompt_adapter_retains_resume_resize_before_authenticated_denial() {
|
||||||
let generation = GenerationIdentity {
|
let generation = GenerationIdentity {
|
||||||
agent_profile: "trusted".into(),
|
agent_profile: "trusted".into(),
|
||||||
domain_id: DomainId::new(bytes(1)),
|
domain_id: DomainId::new(bytes(1)),
|
||||||
|
|
|
||||||
|
|
@ -436,7 +436,7 @@ mod tests {
|
||||||
use lazaret_core::identity::{CoordinatorId, NetworkSidecarId, SandboxId};
|
use lazaret_core::identity::{CoordinatorId, NetworkSidecarId, SandboxId};
|
||||||
use lazaret_core::records::{Domain, DurableDecision, Generation};
|
use lazaret_core::records::{Domain, DurableDecision, Generation};
|
||||||
use lazaret_linux::filesystem::try_exclusive_flock;
|
use lazaret_linux::filesystem::try_exclusive_flock;
|
||||||
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
|
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
|
||||||
|
|
||||||
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
||||||
use crate::lifecycle::{LifecycleAction, SessionAdmissionIdentity};
|
use crate::lifecycle::{LifecycleAction, SessionAdmissionIdentity};
|
||||||
|
|
@ -757,7 +757,7 @@ mod tests {
|
||||||
xdg_data_home: None,
|
xdg_data_home: None,
|
||||||
xdg_runtime_dir: runtime,
|
xdg_runtime_dir: runtime,
|
||||||
environment: BTreeMap::new(),
|
environment: BTreeMap::new(),
|
||||||
sandbox_layout: production_sandbox_layout().unwrap(),
|
sandbox_layout: default_sandbox_layout().unwrap(),
|
||||||
})
|
})
|
||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue