ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Remove phase vocabulary from Lazaret interfaces

Browse source
This commit is contained in:
ms2k 2026-07-18 22:35:48 +09:00
commit e7dd95aea6
16 changed files with 547 additions and 579 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/lazaret-host-coordinator/Cargo.toml
View file

@ -49,8 +49,8 @@ path = "tests/escalation.rs"
harness = false
[[test]]
name = "production"
path = "tests/production.rs"
name = "cold-start"
path = "tests/cold_start.rs"
[lints]
workspace = true

172
crates/lazaret-host-coordinator/src/bootstrap.rs
View file

@ -25,9 +25,7 @@ use crate::control::{
TrustAnchor,
};
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::escalation::{
ProductionEscalation, ProductionEscalationHandle, ProductionEscalationInputs,
};
use crate::escalation::{EscalationService, EscalationServiceHandle, EscalationServiceInputs};
use crate::git_routing::GitRoutingCoordinator;
use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, SessionAdmissionIdentity,
@ -42,27 +40,27 @@ use crate::state_anchor::{StateAnchorCoordinator, StateAnchorReleaseAuthority};
use crate::terminal::SessionPromptTransport;
#[derive(Debug)]
pub struct ProductionLifecycleError(String);
pub struct LifecycleServiceError(String);
impl fmt::Display for ProductionLifecycleError {
impl fmt::Display for LifecycleServiceError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
impl std::error::Error for ProductionLifecycleError {}
impl std::error::Error for LifecycleServiceError {}
impl ProductionLifecycleError {
impl LifecycleServiceError {
pub(crate) fn service(error: impl fmt::Display) -> Self {
Self(error.to_string())
}
}
fn failure(error: impl fmt::Display) -> ProductionLifecycleError {
ProductionLifecycleError(error.to_string())
fn failure(error: impl fmt::Display) -> LifecycleServiceError {
LifecycleServiceError(error.to_string())
}
pub struct ProductionBootstrapPlan<'a> {
pub struct BootstrapPlan<'a> {
pub store: Arc<HostCoordinatorDomainStore>,
pub catalog: &'a Catalog,
pub configuration: &'a ResolvedDomainConfiguration,
@ -81,24 +79,24 @@ pub struct ProductionBootstrapPlan<'a> {
pub non_interactive_executor: Arc<dyn NonInteractiveSessionExecutor>,
pub interactive_executor: Arc<dyn InteractiveSessionExecutor>,
pub prompt_transport: Arc<SessionPromptTransport>,
pub(crate) runtime_staging: &'a crate::production::RuntimeStaging,
pub git_routing: Option<ProductionGitRouting>,
pub(crate) runtime_staging: &'a crate::cold_start::RuntimeStaging,
pub git_routing: Option<BootstrapGitRouting>,
}
pub struct ProductionGitRouting {
pub struct BootstrapGitRouting {
pub mountpoint: HostPath,
pub host_root: HostPath,
pub isolated_root: HostPath,
}
pub struct ProductionLifecycleAdapter<'a> {
plan: ProductionBootstrapPlan<'a>,
pub struct CoordinatorLifecycleAdapter<'a> {
plan: BootstrapPlan<'a>,
registry: Arc<SessionRegistry>,
trust_anchor: Option<TrustAnchor>,
staged_control: Option<StagedControlFace>,
published_control: Option<Arc<PublishedControlFace>>,
escalation: Option<ProductionEscalation<'a>>,
escalation_handle: Option<ProductionEscalationHandle>,
escalation: Option<EscalationService<'a>>,
escalation_handle: Option<EscalationServiceHandle>,
awaiting_runtime: Option<AwaitingAttachedRunsc>,
runtime: Option<CommittedSandboxRuntime>,
state_anchor_mount: Option<MountIdentity>,
@ -107,43 +105,43 @@ pub struct ProductionLifecycleAdapter<'a> {
state_anchor_absence: Option<StateAnchorAbsenceObservation>,
}
pub type ProductionHostCoordinatorLifecycle<'store, 'resources> = HostCoordinatorLifecycle<
pub type CoordinatorLifecycle<'store, 'resources> = HostCoordinatorLifecycle<
'store,
ProductionLifecycleAdapter<'resources>,
ProductionSessionRegistry,
CoordinatorLifecycleAdapter<'resources>,
CoordinatorSessionRegistry,
>;
/// Composes the cold-start lifecycle with the exact shared production session registry.
/// Composes the cold-start lifecycle with the exact shared session registry.
///
/// # Errors
/// Refuses a plan whose Domain Store is not the caller-retained lock-owning store.
pub fn production_lifecycle<'store, 'resources>(
pub fn coordinator_lifecycle<'store, 'resources>(
store: &'store Arc<HostCoordinatorDomainStore>,
plan: ProductionBootstrapPlan<'resources>,
) -> Result<ProductionHostCoordinatorLifecycle<'store, 'resources>, ProductionLifecycleError> {
plan: BootstrapPlan<'resources>,
) -> Result<CoordinatorLifecycle<'store, 'resources>, LifecycleServiceError> {
if !Arc::ptr_eq(store, &plan.store) {
return Err(ProductionLifecycleError(
"production bootstrap Domain Store authority differs".into(),
return Err(LifecycleServiceError(
"bootstrap Domain Store authority differs".into(),
));
}
let (adapter, registry) = ProductionLifecycleAdapter::new(plan);
let (adapter, registry) = CoordinatorLifecycleAdapter::new(plan);
Ok(HostCoordinatorLifecycle::new(store, adapter, registry))
}
#[derive(Clone)]
pub struct ProductionSessionRegistry {
pub struct CoordinatorSessionRegistry {
registry: Arc<SessionRegistry>,
}
impl ProductionSessionRegistry {
impl CoordinatorSessionRegistry {
#[must_use]
pub fn registry(&self) -> Arc<SessionRegistry> {
Arc::clone(&self.registry)
}
}
impl SessionLifecycleAdapter for ProductionSessionRegistry {
type Error = ProductionLifecycleError;
impl SessionLifecycleAdapter for CoordinatorSessionRegistry {
type Error = LifecycleServiceError;
type Admission = ControlSessionAdmission;
fn admission_identity(&self, admission: &Self::Admission) -> SessionAdmissionIdentity {
@ -182,9 +180,9 @@ impl SessionLifecycleAdapter for ProductionSessionRegistry {
}
}
impl<'a> ProductionLifecycleAdapter<'a> {
impl<'a> CoordinatorLifecycleAdapter<'a> {
#[must_use]
pub fn new(plan: ProductionBootstrapPlan<'a>) -> (Self, ProductionSessionRegistry) {
pub fn new(plan: BootstrapPlan<'a>) -> (Self, CoordinatorSessionRegistry) {
let registry = Arc::new(SessionRegistry::new(
Arc::clone(&plan.store),
plan.generation_identity.clone(),
@ -205,7 +203,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
state_anchor_release: None,
state_anchor_absence: None,
},
ProductionSessionRegistry { registry },
CoordinatorSessionRegistry { registry },
)
}
@ -216,11 +214,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
pub(crate) fn take_escalation(
&mut self,
) -> Option<(ProductionEscalation<'a>, ProductionEscalationHandle)> {
) -> Option<(EscalationService<'a>, EscalationServiceHandle)> {
Some((self.escalation.take()?, self.escalation_handle.take()?))
}
fn require_generation(&self, generation: &Generation) -> Result<(), ProductionLifecycleError> {
fn require_generation(&self, generation: &Generation) -> Result<(), LifecycleServiceError> {
if generation.domain_id == self.plan.generation_identity.domain_id
&& generation.generation_id == self.plan.generation_identity.generation_id
&& generation.configuration_fingerprint
@ -230,8 +228,8 @@ impl<'a> ProductionLifecycleAdapter<'a> {
{
Ok(())
} else {
Err(ProductionLifecycleError(
"production bootstrap generation identity changed".into(),
Err(LifecycleServiceError(
"bootstrap generation identity changed".into(),
))
}
}
@ -252,18 +250,16 @@ impl<'a> ProductionLifecycleAdapter<'a> {
StateAnchorCoordinator::new(self.store(), self.plan.catalog)
}
fn create_trust_anchor(&mut self) -> Result<(), ProductionLifecycleError> {
fn create_trust_anchor(&mut self) -> Result<(), LifecycleServiceError> {
if self.trust_anchor.is_some() {
return Err(ProductionLifecycleError(
"Trust Anchor already exists".into(),
));
return Err(LifecycleServiceError("Trust Anchor already exists".into()));
}
self.trust_anchor =
Some(TrustAnchor::create(self.plan.generation_identity.clone()).map_err(failure)?);
Ok(())
}
fn start_runtime_supervisor(&mut self) -> Result<(), ProductionLifecycleError> {
fn start_runtime_supervisor(&mut self) -> Result<(), LifecycleServiceError> {
let sandbox = self.sandbox();
sandbox
.begin_intent(
@ -286,7 +282,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Ok(())
}
fn lease_prepared_lower(&self) -> Result<(), ProductionLifecycleError> {
fn lease_prepared_lower(&self) -> Result<(), LifecycleServiceError> {
self.state_anchor()
.begin_acquisition(
self.plan.generation_identity.generation_id,
@ -297,13 +293,13 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.map_err(failure)
}
fn mount_state_anchor(&mut self) -> Result<(), ProductionLifecycleError> {
fn mount_state_anchor(&mut self) -> Result<(), LifecycleServiceError> {
let store = Arc::clone(&self.plan.store);
let coordinator = StateAnchorCoordinator::new(&store, self.plan.catalog);
let mount = self
.awaiting_runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
.mount_state_anchor(
&coordinator,
self.plan.prepared_artifact.prepared_artifact_id,
@ -323,7 +319,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
let git_mount = self
.awaiting_runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
.start_git_routing(&git_routing)
.map_err(|error| failure(format!("starting Git routing: {error}")))?;
self.git_routing_mount = Some(git_mount);
@ -331,32 +327,32 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Ok(())
}
fn start_connectivity_helper(&mut self) -> Result<(), ProductionLifecycleError> {
fn start_connectivity_helper(&mut self) -> Result<(), LifecycleServiceError> {
let store = Arc::clone(&self.plan.store);
let network = NetworkCoordinator::new(&store);
self.awaiting_runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
.receive_helper_staged(&network)
.map(|_| ())
.map_err(|error| failure(format!("starting connectivity helper: {error}")))
}
fn start_sidecar(&mut self) -> Result<(), ProductionLifecycleError> {
fn start_sidecar(&mut self) -> Result<(), LifecycleServiceError> {
self.awaiting_runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
.start_sidecar()
.map_err(|error| failure(format!("starting sidecar: {error}")))
}
fn mark_network_ready(&mut self) -> Result<(), ProductionLifecycleError> {
fn mark_network_ready(&mut self) -> Result<(), LifecycleServiceError> {
let store = Arc::clone(&self.plan.store);
let network = NetworkCoordinator::new(&store);
let mut probe = self.plan.network_probe.clone();
self.awaiting_runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?
.receive_network_readiness(
&network,
self.plan.network_policy,
@ -370,13 +366,13 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.map_err(|error| failure(format!("marking network ready: {error}")))
}
fn start_sandbox(&mut self) -> Result<(), ProductionLifecycleError> {
fn start_sandbox(&mut self) -> Result<(), LifecycleServiceError> {
let store = Arc::clone(&self.plan.store);
let sandbox = SandboxCoordinator::new(&store);
let mut pending = self
.awaiting_runtime
.take()
.ok_or_else(|| ProductionLifecycleError("Runtime Supervisor is absent".into()))?;
.ok_or_else(|| LifecycleServiceError("Runtime Supervisor is absent".into()))?;
let staged = pending
.receive_staged(&sandbox)
.map_err(|error| failure(format!("receiving attached runsc staging: {error}")))?;
@ -394,11 +390,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Ok(())
}
fn stage_publication(&mut self) -> Result<(), ProductionLifecycleError> {
fn stage_publication(&mut self) -> Result<(), LifecycleServiceError> {
let mut anchor = self
.trust_anchor
.take()
.ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?;
.ok_or_else(|| LifecycleServiceError("Trust Anchor is absent".into()))?;
if self
.plan
.configuration
@ -421,7 +417,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
lazaret_core::identity::RequestId::new([0; 32]),
),
};
let (escalation, handle) = ProductionEscalation::new(ProductionEscalationInputs {
let (escalation, handle) = EscalationService::new(EscalationServiceInputs {
configuration: self.plan.configuration,
publication,
credentials,
@ -476,11 +472,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Ok(())
}
fn expose_publication(&mut self) -> Result<(), ProductionLifecycleError> {
fn expose_publication(&mut self) -> Result<(), LifecycleServiceError> {
let published = self
.staged_control
.take()
.ok_or_else(|| ProductionLifecycleError("staged control face is absent".into()))?
.ok_or_else(|| LifecycleServiceError("staged control face is absent".into()))?
.expose()
.map_err(failure)?;
let ownership = published.recovery_ownership().map_err(failure)?;
@ -500,7 +496,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Ok(())
}
fn clean_publication_after_withdrawal(&self) -> Result<(), ProductionLifecycleError> {
fn clean_publication_after_withdrawal(&self) -> Result<(), LifecycleServiceError> {
let generation_id = self.plan.generation_identity.generation_id;
let current = self
.plan
@ -513,7 +509,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::Publication
})
.ok_or_else(|| ProductionLifecycleError("Publication participant is absent".into()))?;
.ok_or_else(|| LifecycleServiceError("Publication participant is absent".into()))?;
let release = Participant::snapshot(
generation_id,
ParticipantRole::Publication,
@ -537,19 +533,19 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.map_err(failure)
}
fn stop_sandbox(&mut self) -> Result<(), ProductionLifecycleError> {
fn stop_sandbox(&mut self) -> Result<(), LifecycleServiceError> {
let store = Arc::clone(&self.plan.store);
let sandbox = SandboxCoordinator::new(&store);
self.runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
.stop_sandbox(&sandbox)
.map_err(failure)?;
if self.plan.git_routing.is_some() {
let git_routing = GitRoutingCoordinator::new(&store);
self.runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
.stop_git_routing(&git_routing)
.map_err(failure)?;
self.git_routing_mount = None;
@ -557,7 +553,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
self.plan.runtime_staging.cleanup_overlay().map_err(failure)
}
fn stop_sidecar(&mut self) -> Result<(), ProductionLifecycleError> {
fn stop_sidecar(&mut self) -> Result<(), LifecycleServiceError> {
let store = Arc::clone(&self.plan.store);
let network = NetworkCoordinator::new(&store);
network
@ -565,7 +561,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.map_err(failure)?;
self.runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
.stop_sidecar()
.map_err(failure)?;
network
@ -573,7 +569,7 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.map_err(failure)
}
fn unmount_state_anchor(&mut self) -> Result<(), ProductionLifecycleError> {
fn unmount_state_anchor(&mut self) -> Result<(), LifecycleServiceError> {
let coordinator = self.state_anchor();
self.state_anchor_release = Some(
coordinator
@ -586,11 +582,11 @@ impl<'a> ProductionLifecycleAdapter<'a> {
let runtime = self
.runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?;
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?;
let release = self
.state_anchor_release
.as_ref()
.ok_or_else(|| ProductionLifecycleError("State Anchor release is absent".into()))?;
.ok_or_else(|| LifecycleServiceError("State Anchor release is absent".into()))?;
runtime
.unmount_state_anchor(release.mount())
.map_err(failure)?;
@ -607,18 +603,18 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Ok(())
}
fn release_prepared_lower(&mut self) -> Result<(), ProductionLifecycleError> {
fn release_prepared_lower(&mut self) -> Result<(), LifecycleServiceError> {
let release = self
.state_anchor_release
.as_ref()
.ok_or_else(|| ProductionLifecycleError("State Anchor release is absent".into()))?;
.ok_or_else(|| LifecycleServiceError("State Anchor release is absent".into()))?;
let first_observation = self.state_anchor_absence.ok_or_else(|| {
ProductionLifecycleError("State Anchor absence evidence is absent".into())
LifecycleServiceError("State Anchor absence evidence is absent".into())
})?;
let runtime = self
.runtime
.as_mut()
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?;
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?;
let coordinator = StateAnchorCoordinator::new(&self.plan.store, self.plan.catalog);
coordinator
.complete_release_after_unmount(
@ -650,17 +646,17 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Ok(())
}
fn graceful_exit(&mut self) -> Result<(), ProductionLifecycleError> {
fn graceful_exit(&mut self) -> Result<(), LifecycleServiceError> {
self.runtime
.take()
.ok_or_else(|| ProductionLifecycleError("committed runtime is absent".into()))?
.ok_or_else(|| LifecycleServiceError("committed runtime is absent".into()))?
.graceful_exit()
.map_err(failure)
}
}
impl LifecycleAdapter for ProductionLifecycleAdapter<'_> {
type Error = ProductionLifecycleError;
impl LifecycleAdapter for CoordinatorLifecycleAdapter<'_> {
type Error = LifecycleServiceError;
fn execute(
&mut self,
@ -721,13 +717,11 @@ impl LifecycleAdapter for ProductionLifecycleAdapter<'_> {
}
}
/// Serves one production control connection through the exact retained lifecycle owner.
/// Serves one control connection through the exact retained lifecycle owner.
///
/// # Errors
/// Refuses absent publication, poisoned lifecycle ownership, or control/session failure.
pub fn serve_production_control(
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
) -> Result<(), ControlError> {
pub fn serve_control(lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>) -> Result<(), ControlError> {
let face = lifecycle
.lock()
.map_err(|_| ControlError::LockPoisoned)?
@ -735,22 +729,22 @@ pub fn serve_production_control(
.published_control()
.ok_or(ControlError::StaleGeneration)?;
let connection = face.accept_connection()?;
serve_production_control_connection(lifecycle, &face, connection)
serve_control_connection(lifecycle, &face, connection)
}
pub(crate) fn serve_production_control_connection(
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
pub(crate) fn serve_control_connection(
lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>,
face: &PublishedControlFace,
connection: AcceptedControlConnection,
) -> Result<(), ControlError> {
serve_production_control_connection_with_escalation(lifecycle, face, connection, None)
serve_control_connection_with_escalation(lifecycle, face, connection, None)
}
pub(crate) fn serve_production_control_connection_with_escalation(
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
pub(crate) fn serve_control_connection_with_escalation(
lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>,
face: &PublishedControlFace,
connection: AcceptedControlConnection,
escalation: Option<&ProductionEscalationHandle>,
escalation: Option<&EscalationServiceHandle>,
) -> Result<(), ControlError> {
face.serve_connection_coordinated(
connection,

85
crates/lazaret-host-coordinator/src/client_bootstrap.rs
View file

@ -17,17 +17,15 @@ use lazaret_core::identity::{
};
use lazaret_core::records::{Domain, Session, SessionPhase};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
use rustix::fs::{Mode, OFlags, openat};
use crate::bootstrap::{
ProductionHostCoordinatorLifecycle, serve_production_control_connection_with_escalation,
};
use crate::bootstrap::{CoordinatorLifecycle, serve_control_connection_with_escalation};
use crate::cold_start::{BootstrapOwnerGuard, ColdStart, ColdStartError};
use crate::control::ControlError;
use crate::domain_store::HostCoordinatorDomainStore;
use crate::escalation::{ProductionEscalation, ProductionEscalationHandle};
use crate::production::{BootstrapOwnerGuard, ProductionColdStart, ProductionColdStartError};
use crate::production_selector::{InstalledProductPaths, select_production_cold_start};
use crate::escalation::{EscalationService, EscalationServiceHandle};
use crate::runtime_selector::{InstalledProductPaths, select_runtime};
use crate::waiter::BootstrapWaiterFace;
pub const PRIVATE_CLIENT_BOOTSTRAP_ROLE: &str = "--lazaret-private-client-bootstrap-v1";
@ -135,7 +133,7 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
.build()
.map_err(failure)?;
let selection = runtime
.block_on(select_production_cold_start(
.block_on(select_runtime(
&configuration,
Arc::clone(&store),
InstalledProductPaths {
@ -154,18 +152,18 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
session_anchor_id: parsed.session_anchor_id,
session_phase: SessionPhase::New,
};
ProductionColdStart::stage(inputs)
ColdStart::stage(inputs)
.map_err(failure)?
.run(bootstrap_owner, &owner, serve_published)
.map_err(failure)
}
fn serve_published(
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
lifecycle: &Mutex<CoordinatorLifecycle<'_, '_>>,
waiters: &RefCell<BootstrapWaiterFace>,
bootstrap_owner: &BootstrapOwnerGuard,
bootstrap_owner_session_id: SessionId,
) -> Result<(), ProductionColdStartError> {
) -> Result<(), ColdStartError> {
let escalation = take_escalation(lifecycle)?;
std::thread::scope(|scope| {
let escalation_enabled = escalation.is_some();
@ -187,9 +185,9 @@ fn serve_published(
}
require_escalation_running(&escalation_completions, escalation_enabled)?;
let face = {
let mut lifecycle = lifecycle.lock().map_err(|_| {
ProductionColdStartError::service("production lifecycle lock is poisoned")
})?;
let mut lifecycle = lifecycle
.lock()
.map_err(|_| ColdStartError::service("lifecycle lock is poisoned"))?;
let Some(face) = lifecycle.adapter().published_control() else {
for socket in active.values() {
let _ = socket.shutdown(std::net::Shutdown::Both);
@ -202,23 +200,21 @@ fn serve_published(
escalation.stop();
escalation_completions
.recv()
.map_err(ProductionColdStartError::service)?
.map_err(ProductionColdStartError::service)?;
.map_err(ColdStartError::service)?
.map_err(ColdStartError::service)?;
}
return Ok(());
};
lifecycle
.reconcile_published_bootstrap_owner(
bootstrap_owner_session_id,
bootstrap_owner
.is_live()
.map_err(ProductionColdStartError::service)?,
bootstrap_owner.is_live().map_err(ColdStartError::service)?,
)
.map_err(ProductionColdStartError::service)?;
.map_err(ColdStartError::service)?;
waiters
.borrow_mut()
.reconcile_published(&mut lifecycle)
.map_err(ProductionColdStartError::service)?;
.map_err(ColdStartError::service)?;
face
};
if active.len() == MAX_CONCURRENT_CONNECTIONS {
@ -229,16 +225,16 @@ fn serve_published(
Ok(connection) => {
let cancellation = connection
.cancellation_handle()
.map_err(ProductionColdStartError::service)?;
.map_err(ColdStartError::service)?;
let connection_id = next_connection;
next_connection = next_connection.checked_add(1).ok_or_else(|| {
ProductionColdStartError::service("control connection counter overflow")
ColdStartError::service("control connection counter overflow")
})?;
active.insert(connection_id, cancellation);
let completed = completed.clone();
let escalation = escalation_handle.clone();
scope.spawn(move || {
let result = serve_production_control_connection_with_escalation(
let result = serve_control_connection_with_escalation(
lifecycle,
&face,
connection,
@ -252,44 +248,41 @@ fn serve_published(
{
std::thread::sleep(std::time::Duration::from_millis(10));
}
Err(error) => return Err(ProductionColdStartError::service(error)),
Err(error) => return Err(ColdStartError::service(error)),
}
}
})
}
fn take_escalation<'resources>(
lifecycle: &Mutex<ProductionHostCoordinatorLifecycle<'_, 'resources>>,
) -> Result<
Option<(ProductionEscalation<'resources>, ProductionEscalationHandle)>,
ProductionColdStartError,
> {
lifecycle: &Mutex<CoordinatorLifecycle<'_, 'resources>>,
) -> Result<Option<(EscalationService<'resources>, EscalationServiceHandle)>, ColdStartError> {
let mut lifecycle = lifecycle
.lock()
.map_err(|_| ProductionColdStartError::service("production lifecycle lock is poisoned"))?;
.map_err(|_| ColdStartError::service("lifecycle lock is poisoned"))?;
lifecycle
.adapter()
.published_control()
.ok_or_else(|| ProductionColdStartError::service("control face is absent"))?
.ok_or_else(|| ColdStartError::service("control face is absent"))?
.set_nonblocking(true)
.map_err(ProductionColdStartError::service)?;
.map_err(ColdStartError::service)?;
Ok(lifecycle.adapter_mut().take_escalation())
}
fn require_escalation_running(
completions: &std::sync::mpsc::Receiver<Result<(), crate::escalation::EscalationError>>,
enabled: bool,
) -> Result<(), ProductionColdStartError> {
) -> Result<(), ColdStartError> {
match completions.try_recv() {
Ok(result) => {
result.map_err(ProductionColdStartError::service)?;
Err(ProductionColdStartError::service(
"production escalation service stopped before teardown",
result.map_err(ColdStartError::service)?;
Err(ColdStartError::service(
"escalation service stopped before teardown",
))
}
Err(std::sync::mpsc::TryRecvError::Disconnected) if enabled => Err(
ProductionColdStartError::service("production escalation service was abandoned"),
),
Err(std::sync::mpsc::TryRecvError::Disconnected) if enabled => {
Err(ColdStartError::service("escalation service was abandoned"))
}
Err(std::sync::mpsc::TryRecvError::Empty | std::sync::mpsc::TryRecvError::Disconnected) => {
Ok(())
}
@ -299,15 +292,13 @@ fn require_escalation_running(
fn wait_for_connection_completion(
completions: &std::sync::mpsc::Receiver<(u64, Result<(), ControlError>)>,
active: &mut BTreeMap<u64, std::os::unix::net::UnixStream>,
) -> Result<(), ProductionColdStartError> {
let (connection_id, result) = completions
.recv()
.map_err(ProductionColdStartError::service)?;
) -> Result<(), ColdStartError> {
let (connection_id, result) = completions.recv().map_err(ColdStartError::service)?;
active.remove(&connection_id);
connection_completion(result)
}
fn connection_completion(result: Result<(), ControlError>) -> Result<(), ProductionColdStartError> {
fn connection_completion(result: Result<(), ControlError>) -> Result<(), ColdStartError> {
match result {
Ok(()) => Ok(()),
Err(ControlError::Io(error))
@ -322,7 +313,7 @@ fn connection_completion(result: Result<(), ControlError>) -> Result<(), Product
{
Ok(())
}
Err(error) => Err(ProductionColdStartError::service(error)),
Err(error) => Err(ColdStartError::service(error)),
}
}
@ -470,7 +461,7 @@ fn load_configuration(
xdg_data_home: optional_environment_path("XDG_DATA_HOME"),
xdg_runtime_dir: runtime,
environment: environment(),
sandbox_layout: production_sandbox_layout()?,
sandbox_layout: default_sandbox_layout()?,
})?)
}

1091
crates/lazaret-host-coordinator/src/cold_start.rs Normal file
View file

File diff suppressed because it is too large Load diff

50
crates/lazaret-host-coordinator/src/escalation.rs
View file

@ -993,7 +993,7 @@ struct SessionAuthorityFiles {
registration_identity: MailboxEntryIdentity,
}
struct ProductionSessionAuthority {
struct SessionEscalationAuthority {
session: VerifiedSession,
files: SessionAuthorityFiles,
}
@ -1004,7 +1004,7 @@ struct RegistrationCommand {
completed: mpsc::SyncSender<Result<(), String>>,
}
enum ProductionEscalationCommand {
enum EscalationServiceCommand {
Register(Box<RegistrationCommand>),
Disconnect {
session_id: SessionId,
@ -1014,11 +1014,11 @@ enum ProductionEscalationCommand {
}
#[derive(Clone)]
pub(crate) struct ProductionEscalationHandle {
commands: mpsc::Sender<ProductionEscalationCommand>,
pub(crate) struct EscalationServiceHandle {
commands: mpsc::Sender<EscalationServiceCommand>,
}
impl ProductionEscalationHandle {
impl EscalationServiceHandle {
pub(crate) fn register(
&self,
binding: SessionBinding,
@ -1026,40 +1026,40 @@ impl ProductionEscalationHandle {
) -> Result<(), EscalationError> {
let (completed, result) = mpsc::sync_channel(1);
self.commands
.send(ProductionEscalationCommand::Register(Box::new(
.send(EscalationServiceCommand::Register(Box::new(
RegistrationCommand {
binding,
registration,
completed,
},
)))
.map_err(|_| io::Error::other("production escalation service stopped"))?;
.map_err(|_| io::Error::other("escalation service stopped"))?;
result
.recv()
.map_err(|_| io::Error::other("production escalation registration was abandoned"))?
.map_err(|_| io::Error::other("escalation registration was abandoned"))?
.map_err(|error| io::Error::other(error).into())
}
pub(crate) fn disconnect(&self, session_id: SessionId) -> Result<(), EscalationError> {
let (completed, result) = mpsc::sync_channel(1);
self.commands
.send(ProductionEscalationCommand::Disconnect {
.send(EscalationServiceCommand::Disconnect {
session_id,
completed,
})
.map_err(|_| io::Error::other("production escalation service stopped"))?;
.map_err(|_| io::Error::other("escalation service stopped"))?;
result
.recv()
.map_err(|_| io::Error::other("production escalation disconnect was abandoned"))?
.map_err(|_| io::Error::other("escalation disconnect was abandoned"))?
.map_err(|error| io::Error::other(error).into())
}
pub(crate) fn stop(&self) {
let _ = self.commands.send(ProductionEscalationCommand::Stop);
let _ = self.commands.send(EscalationServiceCommand::Stop);
}
}
pub(crate) struct ProductionEscalation<'a> {
pub(crate) struct EscalationService<'a> {
configuration: &'a ResolvedDomainConfiguration,
publication: ProtocolPublicationIdentity,
credentials: SandboxCredentialBundle,
@ -1068,12 +1068,12 @@ pub(crate) struct ProductionEscalation<'a> {
prompt_transport: Arc<SessionPromptTransport>,
coordinator: EscalationCoordinator,
ownership: LocalExecutionOwnership,
sessions: BTreeMap<SessionId, ProductionSessionAuthority>,
sessions: BTreeMap<SessionId, SessionEscalationAuthority>,
active: BTreeMap<RequestId, SessionId>,
commands: mpsc::Receiver<ProductionEscalationCommand>,
commands: mpsc::Receiver<EscalationServiceCommand>,
}
pub(crate) struct ProductionEscalationInputs<'a> {
pub(crate) struct EscalationServiceInputs<'a> {
pub configuration: &'a ResolvedDomainConfiguration,
pub publication: ProtocolPublicationIdentity,
pub credentials: SandboxCredentialBundle,
@ -1086,10 +1086,10 @@ pub(crate) struct ProductionEscalationInputs<'a> {
pub invoking_uid: u32,
}
impl<'a> ProductionEscalation<'a> {
impl<'a> EscalationService<'a> {
pub(crate) fn new(
inputs: ProductionEscalationInputs<'a>,
) -> Result<(Self, ProductionEscalationHandle), EscalationError> {
inputs: EscalationServiceInputs<'a>,
) -> Result<(Self, EscalationServiceHandle), EscalationError> {
inputs.credential_files.publish(
OsStr::new("publication.postcard"),
&lazaret_core::codec::encode(&inputs.publication)
@ -1122,7 +1122,7 @@ impl<'a> ProductionEscalation<'a> {
active: BTreeMap::new(),
commands: receiver,
},
ProductionEscalationHandle { commands },
EscalationServiceHandle { commands },
))
}
@ -1141,10 +1141,10 @@ impl<'a> ProductionEscalation<'a> {
fn apply_command(
&mut self,
command: ProductionEscalationCommand,
command: EscalationServiceCommand,
) -> Result<bool, EscalationError> {
match command {
ProductionEscalationCommand::Register(command) => {
EscalationServiceCommand::Register(command) => {
let RegistrationCommand {
binding,
registration,
@ -1160,7 +1160,7 @@ impl<'a> ProductionEscalation<'a> {
}
Ok(false)
}
ProductionEscalationCommand::Disconnect {
EscalationServiceCommand::Disconnect {
session_id,
completed,
} => {
@ -1174,7 +1174,7 @@ impl<'a> ProductionEscalation<'a> {
}
Ok(false)
}
ProductionEscalationCommand::Stop => {
EscalationServiceCommand::Stop => {
let sessions = self.sessions.keys().copied().collect::<Vec<_>>();
for session_id in sessions {
self.disconnect(session_id)?;
@ -1226,7 +1226,7 @@ impl<'a> ProductionEscalation<'a> {
};
self.sessions.insert(
session_id,
ProductionSessionAuthority {
SessionEscalationAuthority {
session,
files: SessionAuthorityFiles {
binding_name,

4
crates/lazaret-host-coordinator/src/lib.rs
View file

@ -2,16 +2,16 @@
pub mod bootstrap;
pub mod client_bootstrap;
pub mod cold_start;
pub mod control;
pub mod domain_store;
pub mod escalation;
pub mod git_routing;
pub mod lifecycle;
pub mod network;
pub mod production;
pub mod production_selector;
pub mod recovery;
pub mod runtime;
pub mod runtime_selector;
pub mod sandbox_init;
pub mod state_anchor;
pub mod state_maintenance;

4
crates/lazaret-host-coordinator/src/lifecycle.rs
View file

@ -276,7 +276,7 @@ where
/// Drives one cold-start leader through the complete normal publication boundary.
///
/// # Errors
/// Returns the first invalid transition, durable update, or production adapter failure.
/// Returns the first invalid transition, durable update, or lifecycle adapter failure.
pub fn bootstrap_new_domain(
&mut self,
generation: Generation,
@ -291,7 +291,7 @@ where
/// Returns `false` after applying the modeled bootstrap-owner disconnect transition.
///
/// # Errors
/// Returns the first invalid transition, durable update, or production adapter failure.
/// Returns the first invalid transition, durable update, or lifecycle adapter failure.
pub fn bootstrap_new_domain_while_owner_live(
&mut self,
generation: Generation,

117
crates/lazaret-host-coordinator/src/recovery.rs
View file

@ -565,7 +565,7 @@ fn require_mount_absent(
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ProductionRecoveryBoundary {
pub(crate) enum RecoveryBoundary {
Snapshot,
Publication,
Sandbox,
@ -578,22 +578,16 @@ pub(crate) enum ProductionRecoveryBoundary {
FinishGeneration,
}
pub(crate) fn recover_production_before_cold_start(
pub(crate) fn recover_before_cold_start(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
prepared: &PreparedLowerStore,
) -> Result<(), String> {
recover_production_before_cold_start_with(
configuration,
store,
catalog,
prepared,
|_, _| Ok(()),
)
recover_before_cold_start_with(configuration, store, catalog, prepared, |_, _| Ok(()))
}
fn recover_production_before_cold_start_with<Failpoint>(
fn recover_before_cold_start_with<Failpoint>(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
@ -601,9 +595,9 @@ fn recover_production_before_cold_start_with<Failpoint>(
failpoint: Failpoint,
) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
{
recover_production_before_cold_start_with_opener(
recover_before_cold_start_with_opener(
configuration,
store,
catalog,
@ -614,7 +608,7 @@ where
)
}
fn recover_production_before_cold_start_with_opener<Failpoint, Open, Delete>(
fn recover_before_cold_start_with_opener<Failpoint, Open, Delete>(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
@ -624,12 +618,12 @@ fn recover_production_before_cold_start_with_opener<Failpoint, Open, Delete>(
mut delete: Delete,
) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
failpoint(RecoveryBoundary::Snapshot, GenerationId::new(0))?;
let snapshot = store
.snapshot()
.map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
@ -661,14 +655,11 @@ where
)
})?;
}
failpoint(
ProductionRecoveryBoundary::PrepareOperations,
GenerationId::new(0),
)?;
failpoint(RecoveryBoundary::PrepareOperations, GenerationId::new(0))?;
prepared
.recover_preparations()
.map_err(|error| error.to_string())?;
failpoint(ProductionRecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
failpoint(RecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
let selected = catalog_snapshot
.prepared_artifacts
@ -697,7 +688,7 @@ fn recover_generation<Failpoint, Open, Delete>(
delete: &mut Delete,
) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
@ -744,11 +735,11 @@ where
let mut owners = seal_runtime_owners(&ownership, open)
.map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
failpoint(RecoveryBoundary::Publication, generation_id)?;
recover_publication(configuration, store, generation_id)
.map_err(|error| format!("Publication recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
failpoint(RecoveryBoundary::Sandbox, generation_id)?;
reconcile_runtime_before_state_anchor(&mut owners)
.map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
require_sandbox_absent(configuration, store, generation_id, delete)
@ -756,13 +747,13 @@ where
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
.map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
failpoint(RecoveryBoundary::HelperAndNetwork, generation_id)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
.map_err(|error| format!("Helper cleanup failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Network)
.map_err(|error| format!("Network cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
failpoint(RecoveryBoundary::GitRouting, generation_id)?;
require_mount_absent(
ownership.git_routing_path.as_ref(),
ownership.git_routing_mount,
@ -772,11 +763,11 @@ where
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
.map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
failpoint(RecoveryBoundary::StateAnchor, generation_id)?;
recover_state_anchor(configuration, store, catalog, generation, &mut owners)
.map_err(|error| format!("State Anchor recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
failpoint(RecoveryBoundary::PreparedLower, generation_id)?;
recover_partial_prepared_lower(configuration, store, catalog, generation_id)
.map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
@ -787,7 +778,7 @@ where
}) {
return Err("generation retains non-Clean participants after recovery".into());
}
failpoint(ProductionRecoveryBoundary::FinishGeneration, generation_id)?;
failpoint(RecoveryBoundary::FinishGeneration, generation_id)?;
let mut clean = generation.clone();
clean.durable_decision = DurableDecision::Clean;
store
@ -1341,7 +1332,7 @@ mod tests {
SandboxOwnership, StateAnchorOwnership,
};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
use super::*;
@ -1504,8 +1495,8 @@ mod tests {
}
#[test]
fn production_preflight_recovers_full_runtime_then_is_idempotent() {
let fixture = ProductionRecoveryFixture::new();
fn recovery_preflight_recovers_full_runtime_then_is_idempotent() {
let fixture = RecoveryFixture::new();
let mut boundaries = Vec::new();
fixture
.recover_with(|boundary, generation| {
@ -1516,7 +1507,7 @@ mod tests {
assert!(
boundaries
.iter()
.any(|(boundary, _)| { *boundary == ProductionRecoveryBoundary::StateAnchor })
.any(|(boundary, _)| { *boundary == RecoveryBoundary::StateAnchor })
);
assert_eq!(
fixture.store.snapshot().unwrap().generations[0].durable_decision,
@ -1552,7 +1543,7 @@ mod tests {
.exists()
);
recover_production_before_cold_start(
recover_before_cold_start(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
@ -1570,12 +1561,12 @@ mod tests {
#[test]
fn bare_publication_intent_requires_every_fixed_label_to_be_absent() {
let absent = ProductionRecoveryFixture::new();
let absent = RecoveryFixture::new();
absent.prepare_bare_publication(PublicationFixtureBoundary::TrulyAbsent);
recover_publication(&absent.configuration, &absent.store, GenerationId::new(2)).unwrap();
assert_eq!(absent.publication(2).state(), ParticipantState::Clean);
let identity = ProductionRecoveryFixture::new();
let identity = RecoveryFixture::new();
identity.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedIdentity);
assert!(
recover_publication(
@ -1594,7 +1585,7 @@ mod tests {
assert_eq!(identity.publication(2).state(), ParticipantState::Intent);
assert!(identity.publication(2).ownership().is_none());
let socket = ProductionRecoveryFixture::new();
let socket = RecoveryFixture::new();
socket.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedSocket);
assert!(
recover_publication(&socket.configuration, &socket.store, GenerationId::new(2))
@ -1617,27 +1608,27 @@ mod tests {
}
#[test]
fn production_preflight_retries_every_recovery_boundary() {
fn recovery_preflight_retries_every_recovery_boundary() {
for failed in [
ProductionRecoveryBoundary::Snapshot,
ProductionRecoveryBoundary::Publication,
ProductionRecoveryBoundary::Sandbox,
ProductionRecoveryBoundary::HelperAndNetwork,
ProductionRecoveryBoundary::GitRouting,
ProductionRecoveryBoundary::StateAnchor,
ProductionRecoveryBoundary::PreparedLower,
ProductionRecoveryBoundary::PrepareOperations,
ProductionRecoveryBoundary::ArtifactGc,
ProductionRecoveryBoundary::FinishGeneration,
RecoveryBoundary::Snapshot,
RecoveryBoundary::Publication,
RecoveryBoundary::Sandbox,
RecoveryBoundary::HelperAndNetwork,
RecoveryBoundary::GitRouting,
RecoveryBoundary::StateAnchor,
RecoveryBoundary::PreparedLower,
RecoveryBoundary::PrepareOperations,
RecoveryBoundary::ArtifactGc,
RecoveryBoundary::FinishGeneration,
] {
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let mut injected = false;
assert!(
fixture
.recover_with(|boundary, _| {
if boundary == failed && !injected {
injected = true;
Err("injected production recovery boundary".into())
Err("injected recovery boundary".into())
} else {
Ok(())
}
@ -1654,10 +1645,10 @@ mod tests {
}
#[test]
fn production_preflight_opens_every_exact_owner_once() {
let fixture = ProductionRecoveryFixture::new();
fn recovery_preflight_opens_every_exact_owner_once() {
let fixture = RecoveryFixture::new();
let mut opened = BTreeMap::new();
recover_production_before_cold_start_with_opener(
recover_before_cold_start_with_opener(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
@ -1678,7 +1669,7 @@ mod tests {
#[test]
fn publication_replacement_and_runsc_root_replacement_refuse_untouched() {
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1701,7 +1692,7 @@ mod tests {
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert!(socket.exists());
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1720,7 +1711,7 @@ mod tests {
.is_symlink()
);
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1734,7 +1725,7 @@ mod tests {
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert_eq!(fs::read(&identity).unwrap(), b"replacement");
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1747,7 +1738,7 @@ mod tests {
fs::rename(&runsc, &old_runsc).unwrap();
fs::create_dir(&runsc).unwrap();
let mut deletes = 0;
let result = recover_production_before_cold_start_with_opener(
let result = recover_before_cold_start_with_opener(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
@ -1763,7 +1754,7 @@ mod tests {
assert_eq!(deletes, 0);
assert!(runsc.exists());
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1783,7 +1774,7 @@ mod tests {
);
}
struct ProductionRecoveryFixture {
struct RecoveryFixture {
_tree: tempfile::TempDir,
configuration: ResolvedDomainConfiguration,
store: HostCoordinatorDomainStore,
@ -1791,12 +1782,12 @@ mod tests {
prepared: PreparedLowerStore,
}
impl ProductionRecoveryFixture {
impl RecoveryFixture {
fn recover_with<Failpoint>(&self, failpoint: Failpoint) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
{
recover_production_before_cold_start_with_opener(
recover_before_cold_start_with_opener(
&self.configuration,
&self.store,
&self.catalog,
@ -2221,7 +2212,7 @@ mod tests {
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: production_sandbox_layout().unwrap(),
sandbox_layout: default_sandbox_layout().unwrap(),
})
.unwrap()
}

629
crates/lazaret-host-coordinator/src/runtime_selector.rs Normal file
View file

@ -0,0 +1,629 @@
use std::collections::BTreeSet;
use std::fmt;
use std::fs::File;
use std::os::unix::ffi::OsStringExt;
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use lazaret_core::configuration::ResolvedDomainConfiguration;
use lazaret_core::identity::{
ConfigurationFingerprint, ContainerId, CoordinatorId, DomainId, GenerationId, NetworkSidecarId,
PrepareOperationId, SandboxId,
};
use lazaret_core::network::{DnsQueryExpectation, ReachabilityExpectation};
use lazaret_core::records::{DurableDecision, Generation, IpAddress, PreparedArtifact, VolumeKind};
use lazaret_linux::filesystem::namespace_identity;
use lazaret_linux::platform::{PlatformObservation, check_platform};
use lazaret_prepared_lower::{
IdMap, IdMapRange, OciRegistryTransport, Platform, PreparedDestination,
PreparedDestinationKind, PreparedLowerStore, RegistryCredentials,
};
use lazaret_runtime_supervisor::network::{DnsProbeTarget, NetworkProbeRequest, TcpProbeTarget};
use lazaret_runtime_supervisor::sandbox::{
CREDENTIALS_DESTINATION, ESCALATION_CLIENT_DESTINATION, MAILBOX_REQUESTS_DESTINATION,
MAILBOX_RESULTS_DESTINATION, RootProcess, SANDBOX_INIT_DESTINATION, TERMINFO_ROOT_DESTINATION,
WORKSPACE_DESTINATION,
};
use lazaret_storage_catalog::{Catalog, CatalogError};
use rustix::fs::{Mode, OFlags, ResolveFlags, openat2};
use crate::cold_start::ColdStartInputs;
use crate::domain_store::HostCoordinatorDomainStore;
use crate::recovery::recover_before_cold_start;
use crate::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE;
const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
#[derive(Debug)]
pub struct RuntimeSelectionError(String);
impl fmt::Display for RuntimeSelectionError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
impl std::error::Error for RuntimeSelectionError {}
fn failure(error: impl fmt::Display) -> RuntimeSelectionError {
RuntimeSelectionError(error.to_string())
}
pub struct InstalledProductPaths<'a> {
pub escalation_client: &'a Path,
pub sandbox_init: &'a Path,
}
pub struct RuntimeSelection {
store: Arc<HostCoordinatorDomainStore>,
domain_id: DomainId,
configuration_fingerprint: ConfigurationFingerprint,
catalog: Catalog,
artifact: PreparedArtifact,
artifact_authority: File,
artifact_label: PathBuf,
generation: Generation,
resolver_addresses: Vec<IpAddress>,
network_probe: NetworkProbeRequest,
reachability: ReachabilityExpectation,
container_id: ContainerId,
escalation_client: File,
sandbox_init: File,
}
impl RuntimeSelection {
/// Borrows the selected resources as the existing cold-start input.
///
/// # Errors
/// Refuses a configuration other than the exact one used for selection.
pub fn cold_start_inputs<'a>(
&'a self,
configuration: &'a ResolvedDomainConfiguration,
) -> Result<ColdStartInputs<'a>, RuntimeSelectionError> {
if configuration.domain_id() != self.domain_id
|| configuration.configuration_fingerprint() != self.configuration_fingerprint
{
return Err(failure("selected cold start does not match configuration"));
}
Ok(ColdStartInputs {
store: Arc::clone(&self.store),
catalog: &self.catalog,
configuration,
generation: self.generation.clone(),
prepared_artifact: &self.artifact,
prepared_artifact_authority: &self.artifact_authority,
prepared_artifact_label: &self.artifact_label,
root_process: RootProcess::new(
SANDBOX_INIT_DESTINATION,
[PRIVATE_SANDBOX_INIT_ROLE.to_owned()],
)
.map_err(failure)?,
resolver_addresses: self.resolver_addresses.clone(),
network_probe: self.network_probe.clone(),
reachability: self.reachability.clone(),
container_id: self.container_id,
invoking_uid: rustix::process::getuid().as_raw(),
escalation_client: &self.escalation_client,
sandbox_init: &self.sandbox_init,
})
}
}
/// Selects and prepares every Host Coordinator-owned cold-start input.
///
/// # Errors
/// Refuses invalid installed paths, unsupported architectures, registry or image failures,
/// changed Storage Root authority, exhausted identities, and configuration-policy mismatch.
pub async fn select_runtime(
configuration: &ResolvedDomainConfiguration,
store: Arc<HostCoordinatorDomainStore>,
product_paths: InstalledProductPaths<'_>,
) -> Result<RuntimeSelection, RuntimeSelectionError> {
let kernel_release = std::fs::read_to_string("/proc/sys/kernel/osrelease").map_err(failure)?;
select_runtime_for_platform(
configuration,
store,
product_paths,
&PlatformObservation {
operating_system: std::env::consts::OS,
architecture: std::env::consts::ARCH,
kernel_release: kernel_release.trim(),
},
)
.await
}
async fn select_runtime_for_platform(
configuration: &ResolvedDomainConfiguration,
store: Arc<HostCoordinatorDomainStore>,
product_paths: InstalledProductPaths<'_>,
observation: &PlatformObservation<'_>,
) -> Result<RuntimeSelection, RuntimeSelectionError> {
check_platform(observation).map_err(failure)?;
let escalation_client = open_installed_executable(product_paths.escalation_client)?;
let sandbox_init = open_installed_executable(product_paths.sandbox_init)?;
let catalog = Catalog::under_storage_root(configuration.storage_root()).map_err(failure)?;
match catalog.initialize() {
Ok(()) | Err(CatalogError::AlreadyInitialized) => {}
Err(error) => return Err(failure(error)),
}
let prepared =
PreparedLowerStore::under_storage_root(configuration.storage_root(), catalog.clone())
.map_err(failure)?;
recover_before_cold_start(configuration, &store, &catalog, &prepared).map_err(failure)?;
let image = OciRegistryTransport
.pull_confined(
&configuration.fingerprint_record().image,
&RegistryCredentials::Anonymous,
&Platform::linux("amd64"),
configuration.storage_root(),
random_id()?,
)
.await
.map_err(|error| failure(format!("pulling configured anonymous OCI image: {error:#}")))?;
let artifact = prepared
.prepare_transported(
PrepareOperationId::new(random_id()?),
&image,
rootless_id_map(),
prepared_destinations(configuration),
)
.map_err(failure)?;
let artifact_authority = prepared
.open_artifact_authority(&artifact)
.map_err(failure)?;
let artifact_label = PathBuf::from(std::ffi::OsString::from_vec(
configuration
.storage_root()
.canonical_label()
.as_bytes()
.to_vec(),
))
.join("prepared-lower")
.join(PathBuf::from(std::ffi::OsString::from_vec(
artifact.path.as_bytes().to_vec(),
)));
image.cleanup().map_err(failure)?;
let next_generation = next_generation(&store)?;
let (coordinator_id, network_sidecar_id, sandbox_id) = distinct_generation_ids()?;
let generation = Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(next_generation),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id,
network_sidecar_id,
sandbox_id,
durable_decision: DurableDecision::None,
};
let namespace = namespace_identity(Path::new(NETWORK_NAMESPACE_PATH)).map_err(failure)?;
let (resolver_addresses, network_probe, reachability) =
select_network_readiness(configuration, &generation, namespace);
Ok(RuntimeSelection {
store,
domain_id: configuration.domain_id(),
configuration_fingerprint: configuration.configuration_fingerprint(),
catalog,
artifact,
artifact_authority,
artifact_label,
generation,
resolver_addresses,
network_probe,
reachability,
container_id: ContainerId::new(random_id()?),
escalation_client,
sandbox_init,
})
}
fn prepared_destinations(configuration: &ResolvedDomainConfiguration) -> Vec<PreparedDestination> {
let mut destinations = [
"/dev",
"/proc",
"/sys",
"/tmp",
"/etc",
"/home",
"/run",
WORKSPACE_DESTINATION,
CREDENTIALS_DESTINATION,
MAILBOX_REQUESTS_DESTINATION,
MAILBOX_RESULTS_DESTINATION,
TERMINFO_ROOT_DESTINATION,
]
.into_iter()
.map(|path| PreparedDestination {
path: path.into(),
kind: PreparedDestinationKind::Directory,
})
.chain(
[
"/etc/resolv.conf",
"/etc/hosts",
"/etc/hostname",
ESCALATION_CLIENT_DESTINATION,
SANDBOX_INIT_DESTINATION,
]
.into_iter()
.map(|path| PreparedDestination {
path: path.into(),
kind: PreparedDestinationKind::RegularFile,
}),
)
.collect::<Vec<_>>();
destinations.extend(
configuration
.fingerprint_record()
.persistent_dirs
.iter()
.map(|path| PreparedDestination {
path: path.into(),
kind: PreparedDestinationKind::Directory,
}),
);
destinations.extend(
configuration
.volumes()
.iter()
.map(|volume| PreparedDestination {
path: volume.record().container_path.as_str().into(),
kind: match volume.record().kind {
VolumeKind::Directory => PreparedDestinationKind::Directory,
VolumeKind::File => PreparedDestinationKind::RegularFile,
},
}),
);
destinations
}
fn next_generation(store: &HostCoordinatorDomainStore) -> Result<u64, RuntimeSelectionError> {
store
.snapshot()
.map_err(failure)?
.generations
.iter()
.map(|generation| generation.generation_id.get())
.max()
.unwrap_or(0)
.checked_add(1)
.ok_or_else(|| failure("generation identity exhausted"))
}
fn select_network_readiness(
configuration: &ResolvedDomainConfiguration,
generation: &Generation,
enforcement_namespace: lazaret_core::identity::NamespaceIdentity,
) -> (Vec<IpAddress>, NetworkProbeRequest, ReachabilityExpectation) {
let readiness = configuration.network_readiness();
let denied = readiness
.denied
.iter()
.map(|target| TcpProbeTarget {
address: IpAddress::V4(target.address),
port: target.port,
})
.collect::<Vec<_>>();
let permitted = readiness
.permitted
.iter()
.map(|target| TcpProbeTarget {
address: IpAddress::V4(target.address),
port: target.port,
})
.collect::<Vec<_>>();
let dns = readiness
.dns
.iter()
.map(|target| DnsProbeTarget {
resolver: IpAddress::V4(target.resolver),
query_name: target.query.clone(),
})
.collect::<Vec<_>>();
let reachability = ReachabilityExpectation {
denied_destinations: denied.iter().map(|target| target.address).collect(),
permitted_destinations: permitted.iter().map(|target| target.address).collect(),
dns_queries: dns
.iter()
.map(|target| DnsQueryExpectation {
resolver: target.resolver,
query_name: target.query_name.clone(),
})
.collect(),
};
let resolvers = dns
.iter()
.map(|target| target.resolver)
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
(
resolvers,
NetworkProbeRequest {
domain_id: generation.domain_id,
generation_id: generation.generation_id,
enforcement_namespace,
denied,
permitted,
dns,
},
reachability,
)
}
fn open_installed_executable(path: &Path) -> Result<File, RuntimeSelectionError> {
let descriptor = openat2(
rustix::fs::CWD,
path,
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
)
.map_err(failure)?;
let file = File::from(descriptor);
let metadata = file.metadata().map_err(failure)?;
if !metadata.is_file() || metadata.permissions().mode() & 0o111 == 0 {
return Err(failure(
"installed product path is not an executable regular file",
));
}
Ok(file)
}
fn rootless_id_map() -> IdMap {
IdMap {
uid: vec![IdMapRange {
container_id: 0,
namespace_id: rustix::process::getuid().as_raw(),
size: 1,
}],
gid: vec![IdMapRange {
container_id: 0,
namespace_id: rustix::process::getgid().as_raw(),
size: 1,
}],
}
}
fn distinct_generation_ids()
-> Result<(CoordinatorId, NetworkSidecarId, SandboxId), RuntimeSelectionError> {
for _ in 0..8 {
let coordinator = CoordinatorId::new(random_id()?);
let network = NetworkSidecarId::new(random_id()?);
let sandbox = SandboxId::new(random_id()?);
if coordinator.as_bytes() != network.as_bytes()
&& coordinator.as_bytes() != sandbox.as_bytes()
&& network.as_bytes() != sandbox.as_bytes()
{
return Ok((coordinator, network, sandbox));
}
}
Err(failure(
"fresh generation role identities repeatedly collided",
))
}
fn random_id() -> Result<[u8; 32], RuntimeSelectionError> {
let mut bytes = [0; 32];
getrandom::fill(&mut bytes).map_err(failure)?;
Ok(bytes)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use std::fs::{self, OpenOptions};
use std::future::Future;
use std::os::unix::ffi::OsStringExt;
use std::path::PathBuf;
use std::task::{Context, Poll, Waker};
use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration};
use lazaret_core::identity::{DeviceId, InodeId, NamespaceIdentity};
use lazaret_core::records::Domain;
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
use crate::domain_store::DomainStoreMutation;
use super::*;
#[test]
fn resolved_readiness_maps_exactly_and_rootless_id_map_matches_builder_policy() {
let tree = tempfile::tempdir().unwrap();
let configuration = configuration(tree.path());
let generation = Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(9),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([1; 32]),
network_sidecar_id: NetworkSidecarId::new([2; 32]),
sandbox_id: SandboxId::new([3; 32]),
durable_decision: DurableDecision::None,
};
let namespace = NamespaceIdentity {
device: DeviceId::new(4),
inode: InodeId::new(5),
};
let (resolvers, request, expectation) =
select_network_readiness(&configuration, &generation, namespace);
assert_eq!(resolvers, [IpAddress::V4([198, 51, 100, 53])]);
assert_eq!(request.denied[0].port, 8443);
assert_eq!(request.permitted[0].port, 443);
assert_eq!(request.dns[0].query_name, "example.invalid");
assert_eq!(expectation.denied_destinations, [request.denied[0].address]);
assert_eq!(
expectation.permitted_destinations,
[request.permitted[0].address]
);
assert_eq!(expectation.dns_queries[0].resolver, request.dns[0].resolver);
let id_map = rootless_id_map();
assert_eq!(id_map.uid[0].container_id, 0);
assert_eq!(
id_map.uid[0].namespace_id,
rustix::process::getuid().as_raw()
);
assert_eq!(
id_map.gid[0].namespace_id,
rustix::process::getgid().as_raw()
);
let destinations = prepared_destinations(&configuration);
assert!(destinations.iter().any(|destination| {
destination.path == Path::new(WORKSPACE_DESTINATION)
&& destination.kind == PreparedDestinationKind::Directory
}));
assert!(destinations.iter().any(|destination| {
destination.path == Path::new(SANDBOX_INIT_DESTINATION)
&& destination.kind == PreparedDestinationKind::RegularFile
}));
}
#[test]
fn installed_product_selection_requires_exact_executable_and_fresh_roles() {
let tree = tempfile::tempdir().unwrap();
let executable = tree.path().join("product");
fs::copy("/bin/true", &executable).unwrap();
fs::set_permissions(&executable, fs::Permissions::from_mode(0o700)).unwrap();
assert!(open_installed_executable(&executable).is_ok());
fs::set_permissions(&executable, fs::Permissions::from_mode(0o600)).unwrap();
assert!(open_installed_executable(&executable).is_err());
let (coordinator, network, sandbox) = distinct_generation_ids().unwrap();
assert_ne!(coordinator.as_bytes(), network.as_bytes());
assert_ne!(coordinator.as_bytes(), sandbox.as_bytes());
assert_ne!(network.as_bytes(), sandbox.as_bytes());
}
#[test]
fn unsupported_platforms_refuse_before_installed_or_storage_side_effects() {
for observation in [
PlatformObservation {
operating_system: "linux",
architecture: "x86_64",
kernel_release: "6.8.14",
},
PlatformObservation {
operating_system: "linux",
architecture: "aarch64",
kernel_release: "6.9.0",
},
] {
let tree = tempfile::tempdir().unwrap();
let configuration = configuration(tree.path());
let store = domain_store(&configuration);
let mut future = Box::pin(select_runtime_for_platform(
&configuration,
store,
InstalledProductPaths {
escalation_client: Path::new("/missing-escalation-client"),
sandbox_init: Path::new("/missing-sandbox-init"),
},
&observation,
));
let mut context = Context::from_waker(Waker::noop());
let Poll::Ready(result) = future.as_mut().poll(&mut context) else {
panic!("platform refusal performed asynchronous work");
};
assert!(result.is_err());
let storage = tree.path().join("data");
assert!(!storage.join("storage-catalog.lock").exists());
assert!(!storage.join("catalog.redb").exists());
assert!(!storage.join("prepared-lower").exists());
assert!(!storage.join("image-transports").exists());
}
}
#[test]
fn generation_selection_and_cold_start_retain_one_exact_domain_store() {
let tree = tempfile::tempdir().unwrap();
let configuration = configuration(tree.path());
let first_root = tree.path().join("store-a");
let second_root = tree.path().join("store-b");
fs::create_dir(&first_root).unwrap();
fs::create_dir(&second_root).unwrap();
let first = domain_store_at(&configuration, &first_root);
let second = domain_store_at(&configuration, &second_root);
second
.apply(&[DomainStoreMutation::PutGeneration(Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(1),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([1; 32]),
network_sidecar_id: NetworkSidecarId::new([2; 32]),
sandbox_id: SandboxId::new([3; 32]),
durable_decision: DurableDecision::None,
})])
.unwrap();
assert_eq!(next_generation(&first).unwrap(), 1);
assert_eq!(next_generation(&second).unwrap(), 2);
assert!(!Arc::ptr_eq(&first, &second));
}
fn domain_store(
configuration: &ResolvedDomainConfiguration,
) -> Arc<HostCoordinatorDomainStore> {
let root = PathBuf::from(std::ffi::OsString::from_vec(
configuration
.persistent_domain_root()
.canonical_label()
.as_bytes()
.to_vec(),
));
domain_store_at(configuration, &root)
}
fn domain_store_at(
configuration: &ResolvedDomainConfiguration,
root: &Path,
) -> Arc<HostCoordinatorDomainStore> {
let lock = OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(root.join("domain.lock"))
.unwrap();
Arc::new(
HostCoordinatorDomainStore::open(
try_exclusive_flock(lock).unwrap(),
&root.join("domain.redb"),
&Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
},
)
.unwrap(),
)
}
fn configuration(root: &Path) -> ResolvedDomainConfiguration {
let configuration_directory = root.join("configuration");
let workspace = root.join("workspace");
let runtime = root.join("runtime");
fs::create_dir(&configuration_directory).unwrap();
fs::create_dir(&workspace).unwrap();
fs::create_dir(&runtime).unwrap();
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n[storage]\ndata_root = \"{}\"\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 8443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"Example.Invalid.\" }}]\n",
root.join("data").display()
),
)
.unwrap();
load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace,
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: default_sandbox_layout().unwrap(),
})
.unwrap()
}
}

2
crates/lazaret-host-coordinator/src/state_anchor/tests.rs
View file

@ -700,7 +700,7 @@ fn configuration(tree: &TestTree) -> lazaret_core::configuration::ResolvedDomain
),
)
.unwrap();
let layout = SandboxLayoutV1::from_production_layout([
let layout = SandboxLayoutV1::from_protected_destinations([
"/workspace".to_owned(),
"/proc".to_owned(),
"/sys".to_owned(),

4
crates/lazaret-host-coordinator/src/state_maintenance.rs
View file

@ -270,7 +270,7 @@ mod tests {
};
use lazaret_core::records::{Domain, Generation};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
use rustix::fs::{FileType, Mode, XattrFlags, fsetxattr, makedev, mknodat};
use super::*;
@ -491,7 +491,7 @@ mod tests {
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: production_sandbox_layout().unwrap(),
sandbox_layout: default_sandbox_layout().unwrap(),
})
.unwrap()
}

2
crates/lazaret-host-coordinator/src/terminal/tests.rs
View file

@ -544,7 +544,7 @@ fn engine_for(fixture: &Fixture) -> (Arc<MockFactory>, SessionTerminalEngine) {
#[test]
#[allow(clippy::too_many_lines)]
fn production_prompt_adapter_retains_resume_resize_before_authenticated_denial() {
fn prompt_adapter_retains_resume_resize_before_authenticated_denial() {
let generation = GenerationIdentity {
agent_profile: "trusted".into(),
domain_id: DomainId::new(bytes(1)),

4
crates/lazaret-host-coordinator/src/waiter.rs
View file

@ -436,7 +436,7 @@ mod tests {
use lazaret_core::identity::{CoordinatorId, NetworkSidecarId, SandboxId};
use lazaret_core::records::{Domain, DurableDecision, Generation};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::lifecycle::{LifecycleAction, SessionAdmissionIdentity};
@ -757,7 +757,7 @@ mod tests {
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: production_sandbox_layout().unwrap(),
sandbox_layout: default_sandbox_layout().unwrap(),
})
.unwrap()
}

299
crates/lazaret-host-coordinator/tests/cold_start.rs Normal file
View file

@ -0,0 +1,299 @@
use std::collections::BTreeMap;
use std::fs::{self, File, OpenOptions};
use std::os::unix::ffi::{OsStrExt, OsStringExt};
use std::os::unix::fs::{MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use lazaret_core::configuration::{ConfigurationLoadRequest, load_trusted_configuration};
use lazaret_core::identity::{
ContainerId, CoordinatorId, DeviceId, GenerationId, HostPath, IdMapDigest, InodeId, PolicyId,
PreparedArtifactId, SandboxId, SelectedManifestDigest, SourceIdentity,
};
use lazaret_core::records::{
ArtifactState, Domain, DurableDecision, Generation, IpAddress, PreparedArtifact,
};
use lazaret_host_coordinator::cold_start::{ColdStart, ColdStartInputs};
use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore;
use lazaret_linux::filesystem::{namespace_identity, try_exclusive_flock};
use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::{RootProcess, default_sandbox_layout};
use lazaret_storage_catalog::Catalog;
use rustix::process::getuid;
#[test]
fn validated_inputs_stage_one_complete_plan_and_drop_exact_runtime_artifacts() {
let fixture = ColdStartFixture::new();
let staged = ColdStart::stage(fixture.inputs(&fixture.escalation_client)).unwrap();
let bundle = fixture.bundle();
let control = fixture.control();
let overlay = fixture.overlay();
assert_private_artifacts(&bundle, &control);
assert!(overlay.is_dir());
drop(staged);
assert!(!bundle.exists());
assert!(!control.exists());
assert!(!overlay.exists());
assert!(ColdStart::stage(fixture.inputs(&fixture.invalid_escalation_client)).is_err());
assert!(!bundle.exists());
assert!(!control.exists());
}
#[test]
fn non_git_workspace_stages_without_a_routing_runtime() {
let fixture = ColdStartFixture::new();
let staged = ColdStart::stage(fixture.inputs(&fixture.escalation_client)).unwrap();
assert!(!fixture.runtime_domain.join("git-routing").exists());
drop(staged);
}
#[test]
fn cleanup_leaves_a_replacement_directory_untouched() {
let fixture = ColdStartFixture::new();
let staged = ColdStart::stage(fixture.inputs(&fixture.escalation_client)).unwrap();
let bundle = fixture.bundle();
let moved = bundle.with_extension("moved");
fs::rename(&bundle, &moved).unwrap();
fs::create_dir(&bundle).unwrap();
fs::write(bundle.join("replacement"), b"trusted replacement").unwrap();
drop(staged);
assert_eq!(
fs::read(bundle.join("replacement")).unwrap(),
b"trusted replacement"
);
assert!(moved.is_dir());
}
#[test]
fn fixed_runtime_parents_refuse_symlinks_and_wrong_modes() {
let fixture = ColdStartFixture::new();
let bundles = fixture.runtime_domain.join("bundles");
fs::create_dir(&bundles).unwrap();
fs::set_permissions(&bundles, fs::Permissions::from_mode(0o755)).unwrap();
assert!(ColdStart::stage(fixture.inputs(&fixture.escalation_client)).is_err());
fs::remove_dir(&bundles).unwrap();
let controls = fixture.runtime_domain.join("sandbox-control");
let redirect = fixture.tree.path().join("redirect");
fs::create_dir(&redirect).unwrap();
std::os::unix::fs::symlink(&redirect, &controls).unwrap();
assert!(ColdStart::stage(fixture.inputs(&fixture.escalation_client)).is_err());
assert!(redirect.read_dir().unwrap().next().is_none());
}
#[test]
fn replaced_runtime_domain_authority_is_not_used_for_staging() {
let fixture = ColdStartFixture::new();
let moved = fixture.runtime_domain.with_extension("moved");
fs::rename(&fixture.runtime_domain, &moved).unwrap();
fs::create_dir(&fixture.runtime_domain).unwrap();
fs::set_permissions(&fixture.runtime_domain, fs::Permissions::from_mode(0o700)).unwrap();
assert!(ColdStart::stage(fixture.inputs(&fixture.escalation_client)).is_err());
assert!(fixture.runtime_domain.read_dir().unwrap().next().is_none());
}
struct ColdStartFixture {
tree: tempfile::TempDir,
configuration: lazaret_core::configuration::ResolvedDomainConfiguration,
runtime_domain: PathBuf,
catalog: Catalog,
store: Arc<HostCoordinatorDomainStore>,
artifact: PreparedArtifact,
artifact_authority: File,
generation: Generation,
container_id: ContainerId,
escalation_client: File,
invalid_escalation_client: File,
}
impl ColdStartFixture {
fn new() -> Self {
let tree = tempfile::tempdir().unwrap();
let configuration = configuration(tree.path());
let persistent_domain = host_path(
configuration
.persistent_domain_root()
.canonical_label()
.as_bytes(),
);
let runtime_domain = host_path(
configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
);
let catalog = Catalog::new(
tree.path().join("catalog.redb"),
tree.path().join("catalog.lock"),
);
catalog.initialize().unwrap();
let domain = Domain {
domain_id: configuration.domain_id(),
profile_name: configuration.selected_profile().to_owned(),
canonical_workspace: configuration.workspace().canonical_label().clone(),
};
let lock = OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(persistent_domain.join("domain.lock"))
.unwrap();
let store = Arc::new(
HostCoordinatorDomainStore::open(
try_exclusive_flock(lock).unwrap(),
&persistent_domain.join("domain.redb"),
&domain,
)
.unwrap(),
);
let lower = tree.path().join("prepared-lower");
fs::create_dir(&lower).unwrap();
let metadata = lower.metadata().unwrap();
let artifact_authority = File::open(&lower).unwrap();
let artifact = PreparedArtifact {
prepared_artifact_id: PreparedArtifactId::new([8; 32]),
source_identity: SourceIdentity::new(b"synthetic-spike".to_vec()),
selected_manifest_digest: SelectedManifestDigest::new([9; 32]),
policy_id: PolicyId::new([10; 32]),
id_map_digest: IdMapDigest::new([11; 32]),
path: HostPath::new(lower.as_os_str().as_bytes().to_vec()),
device: DeviceId::new(metadata.dev()),
inode: InodeId::new(metadata.ino()),
state: ArtifactState::Available,
};
let generation = Generation {
domain_id: configuration.domain_id(),
generation_id: GenerationId::new(1),
configuration_fingerprint: configuration.configuration_fingerprint(),
coordinator_id: CoordinatorId::new([2; 32]),
network_sidecar_id: lazaret_core::identity::NetworkSidecarId::new([3; 32]),
sandbox_id: SandboxId::new([4; 32]),
durable_decision: DurableDecision::None,
};
Self {
tree,
configuration,
runtime_domain,
catalog,
store,
artifact,
artifact_authority,
generation,
container_id: ContainerId::new([5; 32]),
escalation_client: File::open("/bin/true").unwrap(),
invalid_escalation_client: File::open("/dev/null").unwrap(),
}
}
fn inputs<'a>(&'a self, escalation_client: &'a File) -> ColdStartInputs<'a> {
ColdStartInputs {
store: Arc::clone(&self.store),
catalog: &self.catalog,
configuration: &self.configuration,
generation: self.generation.clone(),
prepared_artifact: &self.artifact,
prepared_artifact_authority: &self.artifact_authority,
prepared_artifact_label: Path::new(
std::str::from_utf8(self.artifact.path.as_bytes()).unwrap(),
),
root_process: RootProcess::new("/bin/sh", ["-c".into(), "sleep infinity".into()])
.unwrap(),
resolver_addresses: vec![IpAddress::V4([10, 0, 2, 3])],
network_probe: NetworkProbeRequest {
domain_id: self.configuration.domain_id(),
generation_id: GenerationId::new(1),
enforcement_namespace: namespace_identity(Path::new("/proc/self/ns/net")).unwrap(),
denied: Vec::new(),
permitted: Vec::new(),
dns: Vec::new(),
},
reachability: lazaret_core::network::ReachabilityExpectation {
denied_destinations: Vec::new(),
permitted_destinations: Vec::new(),
dns_queries: Vec::new(),
},
container_id: self.container_id,
invoking_uid: getuid().as_raw(),
escalation_client,
sandbox_init: escalation_client,
}
}
fn bundle(&self) -> PathBuf {
self.runtime_domain
.join("bundles")
.join(hex(self.container_id.as_bytes()))
}
fn control(&self) -> PathBuf {
self.runtime_domain
.join("sandbox-control")
.join(hex(self.container_id.as_bytes()))
}
fn overlay(&self) -> PathBuf {
self.runtime_domain
.join("runsc-overlay")
.join(hex(self.container_id.as_bytes()))
}
}
fn assert_private_artifacts(bundle: &Path, control: &Path) {
let config = bundle.join("config.json");
let escalation_client = control.join("bin/lazaret-escalate");
assert!(config.is_file());
assert!(escalation_client.is_file());
assert!(control.join("credentials").is_dir());
assert_eq!(mode(bundle), 0o700);
assert_eq!(mode(control), 0o700);
assert_eq!(mode(&config), 0o600);
assert_eq!(mode(&escalation_client), 0o700);
}
fn mode(path: &Path) -> u32 {
path.metadata().unwrap().permissions().mode() & 0o777
}
fn configuration(root: &Path) -> lazaret_core::configuration::ResolvedDomainConfiguration {
let configuration_directory = root.join("configuration");
let workspace = root.join("workspace");
let runtime = root.join("runtime");
fs::create_dir(&configuration_directory).unwrap();
fs::create_dir(&workspace).unwrap();
fs::create_dir(&runtime).unwrap();
let configuration_path = configuration_directory.join("lazaret.toml");
fs::write(
&configuration_path,
format!(
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[network]\ndefault_allow = [\"192.0.2.10/32\"]\n\n[profiles.default]\nimage = \"example.invalid/factory:v1\"\npersistent_dirs = []\n\n[profiles.default.readiness]\ndenied = [{{ address = \"10.20.30.40\", port = 443 }}]\npermitted = [{{ address = \"192.0.2.10\", port = 443 }}]\ndns = [{{ resolver = \"198.51.100.53\", query = \"example.invalid\" }}]\n",
root.join("data").display()
),
)
.unwrap();
load_trusted_configuration(ConfigurationLoadRequest {
explicit_configuration_path: Some(configuration_path),
selected_profile: "default".into(),
workspace,
home: root.join("home"),
xdg_config_home: None,
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: default_sandbox_layout().unwrap(),
})
.unwrap()
}
fn host_path(bytes: &[u8]) -> PathBuf {
PathBuf::from(std::ffi::OsString::from_vec(bytes.to_vec()))
}
fn hex(bytes: &[u8]) -> String {
use std::fmt::Write as _;
bytes.iter().fold(String::new(), |mut output, byte| {
write!(output, "{byte:02x}").unwrap();
output
})
}

2
crates/lazaret-host-coordinator/tests/lifecycle.rs
View file

@ -7,12 +7,12 @@ use lazaret_core::identity::{
NetworkSidecarId, SandboxId, SessionAnchorId, SessionId,
};
use lazaret_core::records::{Domain, DurableDecision, Generation, Session, SessionPhase};
use lazaret_host_coordinator::cold_start::BootstrapOwnerGuard;
use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use lazaret_host_coordinator::lifecycle::{
DomainPhase, HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, LifecycleError,
SessionAdmissionIdentity, SessionLifecycleAdapter,
};
use lazaret_host_coordinator::production::BootstrapOwnerGuard;
use lazaret_linux::filesystem::try_exclusive_flock;
use rustix::pipe::{PipeFlags, pipe_with};

18
crates/lazaret-host-coordinator/tests/runtime-composition.rs
View file

@ -42,7 +42,7 @@ use lazaret_linux::process::{
use lazaret_runtime_supervisor::network::{DnsProbeTarget, NetworkProbeRequest, TcpProbeTarget};
use lazaret_runtime_supervisor::sandbox::{
NETWORK_NAMESPACE_PATH, RootProcess, RunscCommand, SandboxBuildRequest, SandboxBundle,
build_sandbox_bundle, production_sandbox_layout,
build_sandbox_bundle, default_sandbox_layout,
};
use lazaret_runtime_supervisor::supervision::{
AttachedRunscSignal, RuntimeNetworkLaunch, RuntimeSupervisorLaunch,
@ -90,7 +90,7 @@ fn main() -> ExitCode {
if arguments.first().and_then(|argument| argument.to_str()) == Some(NETWORK_MATRIX) {
return match run_network_matrix() {
Ok(()) => {
println!("PASS production network composition matrix");
println!("PASS network composition matrix");
ExitCode::SUCCESS
}
Err(MatrixError::Skip(reason)) => {
@ -122,15 +122,15 @@ fn main() -> ExitCode {
if arguments.first().and_then(|argument| argument.to_str()) == Some(SESSION_TERMINAL_MATRIX) {
return match run_session_terminal_matrix() {
Ok(()) => {
println!("PASS production Session Terminal matrix");
println!("PASS Session Terminal composition matrix");
ExitCode::SUCCESS
}
Err(MatrixError::Skip(reason)) => {
println!("SKIP production Session Terminal matrix: {reason}");
println!("SKIP Session Terminal composition matrix: {reason}");
ExitCode::from(SKIP)
}
Err(MatrixError::Failure(error)) => {
eprintln!("production Session Terminal matrix failed: {error}");
eprintln!("Session Terminal composition matrix failed: {error}");
ExitCode::FAILURE
}
};
@ -329,7 +329,7 @@ fn run_session_terminal_matrix() -> Result<(), MatrixError> {
.arguments()
.iter()
.position(|argument| argument == "run")
.ok_or_else(|| failure("production runsc command has no run action"))?;
.ok_or_else(|| failure("runsc command has no run action"))?;
let workspace = host_path(fixture.configuration.workspace().canonical_label());
let count = workspace.join("session-count");
let finish = workspace.join("session-finish");
@ -1072,7 +1072,7 @@ impl Fixture {
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment,
sandbox_layout: production_sandbox_layout().map_err(failure)?,
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
@ -1136,7 +1136,7 @@ impl Fixture {
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: production_sandbox_layout().map_err(failure)?,
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)?;
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
@ -1462,7 +1462,7 @@ fn network_configuration(
xdg_data_home: None,
xdg_runtime_dir: runtime.to_path_buf(),
environment: BTreeMap::new(),
sandbox_layout: production_sandbox_layout().map_err(failure)?,
sandbox_layout: default_sandbox_layout().map_err(failure)?,
})
.map_err(failure)
}