ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Enforce exact recovery ownership before cold start

Browse source
This commit is contained in:
ms2k 2026-07-17 06:48:49 +09:00
commit 2bc5982a62
8 changed files with 1076 additions and 238 deletions
Download patch file Download diff file Expand all files Collapse all files

125
crates/lazaret-host-coordinator/src/bootstrap.rs
View file

@ -7,7 +7,10 @@ use lazaret_core::configuration::ResolvedDomainConfiguration;
use lazaret_core::identity::{BundleDigest, ContainerId, MountIdentity};
use lazaret_core::network::{CanonicalDerivedNetworkPolicy, ReachabilityExpectation};
use lazaret_core::records::Session;
use lazaret_core::records::{Generation, GenerationIdentity, PreparedArtifact};
use lazaret_core::records::{
Generation, GenerationIdentity, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PreparedArtifact,
};
use lazaret_linux::process::current_process_identity;
use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::SandboxBundle;
@ -20,7 +23,7 @@ use crate::control::{
ControlError, NonInteractiveSessionExecutor, PublishedControlFace, SessionRegistry,
StagedControlFace, TrustAnchor,
};
use crate::domain_store::HostCoordinatorDomainStore;
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, SessionAdmissionIdentity,
SessionLifecycleAdapter,
@ -240,6 +243,10 @@ impl<'a> ProductionLifecycleAdapter<'a> {
self.plan.generation_identity.generation_id,
self.plan.container_id,
&self.plan.runtime_root,
self.plan
.runtime_staging
.runsc_root_identity()
.map_err(failure)?,
self.plan.bundle_digest,
current_process_identity().map_err(failure)?,
)
@ -347,36 +354,105 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.trust_anchor
.take()
.ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?;
self.staged_control = Some(
StagedControlFace::stage_with_executor(
path_from_host(
self.plan
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
),
self.plan.invoking_uid,
anchor,
Arc::clone(&self.registry),
Arc::clone(&self.plan.non_interactive_executor),
)
.map_err(failure)?,
);
let generation_id = self.plan.generation_identity.generation_id;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::bare_intent(generation_id, ParticipantRole::Publication),
)])
.map_err(failure)?;
let staged = StagedControlFace::stage_with_executor(
path_from_host(
self.plan
.configuration
.runtime_domain_root()
.canonical_label()
.as_bytes(),
),
self.plan.invoking_uid,
anchor,
Arc::clone(&self.registry),
Arc::clone(&self.plan.non_interactive_executor),
)
.map_err(failure)?;
let ownership = staged.recovery_ownership().map_err(failure)?;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::Intent,
Some(ParticipantOwnership::Publication(ownership)),
)
.map_err(failure)?,
)])
.map_err(failure)?;
self.staged_control = Some(staged);
Ok(())
}
fn expose_publication(&mut self) -> Result<(), ProductionLifecycleError> {
self.published_control = Some(Arc::new(
self.staged_control
.take()
.ok_or_else(|| ProductionLifecycleError("staged control face is absent".into()))?
.expose()
let published = self
.staged_control
.take()
.ok_or_else(|| ProductionLifecycleError("staged control face is absent".into()))?
.expose()
.map_err(failure)?;
let ownership = published.recovery_ownership().map_err(failure)?;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::snapshot(
self.plan.generation_identity.generation_id,
ParticipantRole::Publication,
ParticipantState::Committed,
Some(ParticipantOwnership::Publication(ownership)),
)
.map_err(failure)?,
));
)])
.map_err(failure)?;
self.published_control = Some(Arc::new(published));
Ok(())
}
fn clean_publication_after_withdrawal(&self) -> Result<(), ProductionLifecycleError> {
let generation_id = self.plan.generation_identity.generation_id;
let current = self
.plan
.store
.snapshot()
.map_err(failure)?
.participants
.into_iter()
.find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::Publication
})
.ok_or_else(|| ProductionLifecycleError("Publication participant is absent".into()))?;
let release = Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::ReleaseIntent,
current.ownership().cloned(),
)
.map_err(failure)?;
let clean = Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::Clean,
Some(ParticipantOwnership::empty(ParticipantRole::Publication)),
)
.map_err(failure)?;
self.plan
.store
.apply(&[
DomainStoreMutation::PutParticipant(release),
DomainStoreMutation::CleanParticipantAfterAbsence(clean),
])
.map_err(failure)
}
fn stop_sandbox(&mut self) -> Result<(), ProductionLifecycleError> {
let store = Arc::clone(&self.plan.store);
let sandbox = SandboxCoordinator::new(&store);
@ -517,6 +593,7 @@ impl LifecycleAdapter for ProductionLifecycleAdapter<'_> {
LifecycleAction::BeginTeardown => {
if let Some(control) = self.published_control.as_ref() {
control.withdraw().map_err(failure)?;
self.clean_publication_after_withdrawal()?;
}
self.published_control.take();
Ok(())

67
crates/lazaret-host-coordinator/src/control.rs
View file

@ -13,7 +13,8 @@ use ed25519_dalek::SigningKey;
use lazaret_core::codec::{CodecError, decode, encode};
use lazaret_core::configuration::ConfinedPathPlan;
use lazaret_core::identity::{
CertificateDigest, ClientId, ContainerId, SessionAnchorId, SessionId, TrustManifestDigest,
CertificateDigest, ClientId, ContainerId, DeviceId, HostPath, InodeId, MountIdentity,
SessionAnchorId, SessionId, TrustManifestDigest,
};
use lazaret_core::protocol::{
ProtocolValidationError, issue_protocol_credential, issue_session_capability,
@ -23,9 +24,10 @@ use lazaret_core::records::{
ClientEnrollmentRequest, ClientEnrollmentResponse, ControlRequest, ControlResponse,
DurableDecision, GenerationIdentity, NonInteractiveSessionMessage,
NonInteractiveSessionRequest, ProtocolCredential, ProtocolCredentialRole,
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, SandboxCredentialBundle,
Session, SessionBinding, SessionExit, SessionPhase, SessionRegistrationRequest,
SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId, TrustManifest,
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, PublicationOwnership,
SandboxCredentialBundle, Session, SessionBinding, SessionExit, SessionPhase,
SessionRegistrationRequest, SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId,
TrustManifest,
};
use rcgen::{
BasicConstraints, CertificateParams, CertificateSigningRequestParams, CertifiedIssuer,
@ -46,6 +48,7 @@ use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinator
use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, SessionAdmissionIdentity, SessionLifecycleAdapter,
};
use lazaret_linux::filesystem::mount_identity;
use lazaret_linux::process::{ProcessExit, TargetCommand, child_broker};
use lazaret_runtime_supervisor::sandbox::RunscCommand;
@ -53,10 +56,10 @@ pub const CONTROL_ALPN: &[u8] = b"lazaret-control/1";
pub const CONTROL_SERVER_NAME: &str = "coordinator.lazaret.invalid";
pub const MAX_ENROLLMENT_BYTES: usize = 8_192;
const MAX_CONTROL_BYTES: usize = 65_536;
const CONTROL_SOCKET_NAME: &str = "c.sock";
const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard";
const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
pub(crate) const CONTROL_SOCKET_NAME: &str = "c.sock";
pub(crate) const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard";
pub(crate) const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
pub(crate) const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
#[derive(Debug)]
pub enum ControlError {
@ -1009,6 +1012,10 @@ pub struct StagedControlFace {
}
impl StagedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Stages an undiscoverable private control face before durable publication.
///
/// # Errors
@ -1160,7 +1167,51 @@ impl Drop for StagedControlFace {
}
}
fn publication_recovery_ownership(
paths: &ControlPaths,
) -> Result<PublicationOwnership, ControlError> {
let socket_path = if paths.staged_socket.exists() {
&paths.staged_socket
} else {
&paths.socket
};
let identity_path = if paths.staged_identity.exists() {
&paths.staged_identity
} else {
&paths.identity
};
Ok(PublicationOwnership {
staged_socket_path: Some(HostPath::new(
STAGED_CONTROL_SOCKET_NAME.as_bytes().to_vec(),
)),
staged_identity_path: Some(HostPath::new(
STAGED_PUBLICATION_IDENTITY_NAME.as_bytes().to_vec(),
)),
socket_path: Some(HostPath::new(CONTROL_SOCKET_NAME.as_bytes().to_vec())),
identity_path: Some(HostPath::new(PUBLICATION_IDENTITY_NAME.as_bytes().to_vec())),
socket_identity: Some(publication_path_identity(paths, socket_path)?),
identity_file_identity: Some(publication_path_identity(paths, identity_path)?),
})
}
fn publication_path_identity(
paths: &ControlPaths,
path: &Path,
) -> Result<MountIdentity, ControlError> {
let metadata = std::fs::symlink_metadata(path)?;
let root = mount_identity(&paths.root_file)?;
Ok(MountIdentity {
mount_id: root.mount_id,
device: DeviceId::new(metadata.dev()),
inode: InodeId::new(metadata.ino()),
})
}
impl PublishedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Selects blocking or polling admission for the publication listener.
///
/// # Errors

8
crates/lazaret-host-coordinator/src/domain_store/tests.rs
View file

@ -112,8 +112,12 @@ fn publication_partial() -> Participant {
ParticipantRole::Publication,
ParticipantState::Intent,
Some(ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: None,
staged_identity_path: None,
socket_path: Some(HostPath::new(b"control.sock".to_vec())),
identity_path: None,
socket_identity: None,
identity_file_identity: None,
})),
)
.unwrap()
@ -364,8 +368,12 @@ fn lifecycle_updates_enforce_transition_and_absence_boundaries() {
ParticipantRole::Publication,
ParticipantState::Clean,
Some(ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: None,
staged_identity_path: None,
socket_path: None,
identity_path: None,
socket_identity: None,
identity_file_identity: None,
})),
)
.unwrap(),

17
crates/lazaret-host-coordinator/src/production.rs
View file

@ -10,12 +10,12 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration};
use lazaret_core::identity::{BundleDigest, ContainerId};
use lazaret_core::identity::{BundleDigest, ContainerId, MountIdentity};
use lazaret_core::network::{ReachabilityExpectation, derive_ipv4_policy};
use lazaret_core::records::{
Generation, GenerationIdentity, HostProcessOwnershipSlot, IpAddress, PreparedArtifact, Session,
};
use lazaret_linux::filesystem::namespace_identity;
use lazaret_linux::filesystem::{mount_identity, namespace_identity};
use lazaret_linux::process::ProcessOwnershipContext;
use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::{
@ -371,6 +371,7 @@ fn require_inputs(inputs: &ProductionColdStartInputs<'_>) -> Result<(), Producti
}
pub(crate) struct RuntimeStaging {
runsc_root: OwnedFd,
bundles: OwnedFd,
bundle: OwnedFd,
controls: OwnedFd,
@ -384,6 +385,10 @@ pub(crate) struct RuntimeStaging {
}
impl RuntimeStaging {
pub(crate) fn runsc_root_identity(&self) -> Result<MountIdentity, ProductionColdStartError> {
mount_identity(&self.runsc_root).map_err(failure)
}
pub(crate) fn cleanup_overlay(&self) -> Result<(), ProductionColdStartError> {
let retained = fstat(&self.overlay).map_err(failure)?;
if !self.overlay_identity.matches(&retained) {
@ -418,12 +423,7 @@ impl RuntimeStaging {
open_or_create_private_directory(&root, "sandbox-control", invoking_uid, false)?;
let overlays =
open_or_create_private_directory(&root, "runsc-overlay", invoking_uid, false)?;
drop(open_or_create_private_directory(
&root,
"runsc",
invoking_uid,
false,
)?);
let runsc_root = open_or_create_private_directory(&root, "runsc", invoking_uid, false)?;
let name = OsString::from(hex(container_id.as_bytes()));
let bundle = open_or_create_private_directory(&bundles, &name, invoking_uid, true)?;
let bundle_identity = FileIdentity::capture(&bundle)?;
@ -445,6 +445,7 @@ impl RuntimeStaging {
};
let overlay_identity = FileIdentity::capture(&overlay)?;
let staging = Self {
runsc_root,
bundles,
bundle,
controls,

905
crates/lazaret-host-coordinator/src/recovery.rs
View file

File diff suppressed because it is too large Load diff

2
crates/lazaret-host-coordinator/src/runtime.rs
View file

@ -107,6 +107,7 @@ impl<'store> SandboxCoordinator<'store> {
generation_id: GenerationId,
container_id: ContainerId,
runtime_root: &Path,
runtime_root_identity: MountIdentity,
bundle_digest: BundleDigest,
host_coordinator: PidfsProcessIdentityV1,
) -> Result<(), SandboxCoordinatorError> {
@ -123,6 +124,7 @@ impl<'store> SandboxCoordinator<'store> {
let ownership = SandboxOwnership {
container_id: Some(container_id),
runtime_root: Some(HostPath::new(runtime_root.as_os_str().as_bytes().to_vec())),
runtime_root_identity: Some(runtime_root_identity),
bundle_digest: Some(bundle_digest),
host_coordinator: PersistedHostProcessOwnership::committed(host_coordinator),
runtime_supervisor: PersistedHostProcessOwnership::unassigned(),

4
crates/lazaret-host-coordinator/src/runtime/tests.rs
View file

@ -6,7 +6,7 @@ use lazaret_core::identity::{
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks, SandboxId,
};
use lazaret_core::records::{Domain, DurableDecision, Generation};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_linux::filesystem::{current_mount_identity, try_exclusive_flock};
use tempfile::TempDir;
use super::*;
@ -91,6 +91,7 @@ fn sandbox_staging_readiness_and_release_preserve_exact_ownership() {
generation().generation_id,
ContainerId::new(bytes(6)),
directory.path(),
current_mount_identity(directory.path()).unwrap(),
BundleDigest::new(bytes(7)),
process(100),
)
@ -155,6 +156,7 @@ fn mismatched_and_skipped_acquisition_boundaries_refuse_without_state_change() {
generation().generation_id,
ContainerId::new(bytes(6)),
directory.path(),
current_mount_identity(directory.path()).unwrap(),
BundleDigest::new(bytes(7)),
process(100),
)

30
crates/lazaret-host-coordinator/src/state_anchor.rs
View file

@ -12,7 +12,7 @@ use lazaret_core::records::{
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
StateAnchorReleaseEvidence,
};
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use lazaret_linux::process::RetainedProcess;
use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness};
use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_mount};
use lazaret_storage_catalog::{Catalog, CatalogError};
@ -80,6 +80,12 @@ pub struct StateAnchorReleaseAuthority {
mount_namespace: NamespaceIdentity,
}
#[derive(Clone, Copy)]
pub(crate) enum StateAnchorRecoveryOwner<'owner> {
Matched(&'owner RetainedProcess),
Absent(&'owner RuntimeOwnersAbsent),
}
impl StateAnchorReleaseAuthority {
#[must_use]
pub const fn generation_id(&self) -> GenerationId {
@ -380,33 +386,28 @@ impl<'a> StateAnchorCoordinator<'a> {
/// Refuses generation drift, pidfs/namespace/mount mismatch, unreadable absence, or helper
/// failure. Boot mismatch and `ESRCH` reconcile physical absence without process action only
/// under the caller-provided ordered runtime-owner absence witness.
pub fn recover_after_runtime_owner_loss(
pub(crate) fn recover_after_runtime_owner_loss(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
owners_absent: Option<&RuntimeOwnersAbsent>,
owner: StateAnchorRecoveryOwner<'_>,
) -> Result<
(StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
StateAnchorCoordinatorError,
> {
let authority = self.begin_release(generation_id, layout)?;
let observed = match match_recovery_process(authority.daemon) {
Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
let observed = match owner {
StateAnchorRecoveryOwner::Matched(holder) => recover_private_state_anchor_mount(
layout,
&holder,
holder,
authority.user_namespace,
authority.mount_namespace,
authority.mount,
)?,
Ok(PidfsRecoveryMatch::BootMismatch) => {
StateAnchorRecoveryOwner::Absent(owners_absent) => {
self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)?
}
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)?
}
Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
};
Ok((
authority.clone(),
@ -421,12 +422,9 @@ impl<'a> StateAnchorCoordinator<'a> {
fn require_absence_witness(
&self,
witness: Option<&RuntimeOwnersAbsent>,
witness: &RuntimeOwnersAbsent,
authority: &StateAnchorReleaseAuthority,
) -> Result<(), StateAnchorCoordinatorError> {
let witness = witness.ok_or(StateAnchorCoordinatorError::InvalidState(
"complete runtime-owner absence proof is required",
))?;
let snapshot = self.domain_store.snapshot()?;
let generation = snapshot
.generations