ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Enforce exact recovery ownership before cold start

Browse source
This commit is contained in:
ms2k 2026-07-17 06:48:49 +09:00
commit 2bc5982a62
14 changed files with 1076 additions and 238 deletions
Download patch file Download diff file Expand all files Collapse all files

45
crates/lazaret-core/src/records/lifecycle.rs
View file

@ -1033,6 +1033,7 @@ pub struct HelperOwnership {
pub struct SandboxOwnership {
pub container_id: Option<ContainerId>,
pub runtime_root: Option<HostPath>,
pub runtime_root_identity: Option<MountIdentity>,
pub bundle_digest: Option<BundleDigest>,
pub host_coordinator: PersistedHostProcessOwnership,
pub runtime_supervisor: PersistedHostProcessOwnership,
@ -1041,8 +1042,12 @@ pub struct SandboxOwnership {
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct PublicationOwnership {
pub staged_socket_path: Option<HostPath>,
pub staged_identity_path: Option<HostPath>,
pub socket_path: Option<HostPath>,
pub identity_path: Option<HostPath>,
pub socket_identity: Option<MountIdentity>,
pub identity_file_identity: Option<MountIdentity>,
}
impl PreparedLowerOwnership {
@ -1309,6 +1314,7 @@ impl SandboxOwnership {
Self {
container_id: None,
runtime_root: None,
runtime_root_identity: None,
bundle_digest: None,
host_coordinator: PersistedHostProcessOwnership::unassigned(),
runtime_supervisor: PersistedHostProcessOwnership::unassigned(),
@ -1319,6 +1325,7 @@ impl SandboxOwnership {
fn is_empty(&self) -> bool {
self.container_id.is_none()
&& self.runtime_root.is_none()
&& self.runtime_root_identity.is_none()
&& self.bundle_digest.is_none()
&& self.host_coordinator.is_unassigned()
&& self.runtime_supervisor.is_unassigned()
@ -1328,6 +1335,7 @@ impl SandboxOwnership {
fn is_committed(&self) -> bool {
self.container_id.is_some()
&& self.runtime_root.is_some()
&& self.runtime_root_identity.is_some()
&& self.bundle_digest.is_some()
&& self.host_coordinator.is_current_committed()
&& self.runtime_supervisor.is_current_committed()
@ -1337,6 +1345,10 @@ impl SandboxOwnership {
fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
validate_option_progress(self.container_id.as_ref(), next.container_id.as_ref())?;
validate_option_progress(self.runtime_root.as_ref(), next.runtime_root.as_ref())?;
validate_option_progress(
self.runtime_root_identity.as_ref(),
next.runtime_root_identity.as_ref(),
)?;
validate_option_progress(self.bundle_digest.as_ref(), next.bundle_digest.as_ref())?;
validate_existing_process_progress(&self.host_coordinator, &next.host_coordinator)?;
validate_process_progress(&self.runtime_supervisor, &next.runtime_supervisor)?;
@ -1347,22 +1359,49 @@ impl SandboxOwnership {
impl PublicationOwnership {
fn empty() -> Self {
Self {
staged_socket_path: None,
staged_identity_path: None,
socket_path: None,
identity_path: None,
socket_identity: None,
identity_file_identity: None,
}
}
fn is_empty(&self) -> bool {
self.socket_path.is_none() && self.identity_path.is_none()
self.staged_socket_path.is_none()
&& self.staged_identity_path.is_none()
&& self.socket_path.is_none()
&& self.identity_path.is_none()
&& self.socket_identity.is_none()
&& self.identity_file_identity.is_none()
}
fn is_committed(&self) -> bool {
self.socket_path.is_some() && self.identity_path.is_some()
self.staged_socket_path.is_some()
&& self.staged_identity_path.is_some()
&& self.socket_path.is_some()
&& self.identity_path.is_some()
&& self.socket_identity.is_some()
&& self.identity_file_identity.is_some()
}
fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
validate_option_progress(
self.staged_socket_path.as_ref(),
next.staged_socket_path.as_ref(),
)?;
validate_option_progress(
self.staged_identity_path.as_ref(),
next.staged_identity_path.as_ref(),
)?;
validate_option_progress(self.socket_path.as_ref(), next.socket_path.as_ref())?;
validate_option_progress(self.identity_path.as_ref(), next.identity_path.as_ref())
validate_option_progress(self.identity_path.as_ref(), next.identity_path.as_ref())?;
validate_option_progress(self.socket_identity.as_ref(), next.socket_identity.as_ref())?;
validate_option_progress(
self.identity_file_identity.as_ref(),
next.identity_file_identity.as_ref(),
)
}
}

8
crates/lazaret-core/tests/schema_records.rs
View file

@ -457,14 +457,19 @@ fn complete_ownership(role: ParticipantRole, state: ParticipantState) -> Partici
ParticipantRole::Sandbox => ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))),
runtime_root: Some(HostPath::new(b"runtime".to_vec())),
runtime_root_identity: Some(mount()),
bundle_digest: Some(BundleDigest::new(opaque(43))),
host_coordinator: owned_process(30),
runtime_supervisor: owned_process(21),
attached_runsc: owned_process(44),
}),
ParticipantRole::Publication => ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: Some(HostPath::new(b"staged-socket".to_vec())),
staged_identity_path: Some(HostPath::new(b"staged-identity".to_vec())),
socket_path: Some(HostPath::new(b"socket".to_vec())),
identity_path: Some(HostPath::new(b"identity".to_vec())),
socket_identity: Some(mount()),
identity_file_identity: Some(mount()),
}),
}
}
@ -558,6 +563,7 @@ fn participant_edge_snapshot(
ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))),
runtime_root: None,
runtime_root_identity: None,
bundle_digest: None,
host_coordinator: process_ownership,
runtime_supervisor: PersistedHostProcessOwnership::unassigned(),
@ -569,6 +575,7 @@ fn participant_edge_snapshot(
ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))),
runtime_root: None,
runtime_root_identity: None,
bundle_digest: None,
host_coordinator: owned_process(122),
runtime_supervisor: process_ownership,
@ -580,6 +587,7 @@ fn participant_edge_snapshot(
ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))),
runtime_root: None,
runtime_root_identity: None,
bundle_digest: None,
host_coordinator: owned_process(122),
runtime_supervisor: owned_process(123),

17
crates/lazaret-core/tests/schema_records/encoding.rs
View file

@ -375,6 +375,11 @@ fn every_schema_shape_round_trips_and_rejects_malformed_or_unknown_versions() {
SandboxOwnership {
container_id: Some(ContainerId::new(opaque(29))),
runtime_root: Some(HostPath::new(b"runtime".to_vec())),
runtime_root_identity: Some(MountIdentity {
mount_id: MountId::new(68),
device: DeviceId::new(69),
inode: InodeId::new(70),
}),
bundle_digest: Some(BundleDigest::new(opaque(66))),
host_coordinator: owned_process(30),
runtime_supervisor: owned_process(21),
@ -384,8 +389,20 @@ fn every_schema_shape_round_trips_and_rejects_malformed_or_unknown_versions() {
);
verify_shape(
PublicationOwnership {
staged_socket_path: Some(HostPath::new(b"staged-socket".to_vec())),
staged_identity_path: Some(HostPath::new(b"staged-identity".to_vec())),
socket_path: Some(HostPath::new(b"socket".to_vec())),
identity_path: Some(HostPath::new(b"identity".to_vec())),
socket_identity: Some(MountIdentity {
mount_id: MountId::new(71),
device: DeviceId::new(72),
inode: InodeId::new(73),
}),
identity_file_identity: Some(MountIdentity {
mount_id: MountId::new(74),
device: DeviceId::new(75),
inode: InodeId::new(76),
}),
},
&mut seen,
);

93
crates/lazaret-host-coordinator/src/bootstrap.rs
View file

@ -7,7 +7,10 @@ use lazaret_core::configuration::ResolvedDomainConfiguration;
use lazaret_core::identity::{BundleDigest, ContainerId, MountIdentity};
use lazaret_core::network::{CanonicalDerivedNetworkPolicy, ReachabilityExpectation};
use lazaret_core::records::Session;
use lazaret_core::records::{Generation, GenerationIdentity, PreparedArtifact};
use lazaret_core::records::{
Generation, GenerationIdentity, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PreparedArtifact,
};
use lazaret_linux::process::current_process_identity;
use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::SandboxBundle;
@ -20,7 +23,7 @@ use crate::control::{
ControlError, NonInteractiveSessionExecutor, PublishedControlFace, SessionRegistry,
StagedControlFace, TrustAnchor,
};
use crate::domain_store::HostCoordinatorDomainStore;
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, LifecycleAdapter, SessionAdmissionIdentity,
SessionLifecycleAdapter,
@ -240,6 +243,10 @@ impl<'a> ProductionLifecycleAdapter<'a> {
self.plan.generation_identity.generation_id,
self.plan.container_id,
&self.plan.runtime_root,
self.plan
.runtime_staging
.runsc_root_identity()
.map_err(failure)?,
self.plan.bundle_digest,
current_process_identity().map_err(failure)?,
)
@ -347,8 +354,14 @@ impl<'a> ProductionLifecycleAdapter<'a> {
.trust_anchor
.take()
.ok_or_else(|| ProductionLifecycleError("Trust Anchor is absent".into()))?;
self.staged_control = Some(
StagedControlFace::stage_with_executor(
let generation_id = self.plan.generation_identity.generation_id;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::bare_intent(generation_id, ParticipantRole::Publication),
)])
.map_err(failure)?;
let staged = StagedControlFace::stage_with_executor(
path_from_host(
self.plan
.configuration
@ -361,22 +374,85 @@ impl<'a> ProductionLifecycleAdapter<'a> {
Arc::clone(&self.registry),
Arc::clone(&self.plan.non_interactive_executor),
)
.map_err(failure)?;
let ownership = staged.recovery_ownership().map_err(failure)?;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::Intent,
Some(ParticipantOwnership::Publication(ownership)),
)
.map_err(failure)?,
);
)])
.map_err(failure)?;
self.staged_control = Some(staged);
Ok(())
}
fn expose_publication(&mut self) -> Result<(), ProductionLifecycleError> {
self.published_control = Some(Arc::new(
self.staged_control
let published = self
.staged_control
.take()
.ok_or_else(|| ProductionLifecycleError("staged control face is absent".into()))?
.expose()
.map_err(failure)?;
let ownership = published.recovery_ownership().map_err(failure)?;
self.plan
.store
.apply(&[DomainStoreMutation::PutParticipant(
Participant::snapshot(
self.plan.generation_identity.generation_id,
ParticipantRole::Publication,
ParticipantState::Committed,
Some(ParticipantOwnership::Publication(ownership)),
)
.map_err(failure)?,
));
)])
.map_err(failure)?;
self.published_control = Some(Arc::new(published));
Ok(())
}
fn clean_publication_after_withdrawal(&self) -> Result<(), ProductionLifecycleError> {
let generation_id = self.plan.generation_identity.generation_id;
let current = self
.plan
.store
.snapshot()
.map_err(failure)?
.participants
.into_iter()
.find(|participant| {
participant.generation_id() == generation_id
&& participant.role() == ParticipantRole::Publication
})
.ok_or_else(|| ProductionLifecycleError("Publication participant is absent".into()))?;
let release = Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::ReleaseIntent,
current.ownership().cloned(),
)
.map_err(failure)?;
let clean = Participant::snapshot(
generation_id,
ParticipantRole::Publication,
ParticipantState::Clean,
Some(ParticipantOwnership::empty(ParticipantRole::Publication)),
)
.map_err(failure)?;
self.plan
.store
.apply(&[
DomainStoreMutation::PutParticipant(release),
DomainStoreMutation::CleanParticipantAfterAbsence(clean),
])
.map_err(failure)
}
fn stop_sandbox(&mut self) -> Result<(), ProductionLifecycleError> {
let store = Arc::clone(&self.plan.store);
let sandbox = SandboxCoordinator::new(&store);
@ -517,6 +593,7 @@ impl LifecycleAdapter for ProductionLifecycleAdapter<'_> {
LifecycleAction::BeginTeardown => {
if let Some(control) = self.published_control.as_ref() {
control.withdraw().map_err(failure)?;
self.clean_publication_after_withdrawal()?;
}
self.published_control.take();
Ok(())

67
crates/lazaret-host-coordinator/src/control.rs
View file

@ -13,7 +13,8 @@ use ed25519_dalek::SigningKey;
use lazaret_core::codec::{CodecError, decode, encode};
use lazaret_core::configuration::ConfinedPathPlan;
use lazaret_core::identity::{
CertificateDigest, ClientId, ContainerId, SessionAnchorId, SessionId, TrustManifestDigest,
CertificateDigest, ClientId, ContainerId, DeviceId, HostPath, InodeId, MountIdentity,
SessionAnchorId, SessionId, TrustManifestDigest,
};
use lazaret_core::protocol::{
ProtocolValidationError, issue_protocol_credential, issue_session_capability,
@ -23,9 +24,10 @@ use lazaret_core::records::{
ClientEnrollmentRequest, ClientEnrollmentResponse, ControlRequest, ControlResponse,
DurableDecision, GenerationIdentity, NonInteractiveSessionMessage,
NonInteractiveSessionRequest, ProtocolCredential, ProtocolCredentialRole,
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, SandboxCredentialBundle,
Session, SessionBinding, SessionExit, SessionPhase, SessionRegistrationRequest,
SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId, TrustManifest,
ProtocolPublicationIdentity, ProtocolSubjectId, ProtocolVersion, PublicationOwnership,
SandboxCredentialBundle, Session, SessionBinding, SessionExit, SessionPhase,
SessionRegistrationRequest, SessionRegistrationResponse, TlsCredential, TlsRole, TlsSubjectId,
TrustManifest,
};
use rcgen::{
BasicConstraints, CertificateParams, CertificateSigningRequestParams, CertifiedIssuer,
@ -46,6 +48,7 @@ use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinator
use crate::lifecycle::{
HostCoordinatorLifecycle, LifecycleAction, SessionAdmissionIdentity, SessionLifecycleAdapter,
};
use lazaret_linux::filesystem::mount_identity;
use lazaret_linux::process::{ProcessExit, TargetCommand, child_broker};
use lazaret_runtime_supervisor::sandbox::RunscCommand;
@ -53,10 +56,10 @@ pub const CONTROL_ALPN: &[u8] = b"lazaret-control/1";
pub const CONTROL_SERVER_NAME: &str = "coordinator.lazaret.invalid";
pub const MAX_ENROLLMENT_BYTES: usize = 8_192;
const MAX_CONTROL_BYTES: usize = 65_536;
const CONTROL_SOCKET_NAME: &str = "c.sock";
const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard";
const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
pub(crate) const CONTROL_SOCKET_NAME: &str = "c.sock";
pub(crate) const PUBLICATION_IDENTITY_NAME: &str = "protocol-identity.postcard";
pub(crate) const STAGED_CONTROL_SOCKET_NAME: &str = ".c.staged";
pub(crate) const STAGED_PUBLICATION_IDENTITY_NAME: &str = ".protocol-identity.staged";
#[derive(Debug)]
pub enum ControlError {
@ -1009,6 +1012,10 @@ pub struct StagedControlFace {
}
impl StagedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Stages an undiscoverable private control face before durable publication.
///
/// # Errors
@ -1160,7 +1167,51 @@ impl Drop for StagedControlFace {
}
}
fn publication_recovery_ownership(
paths: &ControlPaths,
) -> Result<PublicationOwnership, ControlError> {
let socket_path = if paths.staged_socket.exists() {
&paths.staged_socket
} else {
&paths.socket
};
let identity_path = if paths.staged_identity.exists() {
&paths.staged_identity
} else {
&paths.identity
};
Ok(PublicationOwnership {
staged_socket_path: Some(HostPath::new(
STAGED_CONTROL_SOCKET_NAME.as_bytes().to_vec(),
)),
staged_identity_path: Some(HostPath::new(
STAGED_PUBLICATION_IDENTITY_NAME.as_bytes().to_vec(),
)),
socket_path: Some(HostPath::new(CONTROL_SOCKET_NAME.as_bytes().to_vec())),
identity_path: Some(HostPath::new(PUBLICATION_IDENTITY_NAME.as_bytes().to_vec())),
socket_identity: Some(publication_path_identity(paths, socket_path)?),
identity_file_identity: Some(publication_path_identity(paths, identity_path)?),
})
}
fn publication_path_identity(
paths: &ControlPaths,
path: &Path,
) -> Result<MountIdentity, ControlError> {
let metadata = std::fs::symlink_metadata(path)?;
let root = mount_identity(&paths.root_file)?;
Ok(MountIdentity {
mount_id: root.mount_id,
device: DeviceId::new(metadata.dev()),
inode: InodeId::new(metadata.ino()),
})
}
impl PublishedControlFace {
pub(crate) fn recovery_ownership(&self) -> Result<PublicationOwnership, ControlError> {
publication_recovery_ownership(&self.paths)
}
/// Selects blocking or polling admission for the publication listener.
///
/// # Errors

8
crates/lazaret-host-coordinator/src/domain_store/tests.rs
View file

@ -112,8 +112,12 @@ fn publication_partial() -> Participant {
ParticipantRole::Publication,
ParticipantState::Intent,
Some(ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: None,
staged_identity_path: None,
socket_path: Some(HostPath::new(b"control.sock".to_vec())),
identity_path: None,
socket_identity: None,
identity_file_identity: None,
})),
)
.unwrap()
@ -364,8 +368,12 @@ fn lifecycle_updates_enforce_transition_and_absence_boundaries() {
ParticipantRole::Publication,
ParticipantState::Clean,
Some(ParticipantOwnership::Publication(PublicationOwnership {
staged_socket_path: None,
staged_identity_path: None,
socket_path: None,
identity_path: None,
socket_identity: None,
identity_file_identity: None,
})),
)
.unwrap(),

17
crates/lazaret-host-coordinator/src/production.rs
View file

@ -10,12 +10,12 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration};
use lazaret_core::identity::{BundleDigest, ContainerId};
use lazaret_core::identity::{BundleDigest, ContainerId, MountIdentity};
use lazaret_core::network::{ReachabilityExpectation, derive_ipv4_policy};
use lazaret_core::records::{
Generation, GenerationIdentity, HostProcessOwnershipSlot, IpAddress, PreparedArtifact, Session,
};
use lazaret_linux::filesystem::namespace_identity;
use lazaret_linux::filesystem::{mount_identity, namespace_identity};
use lazaret_linux::process::ProcessOwnershipContext;
use lazaret_runtime_supervisor::network::NetworkProbeRequest;
use lazaret_runtime_supervisor::sandbox::{
@ -371,6 +371,7 @@ fn require_inputs(inputs: &ProductionColdStartInputs<'_>) -> Result<(), Producti
}
pub(crate) struct RuntimeStaging {
runsc_root: OwnedFd,
bundles: OwnedFd,
bundle: OwnedFd,
controls: OwnedFd,
@ -384,6 +385,10 @@ pub(crate) struct RuntimeStaging {
}
impl RuntimeStaging {
pub(crate) fn runsc_root_identity(&self) -> Result<MountIdentity, ProductionColdStartError> {
mount_identity(&self.runsc_root).map_err(failure)
}
pub(crate) fn cleanup_overlay(&self) -> Result<(), ProductionColdStartError> {
let retained = fstat(&self.overlay).map_err(failure)?;
if !self.overlay_identity.matches(&retained) {
@ -418,12 +423,7 @@ impl RuntimeStaging {
open_or_create_private_directory(&root, "sandbox-control", invoking_uid, false)?;
let overlays =
open_or_create_private_directory(&root, "runsc-overlay", invoking_uid, false)?;
drop(open_or_create_private_directory(
&root,
"runsc",
invoking_uid,
false,
)?);
let runsc_root = open_or_create_private_directory(&root, "runsc", invoking_uid, false)?;
let name = OsString::from(hex(container_id.as_bytes()));
let bundle = open_or_create_private_directory(&bundles, &name, invoking_uid, true)?;
let bundle_identity = FileIdentity::capture(&bundle)?;
@ -445,6 +445,7 @@ impl RuntimeStaging {
};
let overlay_identity = FileIdentity::capture(&overlay)?;
let staging = Self {
runsc_root,
bundles,
bundle,
controls,

887
crates/lazaret-host-coordinator/src/recovery.rs
View file

File diff suppressed because it is too large Load diff

2
crates/lazaret-host-coordinator/src/runtime.rs
View file

@ -107,6 +107,7 @@ impl<'store> SandboxCoordinator<'store> {
generation_id: GenerationId,
container_id: ContainerId,
runtime_root: &Path,
runtime_root_identity: MountIdentity,
bundle_digest: BundleDigest,
host_coordinator: PidfsProcessIdentityV1,
) -> Result<(), SandboxCoordinatorError> {
@ -123,6 +124,7 @@ impl<'store> SandboxCoordinator<'store> {
let ownership = SandboxOwnership {
container_id: Some(container_id),
runtime_root: Some(HostPath::new(runtime_root.as_os_str().as_bytes().to_vec())),
runtime_root_identity: Some(runtime_root_identity),
bundle_digest: Some(bundle_digest),
host_coordinator: PersistedHostProcessOwnership::committed(host_coordinator),
runtime_supervisor: PersistedHostProcessOwnership::unassigned(),

4
crates/lazaret-host-coordinator/src/runtime/tests.rs
View file

@ -6,7 +6,7 @@ use lazaret_core::identity::{
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks, SandboxId,
};
use lazaret_core::records::{Domain, DurableDecision, Generation};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_linux::filesystem::{current_mount_identity, try_exclusive_flock};
use tempfile::TempDir;
use super::*;
@ -91,6 +91,7 @@ fn sandbox_staging_readiness_and_release_preserve_exact_ownership() {
generation().generation_id,
ContainerId::new(bytes(6)),
directory.path(),
current_mount_identity(directory.path()).unwrap(),
BundleDigest::new(bytes(7)),
process(100),
)
@ -155,6 +156,7 @@ fn mismatched_and_skipped_acquisition_boundaries_refuse_without_state_change() {
generation().generation_id,
ContainerId::new(bytes(6)),
directory.path(),
current_mount_identity(directory.path()).unwrap(),
BundleDigest::new(bytes(7)),
process(100),
)

30
crates/lazaret-host-coordinator/src/state_anchor.rs
View file

@ -12,7 +12,7 @@ use lazaret_core::records::{
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
StateAnchorReleaseEvidence,
};
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use lazaret_linux::process::RetainedProcess;
use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness};
use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_mount};
use lazaret_storage_catalog::{Catalog, CatalogError};
@ -80,6 +80,12 @@ pub struct StateAnchorReleaseAuthority {
mount_namespace: NamespaceIdentity,
}
#[derive(Clone, Copy)]
pub(crate) enum StateAnchorRecoveryOwner<'owner> {
Matched(&'owner RetainedProcess),
Absent(&'owner RuntimeOwnersAbsent),
}
impl StateAnchorReleaseAuthority {
#[must_use]
pub const fn generation_id(&self) -> GenerationId {
@ -380,33 +386,28 @@ impl<'a> StateAnchorCoordinator<'a> {
/// Refuses generation drift, pidfs/namespace/mount mismatch, unreadable absence, or helper
/// failure. Boot mismatch and `ESRCH` reconcile physical absence without process action only
/// under the caller-provided ordered runtime-owner absence witness.
pub fn recover_after_runtime_owner_loss(
pub(crate) fn recover_after_runtime_owner_loss(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
owners_absent: Option<&RuntimeOwnersAbsent>,
owner: StateAnchorRecoveryOwner<'_>,
) -> Result<
(StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
StateAnchorCoordinatorError,
> {
let authority = self.begin_release(generation_id, layout)?;
let observed = match match_recovery_process(authority.daemon) {
Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
let observed = match owner {
StateAnchorRecoveryOwner::Matched(holder) => recover_private_state_anchor_mount(
layout,
&holder,
holder,
authority.user_namespace,
authority.mount_namespace,
authority.mount,
)?,
Ok(PidfsRecoveryMatch::BootMismatch) => {
StateAnchorRecoveryOwner::Absent(owners_absent) => {
self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)?
}
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)?
}
Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
};
Ok((
authority.clone(),
@ -421,12 +422,9 @@ impl<'a> StateAnchorCoordinator<'a> {
fn require_absence_witness(
&self,
witness: Option<&RuntimeOwnersAbsent>,
witness: &RuntimeOwnersAbsent,
authority: &StateAnchorReleaseAuthority,
) -> Result<(), StateAnchorCoordinatorError> {
let witness = witness.ok_or(StateAnchorCoordinatorError::InvalidState(
"complete runtime-owner absence proof is required",
))?;
let snapshot = self.domain_store.snapshot()?;
let generation = snapshot
.generations

79
crates/lazaret-host-coordinator/tests/recovery-process.rs
View file

@ -1,14 +1,22 @@
use std::fs::{self, File};
use std::io;
use std::os::fd::OwnedFd;
use std::os::unix::fs::PermissionsExt;
use std::process::ExitCode;
use lazaret_core::identity::{
ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, NetworkSidecarId, SandboxId,
BootId, BundleDigest, ConfigurationFingerprint, ContainerId, CoordinatorId, DomainId,
GenerationId, HostPath, HostPidLocator, MountId, MountIdentity, NetworkSidecarId,
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, PidfsProcessIdentityV1,
ProcessStartTimeTicks, SandboxId,
};
use lazaret_core::records::{
DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PersistedHostProcessOwnership, StateAnchorOwnership,
ParticipantState, PersistedHostProcessOwnership, SandboxOwnership, StateAnchorOwnership,
};
use lazaret_host_coordinator::recovery::{
delete_runsc_container, prove_runtime_owners_absent, runtime_recovery_ownership,
};
use lazaret_host_coordinator::recovery::{prove_runtime_owners_absent, runtime_recovery_ownership};
use lazaret_linux::process::{TargetCommand, child_broker, dispatch_private_process_role};
use rustix::process::Signal;
@ -31,19 +39,61 @@ fn exit(result: io::Result<()>) -> ExitCode {
}
fn run() -> io::Result<()> {
verify_exact_runsc_delete()?;
verify_exact_action(None)?;
verify_exact_action(Some(Signal::STOP))
}
fn verify_exact_runsc_delete() -> io::Result<()> {
let tree = tempfile::tempdir()?;
let root_path = tree.path().join("runsc");
fs::create_dir(&root_path)?;
fs::create_dir(root_path.join("container"))?;
let program = tree.path().join("runsc-fixture");
fs::write(
&program,
b"#!/bin/sh\nroot=${1#-root=}\n[ \"$2\" = delete ] && [ \"$3\" = --force ] && rmdir \"$root/$4\"\n",
)?;
fs::set_permissions(&program, fs::Permissions::from_mode(0o700))?;
let root: OwnedFd = File::open(&root_path)?.into();
delete_runsc_container(&root, "container", program.as_os_str()).map_err(io::Error::other)?;
if root_path.join("container").exists() {
return Err(io::Error::other("exact runsc container remained"));
}
Ok(())
}
fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
let captured =
let state =
child_broker().run_transient_captured(&TargetCommand::new("/bin/sleep").argument("60"))?;
if let Some(signal) = signal {
captured.process().send_signal(signal)?;
state.process().send_signal(signal)?;
}
let process = captured.into_process();
let state = state.into_process();
let supervisor = child_broker()
.run_transient_captured(&TargetCommand::new("/bin/sleep").argument("60"))?
.into_process();
let generation = generation();
let participants = [
Participant::snapshot(
generation.generation_id,
ParticipantRole::Sandbox,
ParticipantState::Intent,
Some(ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new([6; 32])),
runtime_root: Some(HostPath::new(b"/runtime/runsc".to_vec())),
runtime_root_identity: Some(MountIdentity {
mount_id: MountId::new(7),
device: lazaret_core::identity::DeviceId::new(8),
inode: lazaret_core::identity::InodeId::new(9),
}),
bundle_digest: Some(BundleDigest::new([10; 32])),
host_coordinator: PersistedHostProcessOwnership::committed(absent_process()),
runtime_supervisor: PersistedHostProcessOwnership::staged(supervisor.identity()),
attached_runsc: PersistedHostProcessOwnership::unassigned(),
})),
)
.map_err(|_| io::Error::other("Sandbox participant is invalid"))?,
Participant::bare_intent(generation.generation_id, ParticipantRole::PreparedLower),
Participant::snapshot(
generation.generation_id,
@ -54,7 +104,7 @@ fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
upper_path: None,
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process.identity()),
daemon: PersistedHostProcessOwnership::staged(state.identity()),
user_namespace: None,
mount_namespace: None,
mount: None,
@ -65,7 +115,20 @@ fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
let ownership =
runtime_recovery_ownership(&participants, &generation).map_err(io::Error::other)?;
prove_runtime_owners_absent(&ownership)?;
child_broker().reap(process)
child_broker().reap(state)?;
child_broker().reap(supervisor)
}
fn absent_process() -> PidfsProcessIdentityV1 {
PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new([0xff; 16]),
pidfs_st_dev: PidfsDeviceId::new(1),
pidfs_st_ino: PidfsInodeId::new(2),
},
HostPidLocator::new(1),
ProcessStartTimeTicks::new(3),
)
}
fn generation() -> Generation {

4
crates/lazaret-host-coordinator/tests/runtime-composition.rs
View file

@ -33,7 +33,7 @@ use lazaret_host_coordinator::terminal::{
NoActivePrompt, RunscSessionTerminalFactory, SessionTerminalEngine, TerminalAcquisition,
TerminalRoute,
};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_linux::filesystem::{current_mount_identity, try_exclusive_flock};
use lazaret_linux::platform::{PlatformObservation, check_platform};
use lazaret_linux::process::{
ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, current_process_identity,
@ -1023,6 +1023,7 @@ impl Fixture {
GenerationId::new(7),
container_id,
&runtime_domain.join("runsc"),
current_mount_identity(&runtime_domain.join("runsc"))?,
BundleDigest::new([7; 32]),
current_process_identity()?,
)
@ -1306,6 +1307,7 @@ fn network_store(
GenerationId::new(7),
container_id,
&runtime_domain.join("runsc"),
current_mount_identity(&runtime_domain.join("runsc"))?,
BundleDigest::new([17; 32]),
current_process_identity()?,
)

9
docs/schema.md
View file

@ -1051,6 +1051,7 @@ classDiagram
<<serialized>>
optional container_id
optional runtime_root
optional runtime_root_identity
optional bundle_digest
host_coordinator
runtime_supervisor
@ -1059,8 +1060,12 @@ classDiagram
class PublicationOwnership {
<<serialized>>
optional staged_socket_path
optional staged_identity_path
optional socket_path
optional identity_path
optional socket_identity
optional identity_file_identity
}
ParticipantOwnership --> PreparedLowerOwnership
@ -1106,8 +1111,8 @@ The only serviceable role/state pairs are:
| `GitRouting` | `Prepared` | three paths, mount, current committed daemon |
| `Network` | `Committed` | policy digest, fingerprint, readiness evidence, namespace, current committed RS |
| `Helper` | `Committed` | sidecar ID, current committed helper and RS |
| `Sandbox` | `Committed` | container, runtime root, bundle digest, current committed HC, RS, and runsc |
| `Publication` | `Committed` | socket and identity paths |
| `Sandbox` | `Committed` | container, runtime root and exact mount/device/inode identity, bundle digest, current committed HC, RS, and runsc |
| `Publication` | `Committed` | staged and exposed fixed labels plus exact socket and identity-file mount/device/inode identities |
Network readiness evidence must name the enclosing Participant generation and
match its policy digest, fingerprint, namespace, and committed RS identity.