ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Implement exact bootstrap recovery and artifact GC

Browse source
This commit is contained in:
ms2k 2026-07-17 05:38:43 +09:00
commit d25942596f
15 changed files with 1631 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

31
crates/lazaret-core/src/records/lifecycle.rs
View file

@ -495,6 +495,9 @@ impl Participant {
| ParticipantState::Committed,
ParticipantState::ReleaseIntent,
) => validate_participant_release(self.ownership.as_ref(), next.ownership.as_ref()),
(ParticipantState::ReleaseIntent, ParticipantState::ReleaseIntent) => {
validate_participant_release(self.ownership.as_ref(), next.ownership.as_ref())
}
(old_state, new_state)
if old_state == new_state
&& matches!(
@ -502,7 +505,6 @@ impl Participant {
ParticipantState::LeaseHeld
| ParticipantState::Prepared
| ParticipantState::Committed
| ParticipantState::ReleaseIntent
| ParticipantState::Clean
)
&& self.ownership == next.ownership =>
@ -979,6 +981,15 @@ pub struct PreparedLowerOwnership {
pub path: Option<HostPath>,
pub device: Option<DeviceId>,
pub inode: Option<InodeId>,
pub released_state_anchor: Option<StateAnchorReleaseEvidence>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct StateAnchorReleaseEvidence {
pub daemon: PidfsProcessIdentityV1,
pub user_namespace: NamespaceIdentity,
pub mount_namespace: NamespaceIdentity,
pub mount: MountIdentity,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
@ -1043,6 +1054,7 @@ impl PreparedLowerOwnership {
path: None,
device: None,
inode: None,
released_state_anchor: None,
}
}
@ -1053,6 +1065,7 @@ impl PreparedLowerOwnership {
&& self.path.is_none()
&& self.device.is_none()
&& self.inode.is_none()
&& self.released_state_anchor.is_none()
}
fn is_lease_held(&self) -> bool {
@ -1062,6 +1075,7 @@ impl PreparedLowerOwnership {
&& self.path.is_some()
&& self.device.is_some()
&& self.inode.is_some()
&& self.released_state_anchor.is_none()
}
fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
@ -1073,7 +1087,11 @@ impl PreparedLowerOwnership {
validate_prepared_lower_lease_progress(self.lease_state, next.lease_state)?;
validate_option_progress(self.path.as_ref(), next.path.as_ref())?;
validate_option_progress(self.device.as_ref(), next.device.as_ref())?;
validate_option_progress(self.inode.as_ref(), next.inode.as_ref())
validate_option_progress(self.inode.as_ref(), next.inode.as_ref())?;
validate_option_progress(
self.released_state_anchor.as_ref(),
next.released_state_anchor.as_ref(),
)
}
fn validate_release_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
@ -1085,6 +1103,15 @@ impl PreparedLowerOwnership {
{
return Err(RecordTransitionError::EvidenceChanged);
}
let release_evidence_valid = match self.lease_state {
Some(ParticipantState::ReleaseIntent) => {
self.released_state_anchor == next.released_state_anchor
}
_ => next.released_state_anchor.is_some() || self.released_state_anchor.is_none(),
};
if !release_evidence_valid {
return Err(RecordTransitionError::EvidenceChanged);
}
match (self.lease_state, next.lease_state) {
(None, None)
| (Some(ParticipantState::ReleaseIntent), Some(ParticipantState::ReleaseIntent)) => {

1
crates/lazaret-core/tests/schema_records.rs
View file

@ -366,6 +366,7 @@ fn prepared_ownership() -> PreparedLowerOwnership {
path: Some(HostPath::new(vec![b'/', 0xff])),
device: Some(DeviceId::new(27)),
inode: Some(InodeId::new(28)),
released_state_anchor: None,
}
}

45
crates/lazaret-core/tests/schema_records/participant_transitions.rs
View file

@ -514,6 +514,51 @@ fn prepared_lower_release_changes_only_the_lease_phase() {
.unwrap();
assert_eq!(active.validate_transition_to(&release), Ok(()));
let ParticipantOwnership::PreparedLower(mut with_anchor_evidence) =
release_ownership(ParticipantRole::PreparedLower)
else {
unreachable!()
};
with_anchor_evidence.released_state_anchor = Some(StateAnchorReleaseEvidence {
daemon: process(120),
user_namespace: namespace(),
mount_namespace: NamespaceIdentity {
device: DeviceId::new(121),
inode: InodeId::new(122),
},
mount: mount(),
});
let release_with_anchor_evidence = Participant::snapshot(
generation_id(),
ParticipantRole::PreparedLower,
ParticipantState::ReleaseIntent,
Some(ParticipantOwnership::PreparedLower(
with_anchor_evidence.clone(),
)),
)
.unwrap();
assert_eq!(
active.validate_transition_to(&release_with_anchor_evidence),
Ok(())
);
with_anchor_evidence
.released_state_anchor
.as_mut()
.unwrap()
.mount
.mount_id = MountId::new(123);
let changed_release_evidence = Participant::snapshot(
generation_id(),
ParticipantRole::PreparedLower,
ParticipantState::ReleaseIntent,
Some(ParticipantOwnership::PreparedLower(with_anchor_evidence)),
)
.unwrap();
assert_eq!(
release_with_anchor_evidence.validate_transition_to(&changed_release_evidence),
Err(RecordTransitionError::EvidenceChanged)
);
let ParticipantOwnership::PreparedLower(mut changed) =
release_ownership(ParticipantRole::PreparedLower)
else {

1
crates/lazaret-host-coordinator/src/lib.rs
View file

@ -9,6 +9,7 @@ pub mod lifecycle;
pub mod network;
pub mod production;
pub mod production_selector;
pub mod recovery;
pub mod runtime;
pub mod sandbox_init;
pub mod state_anchor;

7
crates/lazaret-host-coordinator/src/main.rs
View file

@ -18,6 +18,13 @@ fn main() {
}
return;
}
if let Some(result) = lazaret_state_anchor::dispatch_private_state_anchor_recovery(&arguments) {
if let Err(error) = result {
eprintln!("State Anchor recovery failed: {error}");
std::process::exit(1);
}
return;
}
if let Some(result) = lazaret_linux::process::dispatch_private_process_role(&arguments) {
if let Err(error) = result {
eprintln!("private process role failed: {error}");

297
crates/lazaret-host-coordinator/src/recovery.rs Normal file
View file

@ -0,0 +1,297 @@
use lazaret_core::identity::{
GenerationId, MountIdentity, NamespaceIdentity, PidfsProcessIdentityV1,
};
use lazaret_core::records::{
Participant, ParticipantOwnership, ParticipantRole, PersistedHostProcessOwnership,
};
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use crate::state_anchor::RuntimeOwnersAbsent;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct RuntimeRecoveryOwnership {
pub host_coordinator: Option<PidfsProcessIdentityV1>,
pub runtime_supervisor: Option<PidfsProcessIdentityV1>,
pub attached_runsc: Option<PidfsProcessIdentityV1>,
pub connectivity_helper: Option<PidfsProcessIdentityV1>,
pub state_anchor_daemon: Option<PidfsProcessIdentityV1>,
pub git_routing_daemon: Option<PidfsProcessIdentityV1>,
pub state_anchor_user_namespace: Option<NamespaceIdentity>,
pub state_anchor_mount_namespace: Option<NamespaceIdentity>,
pub state_anchor_mount: Option<MountIdentity>,
}
/// Extracts one generation's exact process and private-mount recovery authority.
///
/// # Errors
/// Refuses unequal copies of the constitutive HC or RS identities, and incomplete State Anchor
/// namespace/mount evidence. Staged identities remain cleanup authority.
pub fn runtime_recovery_ownership(
participants: &[Participant],
generation_id: GenerationId,
) -> Result<RuntimeRecoveryOwnership, &'static str> {
let mut result = RuntimeRecoveryOwnership::default();
for participant in participants
.iter()
.filter(|participant| participant.generation_id() == generation_id)
{
let Some(ownership) = participant.ownership() else {
continue;
};
match ownership {
ParticipantOwnership::Sandbox(sandbox) => {
merge_process(
&mut result.host_coordinator,
&sandbox.host_coordinator,
"Sandbox HC identity mismatch",
)?;
merge_process(
&mut result.runtime_supervisor,
&sandbox.runtime_supervisor,
"Sandbox RS identity mismatch",
)?;
merge_process(
&mut result.attached_runsc,
&sandbox.attached_runsc,
"attached runsc identity mismatch",
)?;
}
ParticipantOwnership::Network(network) => merge_process(
&mut result.runtime_supervisor,
&network.runtime_supervisor,
"Network RS identity mismatch",
)?,
ParticipantOwnership::Helper(helper) => {
merge_process(
&mut result.connectivity_helper,
&helper.connectivity_helper,
"connectivity-helper identity mismatch",
)?;
merge_process(
&mut result.runtime_supervisor,
&helper.runtime_supervisor,
"Helper RS identity mismatch",
)?;
}
ParticipantOwnership::StateAnchor(anchor) => {
merge_process(
&mut result.state_anchor_daemon,
&anchor.daemon,
"State Anchor daemon identity mismatch",
)?;
let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
if evidence == (None, None, None) {
continue;
}
let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
return Err("State Anchor recovery evidence is incomplete");
};
merge_value(
&mut result.state_anchor_user_namespace,
user,
"State Anchor user namespace mismatch",
)?;
merge_value(
&mut result.state_anchor_mount_namespace,
mount_namespace,
"State Anchor mount namespace mismatch",
)?;
merge_value(
&mut result.state_anchor_mount,
mount,
"State Anchor mount identity mismatch",
)?;
}
ParticipantOwnership::GitRouting(routing) => merge_process(
&mut result.git_routing_daemon,
&routing.daemon,
"Git-routing daemon identity mismatch",
)?,
ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
}
}
Ok(result)
}
fn merge_process(
target: &mut Option<PidfsProcessIdentityV1>,
observed: &PersistedHostProcessOwnership,
mismatch: &'static str,
) -> Result<(), &'static str> {
let Ok(observed) = observed.authoritative_pidfs() else {
return Ok(());
};
merge_value(target, *observed, mismatch)
}
fn merge_value<T: Copy + Eq>(
target: &mut Option<T>,
observed: T,
mismatch: &'static str,
) -> Result<(), &'static str> {
match target {
None => *target = Some(observed),
Some(current) if *current == observed => {}
Some(_) => return Err(mismatch),
}
Ok(())
}
#[must_use]
pub fn recovery_role_order() -> [ParticipantRole; 7] {
[
ParticipantRole::Publication,
ParticipantRole::Sandbox,
ParticipantRole::Helper,
ParticipantRole::Network,
ParticipantRole::GitRouting,
ParticipantRole::StateAnchor,
ParticipantRole::PreparedLower,
]
}
/// Proves the constitutive RS and attached runsc are absent before private-mount reconciliation.
///
/// # Errors
/// Refuses a live or stopped exact authority and every unreadable or ambiguous observation.
pub fn prove_runtime_owners_absent(
generation_id: GenerationId,
ownership: &RuntimeRecoveryOwnership,
) -> Result<RuntimeOwnersAbsent, std::io::Error> {
for identity in [ownership.attached_runsc, ownership.runtime_supervisor]
.into_iter()
.flatten()
{
match match_recovery_process(identity) {
Ok(PidfsRecoveryMatch::BootMismatch) => {}
Ok(PidfsRecoveryMatch::Matched(process)) if process.has_exited()? => {}
Ok(PidfsRecoveryMatch::Matched(_)) => {
return Err(std::io::Error::other(
"constitutive runtime authority remains present",
));
}
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {}
Err(error) => return Err(error),
}
}
Ok(RuntimeOwnersAbsent::for_generation(generation_id))
}
#[cfg(test)]
mod tests {
use lazaret_core::identity::{
BootId, BundleDigest, ContainerId, DeviceId, HostPath, HostPidLocator, MountId,
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks,
};
use lazaret_core::records::{
HelperOwnership, ParticipantState, SandboxOwnership, StateAnchorOwnership,
};
use super::*;
fn process(value: u64) -> PidfsProcessIdentityV1 {
PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new([1; 16]),
pidfs_st_dev: PidfsDeviceId::new(value),
pidfs_st_ino: PidfsInodeId::new(value),
},
HostPidLocator::new(u32::try_from(value).unwrap()),
ProcessStartTimeTicks::new(value),
)
}
fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant {
Participant::snapshot(
GenerationId::new(1),
role,
ParticipantState::Intent,
Some(ownership),
)
.unwrap()
}
#[test]
fn inventory_accepts_staged_cleanup_authority_and_rejects_cross_role_rs_mismatch() {
let rs = process(2);
let sandbox = participant(
ParticipantRole::Sandbox,
ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new([3; 32])),
runtime_root: Some(HostPath::new(b"/runtime".to_vec())),
bundle_digest: Some(BundleDigest::new([4; 32])),
host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
runtime_supervisor: PersistedHostProcessOwnership::staged(rs),
attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
}),
);
let helper = |runtime_supervisor| {
participant(
ParticipantRole::Helper,
ParticipantOwnership::Helper(HelperOwnership {
network_sidecar_id: None,
connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
runtime_supervisor: PersistedHostProcessOwnership::committed(
runtime_supervisor,
),
}),
)
};
let inventory =
runtime_recovery_ownership(&[sandbox.clone(), helper(rs)], GenerationId::new(1))
.unwrap();
assert_eq!(inventory.runtime_supervisor, Some(rs));
assert_eq!(inventory.attached_runsc, Some(process(3)));
assert_eq!(inventory.connectivity_helper, Some(process(4)));
assert!(
runtime_recovery_ownership(&[sandbox, helper(process(99))], GenerationId::new(1),)
.is_err()
);
}
#[test]
fn inventory_requires_complete_private_mount_evidence() {
let incomplete = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: Some(HostPath::new(b"upper".to_vec())),
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: Some(NamespaceIdentity {
device: DeviceId::new(6),
inode: lazaret_core::identity::InodeId::new(7),
}),
mount_namespace: None,
mount: None,
}),
);
assert!(runtime_recovery_ownership(&[incomplete], GenerationId::new(1)).is_err());
let complete = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: Some(HostPath::new(b"upper".to_vec())),
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: Some(NamespaceIdentity {
device: DeviceId::new(6),
inode: lazaret_core::identity::InodeId::new(7),
}),
mount_namespace: Some(NamespaceIdentity {
device: DeviceId::new(8),
inode: lazaret_core::identity::InodeId::new(9),
}),
mount: Some(MountIdentity {
mount_id: MountId::new(10),
device: DeviceId::new(11),
inode: lazaret_core::identity::InodeId::new(12),
}),
}),
);
assert!(runtime_recovery_ownership(&[complete], GenerationId::new(1)).is_ok());
}
}

147
crates/lazaret-host-coordinator/src/state_anchor.rs
View file

@ -10,8 +10,11 @@ use lazaret_core::records::{
ArtifactLease, HostProcessOwnershipSlot, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PersistedHostProcessOwnership, PreparedArtifact, PreparedLowerOwnership,
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
StateAnchorReleaseEvidence,
};
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness};
use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_mount};
use lazaret_storage_catalog::{Catalog, CatalogError};
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
@ -103,6 +106,17 @@ impl StateAnchorReleaseAuthority {
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct RuntimeOwnersAbsent {
generation_id: GenerationId,
}
impl RuntimeOwnersAbsent {
pub(crate) const fn for_generation(generation_id: GenerationId) -> Self {
Self { generation_id }
}
}
pub struct StateAnchorCoordinator<'a> {
domain_store: &'a HostCoordinatorDomainStore,
catalog: &'a Catalog,
@ -140,6 +154,7 @@ impl<'a> StateAnchorCoordinator<'a> {
path: Some(artifact.path.clone()),
device: Some(artifact.device),
inode: Some(artifact.inode),
released_state_anchor: None,
};
let expected_prepared_intent = participant(
generation_id,
@ -364,6 +379,52 @@ impl<'a> StateAnchorCoordinator<'a> {
Ok(authority)
}
/// Recovers the exact private State Anchor after all constitutive runtime owners are absent.
///
/// # Errors
/// Refuses generation drift, pidfs/namespace/mount mismatch, unreadable absence, or helper
/// failure. Boot mismatch and `ESRCH` reconcile physical absence without process action only
/// under the caller-provided ordered runtime-owner absence witness.
pub fn recover_after_runtime_owner_loss(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
owners_absent: RuntimeOwnersAbsent,
) -> Result<
(StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
StateAnchorCoordinatorError,
> {
if owners_absent.generation_id != generation_id {
return Err(StateAnchorCoordinatorError::InvalidState(
"runtime-owner absence belongs to another generation",
));
}
let authority = self.begin_release(generation_id, layout)?;
let observed = match match_recovery_process(authority.daemon) {
Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
layout,
&holder,
authority.user_namespace,
authority.mount_namespace,
authority.mount,
)?,
Ok(PidfsRecoveryMatch::BootMismatch) => observe_mount_absence(layout, authority.mount)?,
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
observe_mount_absence(layout, authority.mount)?
}
Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
};
Ok((
authority.clone(),
StateAnchorAbsenceObservation {
user_namespace: authority.user_namespace,
mount_namespace: authority.mount_namespace,
mount: authority.mount,
observed,
},
))
}
/// Completes Domain Clean, exact catalog deletion, and prepared-lower Clean after unmount.
///
/// # Errors
@ -404,7 +465,15 @@ impl<'a> StateAnchorCoordinator<'a> {
authority.generation_id,
ParticipantRole::StateAnchor,
)?),
DomainStoreMutation::PutParticipant(prepared_release_intent(&prepared)?),
DomainStoreMutation::PutParticipant(prepared_release_intent(
&prepared,
StateAnchorReleaseEvidence {
daemon: authority.daemon,
user_namespace: authority.user_namespace,
mount_namespace: authority.mount_namespace,
mount: authority.mount,
},
)?),
])?;
}
(ParticipantState::Clean, ParticipantState::ReleaseIntent) => {}
@ -422,6 +491,80 @@ impl<'a> StateAnchorCoordinator<'a> {
Ok(())
}
/// Resumes lease release after the atomic State Anchor Clean checkpoint was committed.
///
/// # Errors
/// Refuses missing retained release evidence, generation or layout drift, a mismatched lease,
/// or failure of the fresh catalog-lock absence observation.
pub fn resume_release_after_checkpoint<Observe>(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
mut observe_again: Observe,
) -> Result<(), StateAnchorCoordinatorError>
where
Observe: FnMut() -> Result<StateAnchorAbsenceObservation, StateAnchorCoordinatorError>,
{
let snapshot = self.domain_store.snapshot()?;
let anchor = role(
&snapshot.participants,
generation_id,
ParticipantRole::StateAnchor,
)?;
let prepared = role(
&snapshot.participants,
generation_id,
ParticipantRole::PreparedLower,
)?;
if anchor.state() != ParticipantState::Clean
|| prepared.state() != ParticipantState::ReleaseIntent
{
return Err(StateAnchorCoordinatorError::InvalidState(
"durable release checkpoint is absent",
));
}
let Some(ParticipantOwnership::PreparedLower(ownership)) = prepared.ownership() else {
return Err(StateAnchorCoordinatorError::InvalidState(
"prepared-lower ownership is invalid",
));
};
let evidence =
ownership
.released_state_anchor
.ok_or(StateAnchorCoordinatorError::InvalidState(
"prepared-lower release has no exact State Anchor evidence",
))?;
let artifact =
ownership
.prepared_artifact_id
.ok_or(StateAnchorCoordinatorError::InvalidState(
"prepared-lower artifact is absent",
))?;
let expected_layout = layout.ownership_intent(artifact);
let merged = expected_layout.merged_path.as_ref().ok_or(
StateAnchorCoordinatorError::InvalidState("State Anchor merged path is absent"),
)?;
let authority = StateAnchorReleaseAuthority {
generation_id,
lease: ArtifactLease {
prepared_artifact_id: artifact,
domain_id: snapshot.domain.domain_id,
generation_id,
},
merged_label: path_from_record(merged).to_path_buf(),
mount: evidence.mount,
daemon: evidence.daemon,
user_namespace: evidence.user_namespace,
mount_namespace: evidence.mount_namespace,
};
self.release_catalog_lease_if_present(&authority, &mut observe_again)?;
self.domain_store
.apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(
clean_participant(generation_id, ParticipantRole::PreparedLower)?,
)])?;
Ok(())
}
fn require_staged_identity(
&self,
staged: StagedProcessOwnership,
@ -754,10 +897,12 @@ fn clean_participant(
fn prepared_release_intent(
prepared: &Participant,
released_state_anchor: StateAnchorReleaseEvidence,
) -> Result<Participant, StateAnchorCoordinatorError> {
let ownership = match prepared.ownership() {
Some(ParticipantOwnership::PreparedLower(ownership)) => PreparedLowerOwnership {
lease_state: Some(ParticipantState::ReleaseIntent),
released_state_anchor: Some(released_state_anchor),
..ownership.clone()
},
_ => {

22
crates/lazaret-host-coordinator/src/state_anchor/tests.rs
View file

@ -232,20 +232,31 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
DomainStoreMutation::CleanParticipantAfterAbsence(
clean_participant(generation_id, ParticipantRole::StateAnchor).unwrap(),
),
DomainStoreMutation::PutParticipant(prepared_release_intent(prepared).unwrap()),
DomainStoreMutation::PutParticipant(
prepared_release_intent(
prepared,
StateAnchorReleaseEvidence {
daemon: authority.daemon(),
user_namespace: authority.user_namespace(),
mount_namespace: authority.mount_namespace(),
mount: authority.mount(),
},
)
.unwrap(),
),
])
.unwrap();
for invalid in invalid_observations {
assert!(
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || Ok(invalid))
.resume_release_after_checkpoint(generation_id, &layout, || Ok(invalid))
.is_err()
);
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
}
assert!(
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || {
.resume_release_after_checkpoint(generation_id, &layout, || {
Err(StateAnchorCoordinatorError::InvalidState(
"State Anchor absence evidence is unreadable",
))
@ -255,7 +266,7 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
assert!(
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || {
.resume_release_after_checkpoint(generation_id, &layout, || {
Err(StateAnchorCoordinatorError::InvalidState(
"second State Anchor observation was skipped",
))
@ -265,7 +276,7 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
let mut observations = 0;
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || {
.resume_release_after_checkpoint(generation_id, &layout, || {
observations += 1;
Ok(valid)
})
@ -552,6 +563,7 @@ impl AcquisitionFixture {
path: Some(self.artifact.path.clone()),
device: Some(self.artifact.device),
inode: Some(self.artifact.inode),
released_state_anchor: None,
}),
)
.unwrap()

102
crates/lazaret-linux/src/ipc.rs
View file

@ -1,12 +1,16 @@
use std::io;
use std::io::{self, IoSlice, IoSliceMut};
use std::mem::MaybeUninit;
use std::os::fd::{AsFd, OwnedFd};
use lazaret_core::codec::{decode, encode};
use rustix::net::{
AddressFamily, RecvFlags, SendFlags, SocketFlags, SocketType, recv, send, socketpair,
AddressFamily, RecvAncillaryBuffer, RecvAncillaryMessage, RecvFlags, ReturnFlags,
SendAncillaryBuffer, SendAncillaryMessage, SendFlags, SocketFlags, SocketType, recv, recvmsg,
send, sendmsg, socketpair,
};
const MAX_PRIVATE_PACKET: usize = 64 * 1024;
const MAX_TRANSFERRED_DESCRIPTORS: usize = 8;
/// # Errors
/// Returns an error when the private sequenced-packet pair cannot be created.
@ -61,13 +65,85 @@ where
decode(&encoded[..received]).map_err(|error| io::Error::other(error.to_string()))
}
/// Sends one nonempty bounded descriptor set with a one-byte packet.
///
/// # Errors
/// Refuses empty, oversized, incomplete, or failed descriptor transfer.
pub fn send_descriptors(
socket: impl AsFd,
descriptors: &[std::os::fd::BorrowedFd<'_>],
) -> io::Result<()> {
if descriptors.is_empty() || descriptors.len() > MAX_TRANSFERRED_DESCRIPTORS {
return Err(io::Error::other("private descriptor set has invalid size"));
}
let mut space =
[MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(MAX_TRANSFERRED_DESCRIPTORS))];
let mut control = SendAncillaryBuffer::new(&mut space);
if !control.push(SendAncillaryMessage::ScmRights(descriptors)) {
return Err(io::Error::other(
"private descriptor control buffer is too small",
));
}
let bytes = [1_u8];
let sent = sendmsg(
socket,
&[IoSlice::new(&bytes)],
&mut control,
SendFlags::empty(),
)?;
if sent == bytes.len() {
Ok(())
} else {
Err(io::Error::other("private descriptor packet was incomplete"))
}
}
/// Receives exactly `expected` descriptors with close-on-exec set atomically.
///
/// # Errors
/// Refuses missing, extra, truncated, malformed, or failed descriptor transfer.
pub fn receive_descriptors(socket: impl AsFd, expected: usize) -> io::Result<Vec<OwnedFd>> {
if expected == 0 || expected > MAX_TRANSFERRED_DESCRIPTORS {
return Err(io::Error::other("expected descriptor count is invalid"));
}
let mut byte = [0_u8];
let mut slices = [IoSliceMut::new(&mut byte)];
let mut space =
[MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(MAX_TRANSFERRED_DESCRIPTORS))];
let mut control = RecvAncillaryBuffer::new(&mut space);
let received = recvmsg(socket, &mut slices, &mut control, RecvFlags::CMSG_CLOEXEC)?;
if received.bytes != 1
|| byte != [1]
|| received
.flags
.intersects(ReturnFlags::TRUNC | ReturnFlags::CTRUNC)
{
return Err(io::Error::other("private descriptor packet is malformed"));
}
let mut descriptors = Vec::new();
for message in control.drain() {
match message {
RecvAncillaryMessage::ScmRights(rights) => descriptors.extend(rights),
_ => return Err(io::Error::other("private ancillary message is unexpected")),
}
}
if descriptors.len() == expected {
Ok(descriptors)
} else {
Err(io::Error::other(
"private descriptor count mismatches request",
))
}
}
#[cfg(test)]
mod tests {
use std::os::fd::AsFd;
use std::fs::File;
use std::os::fd::{AsFd, AsRawFd};
use rustix::net::{RecvFlags, SendFlags, recv, send};
use super::private_seqpacket_pair;
use super::{private_seqpacket_pair, receive_descriptors, send_descriptors};
#[test]
fn preserves_message_boundaries() {
@ -80,4 +156,22 @@ mod tests {
let (_, second) = recv(right.as_fd(), &mut buffer, RecvFlags::empty()).unwrap();
assert_eq!(&buffer[..second], b"second");
}
#[test]
fn transfers_exact_close_on_exec_descriptors() {
let (left, right) = private_seqpacket_pair().unwrap();
let first = File::open("/dev/null").unwrap();
let second = File::open("/dev/zero").unwrap();
send_descriptors(left, &[first.as_fd(), second.as_fd()]).unwrap();
let received = receive_descriptors(right, 2).unwrap();
assert_ne!(received[0].as_raw_fd(), first.as_raw_fd());
assert_ne!(received[1].as_raw_fd(), second.as_raw_fd());
for descriptor in received {
assert!(
rustix::io::fcntl_getfd(&descriptor)
.unwrap()
.contains(rustix::io::FdFlags::CLOEXEC)
);
}
}
}

237
crates/lazaret-linux/src/process.rs
View file

@ -223,6 +223,18 @@ pub struct RetainedProcess {
identity: PidfsProcessIdentityV1,
}
#[derive(Debug)]
pub enum PidfsRecoveryMatch {
BootMismatch,
Matched(RetainedProcess),
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum RecoveryActionResult<T> {
BootReconciled(T),
ProcessAction(T),
}
#[derive(Debug)]
pub struct CapturedProcess {
process: RetainedProcess,
@ -412,6 +424,27 @@ impl ChildBroker {
/// Returns an error when causal launch, exact waiting, or the command exit status fails.
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
pub fn run_transient(&self, target: &TargetCommand) -> io::Result<ProcessExit> {
self.run_transient_with_stdout(target, None)
}
/// Runs a non-durable direct child with one private inherited stdout descriptor.
///
/// # Errors
/// Returns an error when causal launch, exact waiting, or the command exit status fails.
pub fn run_transient_with_private_stdout(
&self,
target: &TargetCommand,
stdout: OwnedFd,
) -> io::Result<ProcessExit> {
self.run_transient_with_stdout(target, Some(stdout))
}
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
fn run_transient_with_stdout(
&self,
target: &TargetCommand,
stdout: Option<OwnedFd>,
) -> io::Result<ProcessExit> {
let _guard = self
.exclusion
.lock()
@ -426,7 +459,7 @@ impl ChildBroker {
target.append_to(&mut command);
command
.stdin(wait.into_stdio())
.stdout(Stdio::null())
.stdout(stdout.map_or_else(Stdio::null, Stdio::from))
.stderr(Stdio::null());
let mut child = command.spawn()?;
let process = match acquire_spawned_child(child.id()) {
@ -911,41 +944,87 @@ fn fail_after_pidfd_closed<T>(process: RetainedProcess, error: io::Error) -> io:
/// Refuses before PID lookup on boot mismatch and before action on every later mismatch.
#[allow(clippy::disallowed_methods)]
pub fn recover_process(identity: PidfsProcessIdentityV1) -> io::Result<RetainedProcess> {
recover_process_with(identity, read_boot_id, |host_pid| {
let raw_pid = i32::try_from(host_pid)
.map_err(|_| io::Error::other("stored PID locator is out of range"))?;
let pid = Pid::from_raw(raw_pid)
.ok_or_else(|| io::Error::other("stored PID locator is invalid"))?;
require_host_pid_namespace(host_pid)?;
rustix::process::pidfd_open(pid, PidfdFlags::empty()).map_err(Into::into)
})
match match_recovery_process(identity)? {
PidfsRecoveryMatch::BootMismatch => Err(io::Error::other(
"stored process authority belongs to another boot",
)),
PidfsRecoveryMatch::Matched(process) => Ok(process),
}
}
fn recover_process_with<Boot, Open>(
/// Reopens one stored process identity or reports boot mismatch before PID lookup.
///
/// # Errors
/// Refuses host-PID ambiguity, PID lookup, pidfs validation, identity, and negative-evidence
/// failures without returning an actionable descriptor.
#[allow(clippy::disallowed_methods)]
pub fn match_recovery_process(identity: PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch> {
match_recovery_process_with(
identity,
read_boot_id,
|host_pid| {
let raw_pid = i32::try_from(host_pid)
.map_err(|_| io::Error::other("stored PID locator is out of range"))?;
let pid = Pid::from_raw(raw_pid)
.ok_or_else(|| io::Error::other("stored PID locator is invalid"))?;
require_host_pid_namespace(host_pid)?;
rustix::process::pidfd_open(pid, PidfdFlags::empty()).map_err(Into::into)
},
|pidfd, boot_id| authority_from_pidfd(pidfd, boot_id),
read_start_time,
)
}
/// Reconciles reboot-safe physical evidence or acts through the exactly reopened pidfd.
///
/// # Errors
/// Propagates reconciliation, pidfs matching, or same-descriptor action failure. The process
/// action is never called on boot mismatch or an unequal process identity.
pub fn recover_process_action<T>(
identity: PidfsProcessIdentityV1,
reconcile_boot: impl FnOnce() -> io::Result<T>,
action: impl FnOnce(&RetainedProcess) -> io::Result<T>,
) -> io::Result<RecoveryActionResult<T>> {
match match_recovery_process(identity)? {
PidfsRecoveryMatch::BootMismatch => {
reconcile_boot().map(RecoveryActionResult::BootReconciled)
}
PidfsRecoveryMatch::Matched(process) => {
action(&process).map(RecoveryActionResult::ProcessAction)
}
}
}
fn match_recovery_process_with<Boot, Open, Inspect, Start>(
identity: PidfsProcessIdentityV1,
boot: Boot,
open: Open,
) -> io::Result<RetainedProcess>
inspect: Inspect,
start: Start,
) -> io::Result<PidfsRecoveryMatch>
where
Boot: FnOnce() -> io::Result<BootId>,
Open: FnOnce(u32) -> io::Result<OwnedFd>,
Inspect: FnOnce(BorrowedFd<'_>, BootId) -> io::Result<PidfsProcessAuthorityV1>,
Start: FnOnce(u32) -> io::Result<u64>,
{
let current_boot = boot()?;
if current_boot != identity.authority().boot_id {
return Err(io::Error::other(
"stored process authority belongs to another boot",
));
return Ok(PidfsRecoveryMatch::BootMismatch);
}
let host_pid = identity.host_pid().get();
let pidfd = open(host_pid)?;
let observed = authority_from_pidfd(pidfd.as_fd(), current_boot)?;
let observed = inspect(pidfd.as_fd(), current_boot)?;
if observed != identity.authority() {
return Err(io::Error::other("pidfs process authority mismatch"));
}
if read_start_time(host_pid)? != identity.start_time_ticks().get() {
if start(host_pid)? != identity.start_time_ticks().get() {
return Err(io::Error::other("process start time mismatch"));
}
Ok(RetainedProcess { pidfd, identity })
Ok(PidfsRecoveryMatch::Matched(RetainedProcess {
pidfd,
identity,
}))
}
/// # Errors
@ -1491,18 +1570,138 @@ mod tests {
ProcessStartTimeTicks::new(1),
);
let opened = Cell::new(false);
let result = recover_process_with(
let result = match_recovery_process_with(
identity,
|| Ok(BootId::new([2; 16])),
|_| {
opened.set(true);
unreachable!()
},
|_, _| unreachable!(),
|_| unreachable!(),
);
assert!(result.is_err());
assert!(matches!(result, Ok(PidfsRecoveryMatch::BootMismatch)));
assert!(!opened.get());
}
#[test]
fn recovery_refuses_each_post_open_mismatch_without_returning_authority() {
let authority = PidfsProcessAuthorityV1 {
boot_id: BootId::new([1; 16]),
pidfs_st_dev: PidfsDeviceId::new(7),
pidfs_st_ino: PidfsInodeId::new(9),
};
let identity = PidfsProcessIdentityV1::new(
authority,
HostPidLocator::new(41),
ProcessStartTimeTicks::new(43),
);
for (observed, start) in [
(
PidfsProcessAuthorityV1 {
pidfs_st_dev: PidfsDeviceId::new(8),
..authority
},
43,
),
(
PidfsProcessAuthorityV1 {
pidfs_st_ino: PidfsInodeId::new(10),
..authority
},
43,
),
(authority, 44),
] {
let events = Rc::new(std::cell::RefCell::new(Vec::new()));
let open_events = Rc::clone(&events);
let inspect_events = Rc::clone(&events);
let start_events = Rc::clone(&events);
let result = match_recovery_process_with(
identity,
|| Ok(authority.boot_id),
move |pid| {
open_events.borrow_mut().push(("open", u64::from(pid)));
Ok(File::open("/dev/null").unwrap().into())
},
move |fd, _| {
inspect_events
.borrow_mut()
.push(("inspect", u64::try_from(fd.as_raw_fd()).unwrap()));
Ok(observed)
},
move |pid| {
start_events.borrow_mut().push(("start", u64::from(pid)));
Ok(start)
},
);
assert!(result.is_err());
assert_eq!(events.borrow()[0], ("open", 41));
assert_eq!(events.borrow()[1].0, "inspect");
if observed == authority {
assert_eq!(events.borrow()[2], ("start", 41));
} else {
assert_eq!(events.borrow().len(), 2);
}
}
for failure in ["open", "inspect", "start"] {
let result = match_recovery_process_with(
identity,
|| Ok(authority.boot_id),
|_| {
if failure == "open" {
Err(io::Error::from_raw_os_error(3))
} else {
Ok(File::open("/dev/null").unwrap().into())
}
},
|_, _| {
if failure == "inspect" {
Err(io::Error::other("injected descriptor inspection error"))
} else {
Ok(authority)
}
},
|_| {
if failure == "start" {
Err(io::Error::other("injected start observation error"))
} else {
Ok(43)
}
},
);
assert!(result.is_err(), "accepted injected {failure} failure");
}
}
#[test]
fn recovery_action_reconciles_boot_without_process_action() {
let mut old_boot = *read_boot_id().unwrap().as_bytes();
old_boot[0] ^= u8::MAX;
let identity = PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new(old_boot),
pidfs_st_dev: PidfsDeviceId::new(1),
pidfs_st_ino: PidfsInodeId::new(1),
},
HostPidLocator::new(u32::MAX),
ProcessStartTimeTicks::new(u64::MAX),
);
let action_called = Cell::new(false);
let result = recover_process_action(
identity,
|| Ok("reconciled"),
|_| {
action_called.set(true);
Ok("acted")
},
)
.unwrap();
assert_eq!(result, RecoveryActionResult::BootReconciled("reconciled"));
assert!(!action_called.get());
}
#[test]
fn pidfs_magic_precedes_same_descriptor_stat_and_errors_refuse() {
let file = File::open("/dev/null").unwrap();

4
crates/lazaret-prepared-lower/src/lib.rs
View file

@ -7,5 +7,7 @@ mod transport;
pub use layer::{
IdMap, IdMapRange, PreparationLimits, PreparedDestination, PreparedDestinationKind,
};
pub use store::{PrepareRecovery, PrepareRequest, PreparedLowerStore, ReclaimOutcome};
pub use store::{
PrepareRecovery, PrepareRequest, PreparedArtifactGcResult, PreparedLowerStore, ReclaimOutcome,
};
pub use transport::{OciRegistryTransport, Platform, RegistryCredentials, TransportedImage};

55
crates/lazaret-prepared-lower/src/store.rs
View file

@ -55,6 +55,12 @@ pub enum ReclaimOutcome {
ReplacementPreserved,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PreparedArtifactGcResult {
pub prepared_artifact_id: PreparedArtifactId,
pub outcome: ReclaimOutcome,
}
#[derive(Clone, Debug)]
pub struct PreparedLowerStore {
root: Arc<File>,
@ -270,6 +276,55 @@ impl PreparedLowerStore {
}
}
/// Recovers the complete catalog snapshot of interrupted preparation operations.
///
/// # Errors
/// Stops at the first foreign, mismatched, unreadable, or changed operation without treating
/// it as owned absence.
pub fn recover_preparations(&self) -> Result<Vec<(PrepareOperationId, PrepareRecovery)>> {
let operations = self.catalog.snapshot()?.prepare_operations;
operations
.iter()
.map(|operation| {
self.recover_prepare(operation)
.map(|outcome| (operation.operation_id, outcome))
})
.collect()
}
/// Reclaims only the caller-selected exact catalog artifacts.
///
/// # Errors
/// Refuses an unknown target, duplicate target, any lease, catalog drift, or physical
/// ownership mismatch. Reclaiming rows resume their existing exact operation.
pub fn gc_artifacts(
&self,
selected: &[PreparedArtifactId],
) -> Result<Vec<PreparedArtifactGcResult>> {
let mut unique = std::collections::HashSet::new();
if selected.iter().any(|id| !unique.insert(*id)) {
bail!("prepared-artifact GC target is duplicated");
}
let snapshot = self.catalog.snapshot()?;
let mut results = Vec::with_capacity(selected.len());
for id in selected {
let artifact = snapshot
.prepared_artifacts
.iter()
.find(|artifact| artifact.prepared_artifact_id == *id)
.context("prepared-artifact GC target is absent")?;
let outcome = match artifact.state {
ArtifactState::Available => self.reclaim(artifact)?,
ArtifactState::Reclaiming => self.resume_reclaim(artifact)?,
};
results.push(PreparedArtifactGcResult {
prepared_artifact_id: *id,
outcome,
});
}
Ok(results)
}
/// Moves an exact zero-lease artifact to quarantine, deletes it, and closes reclaim intent.
///
/// # Errors

37
crates/lazaret-prepared-lower/tests/prepared_lower.rs
View file

@ -1135,6 +1135,43 @@ fn reclaim_refuses_forged_identity_and_leases() {
assert!(fixture.artifact_root(&artifact).exists());
}
#[test]
fn selected_gc_is_idempotent_only_at_the_durable_reclaim_boundary() {
let fixture = Fixture::new();
let artifact = fixture
.store
.prepare(&fixture.request(
71,
&[(
layer(&[Entry {
path: "payload",
kind: EntryType::Regular,
data: b"gc",
target: None,
mode: 0o644,
}]),
OCI_TAR,
)],
))
.unwrap();
let id = artifact.prepared_artifact_id;
assert!(fixture.store.gc_artifacts(&[id, id]).is_err());
let lease = lazaret_core::records::ArtifactLease {
prepared_artifact_id: id,
domain_id: lazaret_core::identity::DomainId::new([72; 32]),
generation_id: lazaret_core::identity::GenerationId::new(1),
};
fixture.catalog.acquire_lease(&lease).unwrap();
assert!(fixture.store.gc_artifacts(&[id]).is_err());
fixture.catalog.release_lease(&lease).unwrap();
let result = fixture.store.gc_artifacts(&[id]).unwrap();
assert_eq!(result[0].prepared_artifact_id, id);
assert_eq!(result[0].outcome, ReclaimOutcome::Removed);
assert!(fixture.store.gc_artifacts(&[id]).is_err());
}
#[test]
fn destination_layout_is_materialized_and_canonical_in_artifact_identity() {
let fixture = Fixture::new();

497
crates/lazaret-state-anchor/src/lib.rs
View file

@ -6,28 +6,35 @@ use std::fs::File;
use std::io::{self, Write};
use std::os::fd::{AsFd, AsRawFd, OwnedFd};
use std::os::unix::ffi::{OsStrExt, OsStringExt};
use std::path::{Path, PathBuf};
use std::path::{Component, Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::thread;
use std::time::{Duration, Instant};
use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration};
use lazaret_core::identity::{HostPath, MountIdentity, PreparedArtifactId};
use lazaret_core::identity::{
DeviceId, HostPath, InodeId, MountIdentity, NamespaceIdentity, PreparedArtifactId,
};
use lazaret_core::records::{
HostProcessOwnershipSlot, PersistedHostProcessOwnership, PreparedArtifact,
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
};
use lazaret_linux::filesystem::mount_identity;
use lazaret_linux::filesystem::{mount_identity, namespace_identity};
use lazaret_linux::ipc::{
private_seqpacket_pair, receive_descriptors, receive_packet, send_descriptors, send_packet,
};
use lazaret_linux::process::{
ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker,
ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, require_pidfs_magic,
};
use rustix::fs::{
AtFlags, FileType, Mode, OFlags, ResolveFlags, mkdirat, openat2, renameat, unlinkat,
};
use rustix::io::dup;
use rustix::thread::{ThreadNameSpaceType, move_into_thread_name_spaces};
const MOUNT_TIMEOUT: Duration = Duration::from_secs(5);
const MOUNT_POLL_INTERVAL: Duration = Duration::from_millis(2);
const PRIVATE_RECOVERY_ROLE: &str = "--lazaret-private-state-anchor-recovery";
static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
pub const PROTECTED_RUNTIME_FILES: [&str; 3] = ["resolv.conf", "hosts", "hostname"];
@ -181,6 +188,12 @@ impl<'configuration> StateAnchorLayout<'configuration> {
&self.merged
}
/// Returns the ordinary host path used as the merged mount label.
#[must_use]
pub fn merged_label(&self) -> &Path {
&self.merged_label
}
#[must_use]
pub fn runtime_files(&self) -> &Path {
&self.runtime_files
@ -1007,6 +1020,259 @@ pub fn unmount_after_coordinator_loss(
Ok(())
}
/// Enters an exact live holder's private namespaces and unmounts only its recorded State Anchor.
///
/// # Errors
/// Refuses descriptor transfer, pidfs, namespace, path, mount, helper, or acknowledgment drift.
pub fn recover_private_state_anchor_mount(
layout: &StateAnchorLayout<'_>,
holder: &RetainedProcess,
user_namespace: NamespaceIdentity,
mount_namespace: NamespaceIdentity,
expected_mount: MountIdentity,
) -> Result<MountIdentity, StateAnchorError> {
let merged = &layout.merged_label;
let (coordinator, helper) = private_seqpacket_pair()?;
send_descriptors(
&coordinator,
&[holder.as_fd(), layout.runtime_anchor.as_fd()],
)?;
let authority = holder.identity().authority();
let target = TargetCommand::new("/proc/self/exe")
.argument(PRIVATE_RECOVERY_ROLE)
.argument(authority.pidfs_st_dev.get().to_string())
.argument(authority.pidfs_st_ino.get().to_string())
.argument(user_namespace.device.get().to_string())
.argument(user_namespace.inode.get().to_string())
.argument(mount_namespace.device.get().to_string())
.argument(mount_namespace.inode.get().to_string())
.argument(expected_mount.mount_id.get().to_string())
.argument(expected_mount.device.get().to_string())
.argument(expected_mount.inode.get().to_string())
.argument(merged.as_os_str());
child_broker().run_transient_with_private_stdout(&target, helper)?;
let observed: MountIdentity = receive_packet(coordinator)?;
if observed == expected_mount {
Err(StateAnchorError::OwnershipMismatch)
} else {
Ok(observed)
}
}
/// Dispatches the bounded private-namespace State Anchor recovery helper.
#[must_use]
pub fn dispatch_private_state_anchor_recovery(arguments: &[OsString]) -> Option<io::Result<()>> {
if arguments
.first()
.is_none_or(|arg| arg != PRIVATE_RECOVERY_ROLE)
{
return None;
}
Some(run_private_state_anchor_recovery(arguments))
}
fn run_private_state_anchor_recovery(arguments: &[OsString]) -> io::Result<()> {
if arguments.len() != 11 {
return Err(io::Error::other(
"private State Anchor recovery arguments are malformed",
));
}
let values = arguments[1..10]
.iter()
.map(|value| {
value
.to_str()
.ok_or_else(|| io::Error::other("private recovery integer is not UTF-8"))?
.parse::<u64>()
.map_err(|_| io::Error::other("private recovery integer is invalid"))
})
.collect::<io::Result<Vec<_>>>()?;
let expected_pidfs = (values[0], values[1]);
let expected_user = NamespaceIdentity {
device: DeviceId::new(values[2]),
inode: InodeId::new(values[3]),
};
let expected_mount_namespace = NamespaceIdentity {
device: DeviceId::new(values[4]),
inode: InodeId::new(values[5]),
};
let expected_mount = MountIdentity {
mount_id: lazaret_core::identity::MountId::new(values[6]),
device: DeviceId::new(values[7]),
inode: InodeId::new(values[8]),
};
let merged = PathBuf::from(&arguments[10]);
let stdout = io::stdout();
let mut descriptors = receive_descriptors(&stdout, 2)?;
let runtime_anchor = descriptors
.pop()
.ok_or_else(|| io::Error::other("runtime anchor descriptor is absent"))?;
let pidfd = descriptors
.pop()
.ok_or_else(|| io::Error::other("pidfd descriptor is absent"))?;
let mut operations = KernelPrivateRecovery {
pidfd,
runtime_anchor,
expected_pidfs,
merged,
reopened_parent: None,
};
let observed = execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
)?;
send_packet(stdout, &observed)
}
trait PrivateRecoveryOperations {
fn validate_pidfs(&mut self) -> io::Result<()>;
fn enter_user_namespace(&mut self) -> io::Result<()>;
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity>;
fn enter_mount_namespace(&mut self) -> io::Result<()>;
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity>;
fn verify_mount(&mut self) -> io::Result<MountIdentity>;
fn unmount(&mut self) -> io::Result<()>;
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity>;
}
fn execute_private_recovery(
operations: &mut impl PrivateRecoveryOperations,
expected_user: NamespaceIdentity,
expected_mount_namespace: NamespaceIdentity,
expected_mount: MountIdentity,
) -> io::Result<MountIdentity> {
operations
.validate_pidfs()
.map_err(|error| io::Error::other(format!("validate pidfs: {error}")))?;
operations
.enter_user_namespace()
.map_err(|error| io::Error::other(format!("enter user namespace: {error}")))?;
if operations
.observe_user_namespace()
.map_err(|error| io::Error::other(format!("observe user namespace: {error}")))?
!= expected_user
{
return Err(io::Error::other("private recovery user namespace mismatch"));
}
operations
.enter_mount_namespace()
.map_err(|error| io::Error::other(format!("enter mount namespace: {error}")))?;
if operations
.observe_mount_namespace()
.map_err(|error| io::Error::other(format!("observe mount namespace: {error}")))?
!= expected_mount_namespace
{
return Err(io::Error::other(
"private recovery mount namespace mismatch",
));
}
if operations
.verify_mount()
.map_err(|error| io::Error::other(format!("verify mount: {error}")))?
!= expected_mount
{
return Err(io::Error::other("private recovery mount identity mismatch"));
}
operations
.unmount()
.map_err(|error| io::Error::other(format!("unmount: {error}")))?;
let observed = operations
.observe_after_unmount()
.map_err(|error| io::Error::other(format!("observe after unmount: {error}")))?;
if observed == expected_mount {
Err(io::Error::other("private recovery mount remains present"))
} else {
Ok(observed)
}
}
struct KernelPrivateRecovery {
pidfd: OwnedFd,
runtime_anchor: OwnedFd,
expected_pidfs: (u64, u64),
merged: PathBuf,
reopened_parent: Option<OwnedFd>,
}
impl PrivateRecoveryOperations for KernelPrivateRecovery {
fn validate_pidfs(&mut self) -> io::Result<()> {
require_pidfs_magic(rustix::fs::fstatfs(&self.pidfd)?.f_type as i64)?;
let stat = rustix::fs::fstat(&self.pidfd)?;
if (stat.st_dev, stat.st_ino) == self.expected_pidfs {
Ok(())
} else {
Err(io::Error::other(
"private recovery pidfs authority mismatch",
))
}
}
fn enter_user_namespace(&mut self) -> io::Result<()> {
move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::USER)?;
Ok(())
}
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
namespace_identity(Path::new("/proc/self/ns/user"))
}
fn enter_mount_namespace(&mut self) -> io::Result<()> {
move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::MOUNT)?;
Ok(())
}
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
namespace_identity(Path::new("/proc/self/ns/mnt"))
}
fn verify_mount(&mut self) -> io::Result<MountIdentity> {
let parent_path = self
.merged
.parent()
.ok_or_else(|| io::Error::other("private recovery merged path has no parent"))?;
let reopened_parent = openat2(
rustix::fs::CWD,
parent_path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
)?;
let retained = rustix::fs::fstat(&self.runtime_anchor)?;
let reopened = rustix::fs::fstat(&reopened_parent)?;
if (retained.st_dev, retained.st_ino) != (reopened.st_dev, reopened.st_ino) {
return Err(io::Error::other("private recovery runtime anchor mismatch"));
}
let mounted = open_final_mount_directory(&reopened_parent, self.merged_name()?)?;
let identity = mount_identity(&mounted)?;
self.reopened_parent = Some(reopened_parent);
Ok(identity)
}
fn unmount(&mut self) -> io::Result<()> {
rustix::mount::unmount(&self.merged, rustix::mount::UnmountFlags::empty())?;
Ok(())
}
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
let parent = self
.reopened_parent
.as_ref()
.ok_or_else(|| io::Error::other("private recovery parent was not verified"))?;
mount_identity(&open_final_mount_directory(parent, self.merged_name()?)?)
}
}
impl KernelPrivateRecovery {
fn merged_name(&self) -> io::Result<&Path> {
self.merged
.file_name()
.map(Path::new)
.ok_or_else(|| io::Error::other("private recovery merged path has no name"))
}
}
/// Independently verifies that no mount is attached at the exact merged path.
///
/// # Errors
@ -1015,6 +1281,22 @@ pub fn require_mount_absent(layout: &StateAnchorLayout<'_>) -> Result<(), StateA
layout.require_unmounted()
}
/// Freshly observes absence of one exact retired mount in the caller's current namespace.
///
/// # Errors
/// Refuses unreadable layout evidence or continued presence of the exact mount.
pub fn observe_mount_absence(
layout: &StateAnchorLayout<'_>,
expected: MountIdentity,
) -> Result<MountIdentity, StateAnchorError> {
let observed = layout.current_merged_identity()?;
if observed == expected {
Err(StateAnchorError::OwnershipMismatch)
} else {
Ok(observed)
}
}
fn create_directory(parent: impl AsFd, name: &str) -> Result<(), StateAnchorError> {
match mkdirat(parent.as_fd(), name, Mode::from_raw_mode(0o700)) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
@ -1045,6 +1327,16 @@ fn open_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchor
Ok(descriptor)
}
fn open_final_mount_directory(parent: impl AsFd, path: &Path) -> io::Result<OwnedFd> {
let mut components = path.components();
if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() {
return Err(io::Error::other(
"final mount path is not one normal component",
));
}
open_mounted_directory(parent, path).map_err(io::Error::other)
}
fn open_mounted_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchorError> {
let descriptor = openat2(
parent.as_fd(),
@ -1331,3 +1623,200 @@ fn path_from_host(path: &HostPath) -> PathBuf {
fn host_path(path: &Path) -> HostPath {
HostPath::new(path.as_os_str().as_bytes().to_vec())
}
#[cfg(test)]
mod recovery_tests {
use super::*;
use std::os::unix::fs::symlink;
struct FailpointRecovery {
fail_at: Option<usize>,
boundary: usize,
events: Vec<&'static str>,
user: NamespaceIdentity,
mount_namespace: NamespaceIdentity,
mount: MountIdentity,
observed: MountIdentity,
}
impl FailpointRecovery {
fn boundary(&mut self, name: &'static str) -> io::Result<()> {
self.boundary += 1;
self.events.push(name);
if self.fail_at == Some(self.boundary) {
Err(io::Error::other("injected recovery boundary failure"))
} else {
Ok(())
}
}
}
impl PrivateRecoveryOperations for FailpointRecovery {
fn validate_pidfs(&mut self) -> io::Result<()> {
self.boundary("pidfs")
}
fn enter_user_namespace(&mut self) -> io::Result<()> {
self.boundary("enter-user")
}
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
self.boundary("observe-user")?;
Ok(self.user)
}
fn enter_mount_namespace(&mut self) -> io::Result<()> {
self.boundary("enter-mount")
}
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
self.boundary("observe-mount")?;
Ok(self.mount_namespace)
}
fn verify_mount(&mut self) -> io::Result<MountIdentity> {
self.boundary("verify-mount")?;
Ok(self.mount)
}
fn unmount(&mut self) -> io::Result<()> {
self.boundary("unmount")
}
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
self.boundary("observe-absence")?;
Ok(self.observed)
}
}
fn fixture(fail_at: Option<usize>) -> FailpointRecovery {
FailpointRecovery {
fail_at,
boundary: 0,
events: Vec::new(),
user: NamespaceIdentity {
device: DeviceId::new(1),
inode: InodeId::new(2),
},
mount_namespace: NamespaceIdentity {
device: DeviceId::new(3),
inode: InodeId::new(4),
},
mount: MountIdentity {
mount_id: lazaret_core::identity::MountId::new(5),
device: DeviceId::new(6),
inode: InodeId::new(7),
},
observed: MountIdentity {
mount_id: lazaret_core::identity::MountId::new(8),
device: DeviceId::new(6),
inode: InodeId::new(7),
},
}
}
#[test]
fn private_recovery_failpoints_refuse_before_every_later_boundary() {
for boundary in 1..=8 {
let mut operations = fixture(Some(boundary));
let expected_user = operations.user;
let expected_mount_namespace = operations.mount_namespace;
let expected_mount = operations.mount;
let result = execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
);
assert!(result.is_err());
assert_eq!(operations.events.len(), boundary);
}
let mut operations = fixture(None);
let expected_user = operations.user;
let expected_mount_namespace = operations.mount_namespace;
let expected_mount = operations.mount;
assert_eq!(
execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
)
.unwrap(),
operations.observed
);
assert_eq!(
operations.events,
[
"pidfs",
"enter-user",
"observe-user",
"enter-mount",
"observe-mount",
"verify-mount",
"unmount",
"observe-absence",
]
);
}
#[test]
fn final_mount_open_refuses_intermediate_and_link_traversal() {
let root = std::env::temp_dir().join(format!(
"lazaret-final-mount-open-{}-{}",
std::process::id(),
TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(root.join("nested/final")).unwrap();
symlink("nested", root.join("link")).unwrap();
let parent = File::open(&root).unwrap();
let descriptor = File::open(root.join("nested")).unwrap();
symlink(
format!("/proc/self/fd/{}", descriptor.as_raw_fd()),
root.join("magic"),
)
.unwrap();
assert!(open_final_mount_directory(&parent, Path::new("nested/final")).is_err());
assert!(open_final_mount_directory(&parent, Path::new("link")).is_err());
assert!(open_final_mount_directory(&parent, Path::new("magic")).is_err());
assert!(open_final_mount_directory(&parent, Path::new("nested")).is_ok());
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn private_recovery_refuses_wrong_or_absent_final_mount_before_action() {
for observed_mount_id in [4, 8] {
let mut operations = fixture(None);
operations.mount.mount_id = lazaret_core::identity::MountId::new(observed_mount_id);
let expected_user = operations.user;
let expected_mount_namespace = operations.mount_namespace;
let expected_mount = MountIdentity {
mount_id: lazaret_core::identity::MountId::new(5),
device: DeviceId::new(6),
inode: InodeId::new(7),
};
assert!(
execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
)
.is_err()
);
assert_eq!(
operations.events,
[
"pidfs",
"enter-user",
"observe-user",
"enter-mount",
"observe-mount",
"verify-mount",
]
);
}
}
}

161
crates/lazaret-state-anchor/tests/state-anchor-lifecycle.rs
View file

@ -1,12 +1,13 @@
use std::collections::BTreeMap;
use std::ffi::OsString;
use std::fs;
use std::io;
use std::io::{self, BufRead, BufReader, Read, Write};
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use lazaret_core::configuration::{
ConfigurationLoadRequest, SandboxLayoutV1, load_trusted_configuration,
@ -16,12 +17,14 @@ use lazaret_core::records::{
ArtifactState, HostProcessOwnershipSlot, PreparedArtifact, ProcessOwnershipCommitAck,
};
use lazaret_linux::process::{
ProcessOwnershipContext, current_process_identity, dispatch_private_process_role,
ProcessOwnershipContext, TargetCommand, child_broker, current_process_identity,
dispatch_private_process_role,
};
use lazaret_state_anchor::{
FuseOverlayfsMount, ProtectedRuntimeFiles, RuntimeStateAnchorAuthority,
RuntimeStateAnchorInherited, StateAnchorAcquisition, StateAnchorLaunchAuthority,
StateAnchorLayout, StateAnchorSandboxView, unmount_after_coordinator_loss,
StateAnchorLayout, StateAnchorSandboxView, dispatch_private_state_anchor_recovery,
recover_private_state_anchor_mount, unmount_after_coordinator_loss,
};
use rustix::process::Signal;
@ -29,6 +32,15 @@ static SEQUENCE: AtomicU64 = AtomicU64::new(0);
fn main() -> ExitCode {
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
if arguments
.first()
.is_some_and(|value| value == "--private-recovery-holder")
{
return exit(run_private_recovery_holder(&arguments));
}
if let Some(result) = dispatch_private_state_anchor_recovery(&arguments) {
return exit(result);
}
if let Some(result) = dispatch_private_process_role(&arguments) {
return exit(result);
}
@ -55,8 +67,9 @@ fn run() -> io::Result<()> {
}
Err(error) => return Err(error),
}
let tree = TestTree::new()?;
let configuration = configuration(&tree)?;
let tree = TestTree::new().map_err(|error| io::Error::other(format!("test tree: {error}")))?;
let configuration = configuration(&tree)
.map_err(|error| io::Error::other(format!("configuration: {error}")))?;
let lower = tree.path.join("prepared-lower");
fs::create_dir_all(lower.join("usr/bin"))?;
fs::create_dir_all(lower.join("etc"))?;
@ -76,7 +89,10 @@ fn run() -> io::Result<()> {
inode: lazaret_core::identity::InodeId::new(metadata.ino()),
state: ArtifactState::Available,
};
let layout = StateAnchorLayout::create(&configuration).map_err(io::Error::other)?;
let layout = StateAnchorLayout::create(&configuration)
.map_err(|error| io::Error::other(format!("State Anchor layout: {error}")))?;
exercise_private_recovery(&layout)
.map_err(|error| io::Error::other(format!("private recovery exercise: {error}")))?;
let launch =
StateAnchorLaunchAuthority::capture(&layout, &artifact).map_err(io::Error::other)?;
@ -197,6 +213,139 @@ fn run() -> io::Result<()> {
Ok(())
}
#[allow(deprecated)]
fn run_private_recovery_holder(arguments: &[OsString]) -> io::Result<()> {
if arguments.len() != 2 {
return Err(io::Error::other("private holder arguments are malformed"));
}
let merged = PathBuf::from(&arguments[1]);
let uid = rustix::process::getuid().as_raw();
let gid = rustix::process::getgid().as_raw();
rustix::thread::unshare(rustix::thread::UnshareFlags::NEWUSER)
.map_err(|error| io::Error::other(format!("holder user namespace: {error}")))?;
fs::write("/proc/self/setgroups", "deny")
.map_err(|error| io::Error::other(format!("holder setgroups: {error}")))?;
fs::write("/proc/self/uid_map", format!("0 {uid} 1\n"))
.map_err(|error| io::Error::other(format!("holder uid map: {error}")))?;
fs::write("/proc/self/gid_map", format!("0 {gid} 1\n"))
.map_err(|error| io::Error::other(format!("holder gid map: {error}")))?;
rustix::thread::unshare(rustix::thread::UnshareFlags::NEWNS)
.map_err(|error| io::Error::other(format!("holder mount namespace: {error}")))?;
rustix::mount::mount_change(
"/",
rustix::mount::MountPropagationFlags::PRIVATE | rustix::mount::MountPropagationFlags::REC,
)
.map_err(|error| io::Error::other(format!("holder private propagation: {error}")))?;
rustix::mount::mount(
"tmpfs",
&merged,
"tmpfs",
rustix::mount::MountFlags::empty(),
c"size=1m",
)
.map_err(|error| io::Error::other(format!("holder synthetic mount: {error}")))?;
let user = lazaret_linux::filesystem::namespace_identity(Path::new("/proc/self/ns/user"))?;
let mount_namespace =
lazaret_linux::filesystem::namespace_identity(Path::new("/proc/self/ns/mnt"))?;
let mount = lazaret_linux::filesystem::current_mount_identity(&merged)?;
writeln!(
io::stdout(),
"{} {} {} {} {} {} {}",
user.device.get(),
user.inode.get(),
mount_namespace.device.get(),
mount_namespace.inode.get(),
mount.mount_id.get(),
mount.device.get(),
mount.inode.get(),
)?;
io::stdout().flush()?;
loop {
std::thread::park();
}
}
fn exercise_private_recovery(layout: &StateAnchorLayout<'_>) -> io::Result<()> {
let target = TargetCommand::new("/proc/self/exe")
.argument("--private-recovery-holder")
.argument(layout.merged_label());
let mut captured = child_broker()
.run_transient_captured(&target)
.map_err(|error| io::Error::other(format!("starting private holder: {error}")))?;
let values = {
let (stdout, stderr) = captured.streams();
let mut reader = BufReader::new(stdout);
let mut line = String::new();
let deadline = Instant::now() + Duration::from_secs(5);
loop {
match reader.read_line(&mut line) {
Ok(0) if Instant::now() < deadline => std::thread::yield_now(),
Ok(0) => {
let mut detail = String::new();
match stderr.read_to_string(&mut detail) {
Ok(_) => {}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {}
Err(error) => return Err(error),
}
return Err(io::Error::other(format!(
"private holder closed without evidence: {}",
detail.trim()
)));
}
Ok(_) => break,
Err(error)
if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline =>
{
std::thread::yield_now();
}
Err(error) => return Err(error),
}
}
line.split_whitespace()
.map(|value| {
value
.parse::<u64>()
.map_err(|_| io::Error::other("private holder output is malformed"))
})
.collect::<io::Result<Vec<_>>>()?
};
if values.len() != 7 {
return Err(io::Error::other("private holder evidence is incomplete"));
}
let holder = captured.into_process();
let user = lazaret_core::identity::NamespaceIdentity {
device: lazaret_core::identity::DeviceId::new(values[0]),
inode: lazaret_core::identity::InodeId::new(values[1]),
};
let mount_namespace = lazaret_core::identity::NamespaceIdentity {
device: lazaret_core::identity::DeviceId::new(values[2]),
inode: lazaret_core::identity::InodeId::new(values[3]),
};
let mount = lazaret_core::identity::MountIdentity {
mount_id: lazaret_core::identity::MountId::new(values[4]),
device: lazaret_core::identity::DeviceId::new(values[5]),
inode: lazaret_core::identity::InodeId::new(values[6]),
};
let wrong = lazaret_core::identity::MountIdentity {
mount_id: lazaret_core::identity::MountId::new(mount.mount_id.get() + 1),
..mount
};
if recover_private_state_anchor_mount(layout, &holder, user, mount_namespace, wrong).is_ok() {
return Err(io::Error::other("private recovery accepted wrong mount"));
}
let observed =
recover_private_state_anchor_mount(layout, &holder, user, mount_namespace, mount)
.map_err(io::Error::other)?;
if observed == mount {
return Err(io::Error::other("private recovery did not prove absence"));
}
holder.send_signal(Signal::KILL)?;
while !holder.has_exited()? {
std::thread::yield_now();
}
child_broker().reap(holder)
}
fn acquire<'layout, 'configuration>(
layout: &'layout StateAnchorLayout<'configuration>,
artifact: &PreparedArtifact,