Implement exact bootstrap recovery and artifact GC

This commit is contained in:
ms2k 2026-07-17 05:38:43 +09:00
commit d25942596f
17 changed files with 1631 additions and 48 deletions

View file

@ -495,6 +495,9 @@ impl Participant {
| ParticipantState::Committed, | ParticipantState::Committed,
ParticipantState::ReleaseIntent, ParticipantState::ReleaseIntent,
) => validate_participant_release(self.ownership.as_ref(), next.ownership.as_ref()), ) => validate_participant_release(self.ownership.as_ref(), next.ownership.as_ref()),
(ParticipantState::ReleaseIntent, ParticipantState::ReleaseIntent) => {
validate_participant_release(self.ownership.as_ref(), next.ownership.as_ref())
}
(old_state, new_state) (old_state, new_state)
if old_state == new_state if old_state == new_state
&& matches!( && matches!(
@ -502,7 +505,6 @@ impl Participant {
ParticipantState::LeaseHeld ParticipantState::LeaseHeld
| ParticipantState::Prepared | ParticipantState::Prepared
| ParticipantState::Committed | ParticipantState::Committed
| ParticipantState::ReleaseIntent
| ParticipantState::Clean | ParticipantState::Clean
) )
&& self.ownership == next.ownership => && self.ownership == next.ownership =>
@ -979,6 +981,15 @@ pub struct PreparedLowerOwnership {
pub path: Option<HostPath>, pub path: Option<HostPath>,
pub device: Option<DeviceId>, pub device: Option<DeviceId>,
pub inode: Option<InodeId>, pub inode: Option<InodeId>,
pub released_state_anchor: Option<StateAnchorReleaseEvidence>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct StateAnchorReleaseEvidence {
pub daemon: PidfsProcessIdentityV1,
pub user_namespace: NamespaceIdentity,
pub mount_namespace: NamespaceIdentity,
pub mount: MountIdentity,
} }
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
@ -1043,6 +1054,7 @@ impl PreparedLowerOwnership {
path: None, path: None,
device: None, device: None,
inode: None, inode: None,
released_state_anchor: None,
} }
} }
@ -1053,6 +1065,7 @@ impl PreparedLowerOwnership {
&& self.path.is_none() && self.path.is_none()
&& self.device.is_none() && self.device.is_none()
&& self.inode.is_none() && self.inode.is_none()
&& self.released_state_anchor.is_none()
} }
fn is_lease_held(&self) -> bool { fn is_lease_held(&self) -> bool {
@ -1062,6 +1075,7 @@ impl PreparedLowerOwnership {
&& self.path.is_some() && self.path.is_some()
&& self.device.is_some() && self.device.is_some()
&& self.inode.is_some() && self.inode.is_some()
&& self.released_state_anchor.is_none()
} }
fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> { fn validate_progress_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
@ -1073,7 +1087,11 @@ impl PreparedLowerOwnership {
validate_prepared_lower_lease_progress(self.lease_state, next.lease_state)?; validate_prepared_lower_lease_progress(self.lease_state, next.lease_state)?;
validate_option_progress(self.path.as_ref(), next.path.as_ref())?; validate_option_progress(self.path.as_ref(), next.path.as_ref())?;
validate_option_progress(self.device.as_ref(), next.device.as_ref())?; validate_option_progress(self.device.as_ref(), next.device.as_ref())?;
validate_option_progress(self.inode.as_ref(), next.inode.as_ref()) validate_option_progress(self.inode.as_ref(), next.inode.as_ref())?;
validate_option_progress(
self.released_state_anchor.as_ref(),
next.released_state_anchor.as_ref(),
)
} }
fn validate_release_to(&self, next: &Self) -> Result<(), RecordTransitionError> { fn validate_release_to(&self, next: &Self) -> Result<(), RecordTransitionError> {
@ -1085,6 +1103,15 @@ impl PreparedLowerOwnership {
{ {
return Err(RecordTransitionError::EvidenceChanged); return Err(RecordTransitionError::EvidenceChanged);
} }
let release_evidence_valid = match self.lease_state {
Some(ParticipantState::ReleaseIntent) => {
self.released_state_anchor == next.released_state_anchor
}
_ => next.released_state_anchor.is_some() || self.released_state_anchor.is_none(),
};
if !release_evidence_valid {
return Err(RecordTransitionError::EvidenceChanged);
}
match (self.lease_state, next.lease_state) { match (self.lease_state, next.lease_state) {
(None, None) (None, None)
| (Some(ParticipantState::ReleaseIntent), Some(ParticipantState::ReleaseIntent)) => { | (Some(ParticipantState::ReleaseIntent), Some(ParticipantState::ReleaseIntent)) => {

View file

@ -366,6 +366,7 @@ fn prepared_ownership() -> PreparedLowerOwnership {
path: Some(HostPath::new(vec![b'/', 0xff])), path: Some(HostPath::new(vec![b'/', 0xff])),
device: Some(DeviceId::new(27)), device: Some(DeviceId::new(27)),
inode: Some(InodeId::new(28)), inode: Some(InodeId::new(28)),
released_state_anchor: None,
} }
} }

View file

@ -514,6 +514,51 @@ fn prepared_lower_release_changes_only_the_lease_phase() {
.unwrap(); .unwrap();
assert_eq!(active.validate_transition_to(&release), Ok(())); assert_eq!(active.validate_transition_to(&release), Ok(()));
let ParticipantOwnership::PreparedLower(mut with_anchor_evidence) =
release_ownership(ParticipantRole::PreparedLower)
else {
unreachable!()
};
with_anchor_evidence.released_state_anchor = Some(StateAnchorReleaseEvidence {
daemon: process(120),
user_namespace: namespace(),
mount_namespace: NamespaceIdentity {
device: DeviceId::new(121),
inode: InodeId::new(122),
},
mount: mount(),
});
let release_with_anchor_evidence = Participant::snapshot(
generation_id(),
ParticipantRole::PreparedLower,
ParticipantState::ReleaseIntent,
Some(ParticipantOwnership::PreparedLower(
with_anchor_evidence.clone(),
)),
)
.unwrap();
assert_eq!(
active.validate_transition_to(&release_with_anchor_evidence),
Ok(())
);
with_anchor_evidence
.released_state_anchor
.as_mut()
.unwrap()
.mount
.mount_id = MountId::new(123);
let changed_release_evidence = Participant::snapshot(
generation_id(),
ParticipantRole::PreparedLower,
ParticipantState::ReleaseIntent,
Some(ParticipantOwnership::PreparedLower(with_anchor_evidence)),
)
.unwrap();
assert_eq!(
release_with_anchor_evidence.validate_transition_to(&changed_release_evidence),
Err(RecordTransitionError::EvidenceChanged)
);
let ParticipantOwnership::PreparedLower(mut changed) = let ParticipantOwnership::PreparedLower(mut changed) =
release_ownership(ParticipantRole::PreparedLower) release_ownership(ParticipantRole::PreparedLower)
else { else {

View file

@ -9,6 +9,7 @@ pub mod lifecycle;
pub mod network; pub mod network;
pub mod production; pub mod production;
pub mod production_selector; pub mod production_selector;
pub mod recovery;
pub mod runtime; pub mod runtime;
pub mod sandbox_init; pub mod sandbox_init;
pub mod state_anchor; pub mod state_anchor;

View file

@ -18,6 +18,13 @@ fn main() {
} }
return; return;
} }
if let Some(result) = lazaret_state_anchor::dispatch_private_state_anchor_recovery(&arguments) {
if let Err(error) = result {
eprintln!("State Anchor recovery failed: {error}");
std::process::exit(1);
}
return;
}
if let Some(result) = lazaret_linux::process::dispatch_private_process_role(&arguments) { if let Some(result) = lazaret_linux::process::dispatch_private_process_role(&arguments) {
if let Err(error) = result { if let Err(error) = result {
eprintln!("private process role failed: {error}"); eprintln!("private process role failed: {error}");

View file

@ -0,0 +1,297 @@
use lazaret_core::identity::{
GenerationId, MountIdentity, NamespaceIdentity, PidfsProcessIdentityV1,
};
use lazaret_core::records::{
Participant, ParticipantOwnership, ParticipantRole, PersistedHostProcessOwnership,
};
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use crate::state_anchor::RuntimeOwnersAbsent;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct RuntimeRecoveryOwnership {
pub host_coordinator: Option<PidfsProcessIdentityV1>,
pub runtime_supervisor: Option<PidfsProcessIdentityV1>,
pub attached_runsc: Option<PidfsProcessIdentityV1>,
pub connectivity_helper: Option<PidfsProcessIdentityV1>,
pub state_anchor_daemon: Option<PidfsProcessIdentityV1>,
pub git_routing_daemon: Option<PidfsProcessIdentityV1>,
pub state_anchor_user_namespace: Option<NamespaceIdentity>,
pub state_anchor_mount_namespace: Option<NamespaceIdentity>,
pub state_anchor_mount: Option<MountIdentity>,
}
/// Extracts one generation's exact process and private-mount recovery authority.
///
/// # Errors
/// Refuses unequal copies of the constitutive HC or RS identities, and incomplete State Anchor
/// namespace/mount evidence. Staged identities remain cleanup authority.
pub fn runtime_recovery_ownership(
participants: &[Participant],
generation_id: GenerationId,
) -> Result<RuntimeRecoveryOwnership, &'static str> {
let mut result = RuntimeRecoveryOwnership::default();
for participant in participants
.iter()
.filter(|participant| participant.generation_id() == generation_id)
{
let Some(ownership) = participant.ownership() else {
continue;
};
match ownership {
ParticipantOwnership::Sandbox(sandbox) => {
merge_process(
&mut result.host_coordinator,
&sandbox.host_coordinator,
"Sandbox HC identity mismatch",
)?;
merge_process(
&mut result.runtime_supervisor,
&sandbox.runtime_supervisor,
"Sandbox RS identity mismatch",
)?;
merge_process(
&mut result.attached_runsc,
&sandbox.attached_runsc,
"attached runsc identity mismatch",
)?;
}
ParticipantOwnership::Network(network) => merge_process(
&mut result.runtime_supervisor,
&network.runtime_supervisor,
"Network RS identity mismatch",
)?,
ParticipantOwnership::Helper(helper) => {
merge_process(
&mut result.connectivity_helper,
&helper.connectivity_helper,
"connectivity-helper identity mismatch",
)?;
merge_process(
&mut result.runtime_supervisor,
&helper.runtime_supervisor,
"Helper RS identity mismatch",
)?;
}
ParticipantOwnership::StateAnchor(anchor) => {
merge_process(
&mut result.state_anchor_daemon,
&anchor.daemon,
"State Anchor daemon identity mismatch",
)?;
let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
if evidence == (None, None, None) {
continue;
}
let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
return Err("State Anchor recovery evidence is incomplete");
};
merge_value(
&mut result.state_anchor_user_namespace,
user,
"State Anchor user namespace mismatch",
)?;
merge_value(
&mut result.state_anchor_mount_namespace,
mount_namespace,
"State Anchor mount namespace mismatch",
)?;
merge_value(
&mut result.state_anchor_mount,
mount,
"State Anchor mount identity mismatch",
)?;
}
ParticipantOwnership::GitRouting(routing) => merge_process(
&mut result.git_routing_daemon,
&routing.daemon,
"Git-routing daemon identity mismatch",
)?,
ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
}
}
Ok(result)
}
fn merge_process(
target: &mut Option<PidfsProcessIdentityV1>,
observed: &PersistedHostProcessOwnership,
mismatch: &'static str,
) -> Result<(), &'static str> {
let Ok(observed) = observed.authoritative_pidfs() else {
return Ok(());
};
merge_value(target, *observed, mismatch)
}
fn merge_value<T: Copy + Eq>(
target: &mut Option<T>,
observed: T,
mismatch: &'static str,
) -> Result<(), &'static str> {
match target {
None => *target = Some(observed),
Some(current) if *current == observed => {}
Some(_) => return Err(mismatch),
}
Ok(())
}
#[must_use]
pub fn recovery_role_order() -> [ParticipantRole; 7] {
[
ParticipantRole::Publication,
ParticipantRole::Sandbox,
ParticipantRole::Helper,
ParticipantRole::Network,
ParticipantRole::GitRouting,
ParticipantRole::StateAnchor,
ParticipantRole::PreparedLower,
]
}
/// Proves the constitutive RS and attached runsc are absent before private-mount reconciliation.
///
/// # Errors
/// Refuses a live or stopped exact authority and every unreadable or ambiguous observation.
pub fn prove_runtime_owners_absent(
generation_id: GenerationId,
ownership: &RuntimeRecoveryOwnership,
) -> Result<RuntimeOwnersAbsent, std::io::Error> {
for identity in [ownership.attached_runsc, ownership.runtime_supervisor]
.into_iter()
.flatten()
{
match match_recovery_process(identity) {
Ok(PidfsRecoveryMatch::BootMismatch) => {}
Ok(PidfsRecoveryMatch::Matched(process)) if process.has_exited()? => {}
Ok(PidfsRecoveryMatch::Matched(_)) => {
return Err(std::io::Error::other(
"constitutive runtime authority remains present",
));
}
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {}
Err(error) => return Err(error),
}
}
Ok(RuntimeOwnersAbsent::for_generation(generation_id))
}
#[cfg(test)]
mod tests {
use lazaret_core::identity::{
BootId, BundleDigest, ContainerId, DeviceId, HostPath, HostPidLocator, MountId,
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks,
};
use lazaret_core::records::{
HelperOwnership, ParticipantState, SandboxOwnership, StateAnchorOwnership,
};
use super::*;
fn process(value: u64) -> PidfsProcessIdentityV1 {
PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new([1; 16]),
pidfs_st_dev: PidfsDeviceId::new(value),
pidfs_st_ino: PidfsInodeId::new(value),
},
HostPidLocator::new(u32::try_from(value).unwrap()),
ProcessStartTimeTicks::new(value),
)
}
fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant {
Participant::snapshot(
GenerationId::new(1),
role,
ParticipantState::Intent,
Some(ownership),
)
.unwrap()
}
#[test]
fn inventory_accepts_staged_cleanup_authority_and_rejects_cross_role_rs_mismatch() {
let rs = process(2);
let sandbox = participant(
ParticipantRole::Sandbox,
ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new([3; 32])),
runtime_root: Some(HostPath::new(b"/runtime".to_vec())),
bundle_digest: Some(BundleDigest::new([4; 32])),
host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
runtime_supervisor: PersistedHostProcessOwnership::staged(rs),
attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
}),
);
let helper = |runtime_supervisor| {
participant(
ParticipantRole::Helper,
ParticipantOwnership::Helper(HelperOwnership {
network_sidecar_id: None,
connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
runtime_supervisor: PersistedHostProcessOwnership::committed(
runtime_supervisor,
),
}),
)
};
let inventory =
runtime_recovery_ownership(&[sandbox.clone(), helper(rs)], GenerationId::new(1))
.unwrap();
assert_eq!(inventory.runtime_supervisor, Some(rs));
assert_eq!(inventory.attached_runsc, Some(process(3)));
assert_eq!(inventory.connectivity_helper, Some(process(4)));
assert!(
runtime_recovery_ownership(&[sandbox, helper(process(99))], GenerationId::new(1),)
.is_err()
);
}
#[test]
fn inventory_requires_complete_private_mount_evidence() {
let incomplete = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: Some(HostPath::new(b"upper".to_vec())),
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: Some(NamespaceIdentity {
device: DeviceId::new(6),
inode: lazaret_core::identity::InodeId::new(7),
}),
mount_namespace: None,
mount: None,
}),
);
assert!(runtime_recovery_ownership(&[incomplete], GenerationId::new(1)).is_err());
let complete = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: Some(HostPath::new(b"upper".to_vec())),
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: Some(NamespaceIdentity {
device: DeviceId::new(6),
inode: lazaret_core::identity::InodeId::new(7),
}),
mount_namespace: Some(NamespaceIdentity {
device: DeviceId::new(8),
inode: lazaret_core::identity::InodeId::new(9),
}),
mount: Some(MountIdentity {
mount_id: MountId::new(10),
device: DeviceId::new(11),
inode: lazaret_core::identity::InodeId::new(12),
}),
}),
);
assert!(runtime_recovery_ownership(&[complete], GenerationId::new(1)).is_ok());
}
}

View file

@ -10,8 +10,11 @@ use lazaret_core::records::{
ArtifactLease, HostProcessOwnershipSlot, Participant, ParticipantOwnership, ParticipantRole, ArtifactLease, HostProcessOwnershipSlot, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PersistedHostProcessOwnership, PreparedArtifact, PreparedLowerOwnership, ParticipantState, PersistedHostProcessOwnership, PreparedArtifact, PreparedLowerOwnership,
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership, ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
StateAnchorReleaseEvidence,
}; };
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness}; use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness};
use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_mount};
use lazaret_storage_catalog::{Catalog, CatalogError}; use lazaret_storage_catalog::{Catalog, CatalogError};
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore}; use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
@ -103,6 +106,17 @@ impl StateAnchorReleaseAuthority {
} }
} }
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct RuntimeOwnersAbsent {
generation_id: GenerationId,
}
impl RuntimeOwnersAbsent {
pub(crate) const fn for_generation(generation_id: GenerationId) -> Self {
Self { generation_id }
}
}
pub struct StateAnchorCoordinator<'a> { pub struct StateAnchorCoordinator<'a> {
domain_store: &'a HostCoordinatorDomainStore, domain_store: &'a HostCoordinatorDomainStore,
catalog: &'a Catalog, catalog: &'a Catalog,
@ -140,6 +154,7 @@ impl<'a> StateAnchorCoordinator<'a> {
path: Some(artifact.path.clone()), path: Some(artifact.path.clone()),
device: Some(artifact.device), device: Some(artifact.device),
inode: Some(artifact.inode), inode: Some(artifact.inode),
released_state_anchor: None,
}; };
let expected_prepared_intent = participant( let expected_prepared_intent = participant(
generation_id, generation_id,
@ -364,6 +379,52 @@ impl<'a> StateAnchorCoordinator<'a> {
Ok(authority) Ok(authority)
} }
/// Recovers the exact private State Anchor after all constitutive runtime owners are absent.
///
/// # Errors
/// Refuses generation drift, pidfs/namespace/mount mismatch, unreadable absence, or helper
/// failure. Boot mismatch and `ESRCH` reconcile physical absence without process action only
/// under the caller-provided ordered runtime-owner absence witness.
pub fn recover_after_runtime_owner_loss(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
owners_absent: RuntimeOwnersAbsent,
) -> Result<
(StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
StateAnchorCoordinatorError,
> {
if owners_absent.generation_id != generation_id {
return Err(StateAnchorCoordinatorError::InvalidState(
"runtime-owner absence belongs to another generation",
));
}
let authority = self.begin_release(generation_id, layout)?;
let observed = match match_recovery_process(authority.daemon) {
Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
layout,
&holder,
authority.user_namespace,
authority.mount_namespace,
authority.mount,
)?,
Ok(PidfsRecoveryMatch::BootMismatch) => observe_mount_absence(layout, authority.mount)?,
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
observe_mount_absence(layout, authority.mount)?
}
Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
};
Ok((
authority.clone(),
StateAnchorAbsenceObservation {
user_namespace: authority.user_namespace,
mount_namespace: authority.mount_namespace,
mount: authority.mount,
observed,
},
))
}
/// Completes Domain Clean, exact catalog deletion, and prepared-lower Clean after unmount. /// Completes Domain Clean, exact catalog deletion, and prepared-lower Clean after unmount.
/// ///
/// # Errors /// # Errors
@ -404,7 +465,15 @@ impl<'a> StateAnchorCoordinator<'a> {
authority.generation_id, authority.generation_id,
ParticipantRole::StateAnchor, ParticipantRole::StateAnchor,
)?), )?),
DomainStoreMutation::PutParticipant(prepared_release_intent(&prepared)?), DomainStoreMutation::PutParticipant(prepared_release_intent(
&prepared,
StateAnchorReleaseEvidence {
daemon: authority.daemon,
user_namespace: authority.user_namespace,
mount_namespace: authority.mount_namespace,
mount: authority.mount,
},
)?),
])?; ])?;
} }
(ParticipantState::Clean, ParticipantState::ReleaseIntent) => {} (ParticipantState::Clean, ParticipantState::ReleaseIntent) => {}
@ -422,6 +491,80 @@ impl<'a> StateAnchorCoordinator<'a> {
Ok(()) Ok(())
} }
/// Resumes lease release after the atomic State Anchor Clean checkpoint was committed.
///
/// # Errors
/// Refuses missing retained release evidence, generation or layout drift, a mismatched lease,
/// or failure of the fresh catalog-lock absence observation.
pub fn resume_release_after_checkpoint<Observe>(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
mut observe_again: Observe,
) -> Result<(), StateAnchorCoordinatorError>
where
Observe: FnMut() -> Result<StateAnchorAbsenceObservation, StateAnchorCoordinatorError>,
{
let snapshot = self.domain_store.snapshot()?;
let anchor = role(
&snapshot.participants,
generation_id,
ParticipantRole::StateAnchor,
)?;
let prepared = role(
&snapshot.participants,
generation_id,
ParticipantRole::PreparedLower,
)?;
if anchor.state() != ParticipantState::Clean
|| prepared.state() != ParticipantState::ReleaseIntent
{
return Err(StateAnchorCoordinatorError::InvalidState(
"durable release checkpoint is absent",
));
}
let Some(ParticipantOwnership::PreparedLower(ownership)) = prepared.ownership() else {
return Err(StateAnchorCoordinatorError::InvalidState(
"prepared-lower ownership is invalid",
));
};
let evidence =
ownership
.released_state_anchor
.ok_or(StateAnchorCoordinatorError::InvalidState(
"prepared-lower release has no exact State Anchor evidence",
))?;
let artifact =
ownership
.prepared_artifact_id
.ok_or(StateAnchorCoordinatorError::InvalidState(
"prepared-lower artifact is absent",
))?;
let expected_layout = layout.ownership_intent(artifact);
let merged = expected_layout.merged_path.as_ref().ok_or(
StateAnchorCoordinatorError::InvalidState("State Anchor merged path is absent"),
)?;
let authority = StateAnchorReleaseAuthority {
generation_id,
lease: ArtifactLease {
prepared_artifact_id: artifact,
domain_id: snapshot.domain.domain_id,
generation_id,
},
merged_label: path_from_record(merged).to_path_buf(),
mount: evidence.mount,
daemon: evidence.daemon,
user_namespace: evidence.user_namespace,
mount_namespace: evidence.mount_namespace,
};
self.release_catalog_lease_if_present(&authority, &mut observe_again)?;
self.domain_store
.apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(
clean_participant(generation_id, ParticipantRole::PreparedLower)?,
)])?;
Ok(())
}
fn require_staged_identity( fn require_staged_identity(
&self, &self,
staged: StagedProcessOwnership, staged: StagedProcessOwnership,
@ -754,10 +897,12 @@ fn clean_participant(
fn prepared_release_intent( fn prepared_release_intent(
prepared: &Participant, prepared: &Participant,
released_state_anchor: StateAnchorReleaseEvidence,
) -> Result<Participant, StateAnchorCoordinatorError> { ) -> Result<Participant, StateAnchorCoordinatorError> {
let ownership = match prepared.ownership() { let ownership = match prepared.ownership() {
Some(ParticipantOwnership::PreparedLower(ownership)) => PreparedLowerOwnership { Some(ParticipantOwnership::PreparedLower(ownership)) => PreparedLowerOwnership {
lease_state: Some(ParticipantState::ReleaseIntent), lease_state: Some(ParticipantState::ReleaseIntent),
released_state_anchor: Some(released_state_anchor),
..ownership.clone() ..ownership.clone()
}, },
_ => { _ => {

View file

@ -232,20 +232,31 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
DomainStoreMutation::CleanParticipantAfterAbsence( DomainStoreMutation::CleanParticipantAfterAbsence(
clean_participant(generation_id, ParticipantRole::StateAnchor).unwrap(), clean_participant(generation_id, ParticipantRole::StateAnchor).unwrap(),
), ),
DomainStoreMutation::PutParticipant(prepared_release_intent(prepared).unwrap()), DomainStoreMutation::PutParticipant(
prepared_release_intent(
prepared,
StateAnchorReleaseEvidence {
daemon: authority.daemon(),
user_namespace: authority.user_namespace(),
mount_namespace: authority.mount_namespace(),
mount: authority.mount(),
},
)
.unwrap(),
),
]) ])
.unwrap(); .unwrap();
for invalid in invalid_observations { for invalid in invalid_observations {
assert!( assert!(
coordinator coordinator
.complete_release_after_unmount(&authority, &layout, valid, || Ok(invalid)) .resume_release_after_checkpoint(generation_id, &layout, || Ok(invalid))
.is_err() .is_err()
); );
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]); assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
} }
assert!( assert!(
coordinator coordinator
.complete_release_after_unmount(&authority, &layout, valid, || { .resume_release_after_checkpoint(generation_id, &layout, || {
Err(StateAnchorCoordinatorError::InvalidState( Err(StateAnchorCoordinatorError::InvalidState(
"State Anchor absence evidence is unreadable", "State Anchor absence evidence is unreadable",
)) ))
@ -255,7 +266,7 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]); assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
assert!( assert!(
coordinator coordinator
.complete_release_after_unmount(&authority, &layout, valid, || { .resume_release_after_checkpoint(generation_id, &layout, || {
Err(StateAnchorCoordinatorError::InvalidState( Err(StateAnchorCoordinatorError::InvalidState(
"second State Anchor observation was skipped", "second State Anchor observation was skipped",
)) ))
@ -265,7 +276,7 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]); assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
let mut observations = 0; let mut observations = 0;
coordinator coordinator
.complete_release_after_unmount(&authority, &layout, valid, || { .resume_release_after_checkpoint(generation_id, &layout, || {
observations += 1; observations += 1;
Ok(valid) Ok(valid)
}) })
@ -552,6 +563,7 @@ impl AcquisitionFixture {
path: Some(self.artifact.path.clone()), path: Some(self.artifact.path.clone()),
device: Some(self.artifact.device), device: Some(self.artifact.device),
inode: Some(self.artifact.inode), inode: Some(self.artifact.inode),
released_state_anchor: None,
}), }),
) )
.unwrap() .unwrap()

View file

@ -1,12 +1,16 @@
use std::io; use std::io::{self, IoSlice, IoSliceMut};
use std::mem::MaybeUninit;
use std::os::fd::{AsFd, OwnedFd}; use std::os::fd::{AsFd, OwnedFd};
use lazaret_core::codec::{decode, encode}; use lazaret_core::codec::{decode, encode};
use rustix::net::{ use rustix::net::{
AddressFamily, RecvFlags, SendFlags, SocketFlags, SocketType, recv, send, socketpair, AddressFamily, RecvAncillaryBuffer, RecvAncillaryMessage, RecvFlags, ReturnFlags,
SendAncillaryBuffer, SendAncillaryMessage, SendFlags, SocketFlags, SocketType, recv, recvmsg,
send, sendmsg, socketpair,
}; };
const MAX_PRIVATE_PACKET: usize = 64 * 1024; const MAX_PRIVATE_PACKET: usize = 64 * 1024;
const MAX_TRANSFERRED_DESCRIPTORS: usize = 8;
/// # Errors /// # Errors
/// Returns an error when the private sequenced-packet pair cannot be created. /// Returns an error when the private sequenced-packet pair cannot be created.
@ -61,13 +65,85 @@ where
decode(&encoded[..received]).map_err(|error| io::Error::other(error.to_string())) decode(&encoded[..received]).map_err(|error| io::Error::other(error.to_string()))
} }
/// Sends one nonempty bounded descriptor set with a one-byte packet.
///
/// # Errors
/// Refuses empty, oversized, incomplete, or failed descriptor transfer.
pub fn send_descriptors(
socket: impl AsFd,
descriptors: &[std::os::fd::BorrowedFd<'_>],
) -> io::Result<()> {
if descriptors.is_empty() || descriptors.len() > MAX_TRANSFERRED_DESCRIPTORS {
return Err(io::Error::other("private descriptor set has invalid size"));
}
let mut space =
[MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(MAX_TRANSFERRED_DESCRIPTORS))];
let mut control = SendAncillaryBuffer::new(&mut space);
if !control.push(SendAncillaryMessage::ScmRights(descriptors)) {
return Err(io::Error::other(
"private descriptor control buffer is too small",
));
}
let bytes = [1_u8];
let sent = sendmsg(
socket,
&[IoSlice::new(&bytes)],
&mut control,
SendFlags::empty(),
)?;
if sent == bytes.len() {
Ok(())
} else {
Err(io::Error::other("private descriptor packet was incomplete"))
}
}
/// Receives exactly `expected` descriptors with close-on-exec set atomically.
///
/// # Errors
/// Refuses missing, extra, truncated, malformed, or failed descriptor transfer.
pub fn receive_descriptors(socket: impl AsFd, expected: usize) -> io::Result<Vec<OwnedFd>> {
if expected == 0 || expected > MAX_TRANSFERRED_DESCRIPTORS {
return Err(io::Error::other("expected descriptor count is invalid"));
}
let mut byte = [0_u8];
let mut slices = [IoSliceMut::new(&mut byte)];
let mut space =
[MaybeUninit::uninit(); rustix::cmsg_space!(ScmRights(MAX_TRANSFERRED_DESCRIPTORS))];
let mut control = RecvAncillaryBuffer::new(&mut space);
let received = recvmsg(socket, &mut slices, &mut control, RecvFlags::CMSG_CLOEXEC)?;
if received.bytes != 1
|| byte != [1]
|| received
.flags
.intersects(ReturnFlags::TRUNC | ReturnFlags::CTRUNC)
{
return Err(io::Error::other("private descriptor packet is malformed"));
}
let mut descriptors = Vec::new();
for message in control.drain() {
match message {
RecvAncillaryMessage::ScmRights(rights) => descriptors.extend(rights),
_ => return Err(io::Error::other("private ancillary message is unexpected")),
}
}
if descriptors.len() == expected {
Ok(descriptors)
} else {
Err(io::Error::other(
"private descriptor count mismatches request",
))
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::os::fd::AsFd; use std::fs::File;
use std::os::fd::{AsFd, AsRawFd};
use rustix::net::{RecvFlags, SendFlags, recv, send}; use rustix::net::{RecvFlags, SendFlags, recv, send};
use super::private_seqpacket_pair; use super::{private_seqpacket_pair, receive_descriptors, send_descriptors};
#[test] #[test]
fn preserves_message_boundaries() { fn preserves_message_boundaries() {
@ -80,4 +156,22 @@ mod tests {
let (_, second) = recv(right.as_fd(), &mut buffer, RecvFlags::empty()).unwrap(); let (_, second) = recv(right.as_fd(), &mut buffer, RecvFlags::empty()).unwrap();
assert_eq!(&buffer[..second], b"second"); assert_eq!(&buffer[..second], b"second");
} }
#[test]
fn transfers_exact_close_on_exec_descriptors() {
let (left, right) = private_seqpacket_pair().unwrap();
let first = File::open("/dev/null").unwrap();
let second = File::open("/dev/zero").unwrap();
send_descriptors(left, &[first.as_fd(), second.as_fd()]).unwrap();
let received = receive_descriptors(right, 2).unwrap();
assert_ne!(received[0].as_raw_fd(), first.as_raw_fd());
assert_ne!(received[1].as_raw_fd(), second.as_raw_fd());
for descriptor in received {
assert!(
rustix::io::fcntl_getfd(&descriptor)
.unwrap()
.contains(rustix::io::FdFlags::CLOEXEC)
);
}
}
} }

View file

@ -223,6 +223,18 @@ pub struct RetainedProcess {
identity: PidfsProcessIdentityV1, identity: PidfsProcessIdentityV1,
} }
#[derive(Debug)]
pub enum PidfsRecoveryMatch {
BootMismatch,
Matched(RetainedProcess),
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum RecoveryActionResult<T> {
BootReconciled(T),
ProcessAction(T),
}
#[derive(Debug)] #[derive(Debug)]
pub struct CapturedProcess { pub struct CapturedProcess {
process: RetainedProcess, process: RetainedProcess,
@ -412,6 +424,27 @@ impl ChildBroker {
/// Returns an error when causal launch, exact waiting, or the command exit status fails. /// Returns an error when causal launch, exact waiting, or the command exit status fails.
#[allow(clippy::disallowed_methods, clippy::disallowed_types)] #[allow(clippy::disallowed_methods, clippy::disallowed_types)]
pub fn run_transient(&self, target: &TargetCommand) -> io::Result<ProcessExit> { pub fn run_transient(&self, target: &TargetCommand) -> io::Result<ProcessExit> {
self.run_transient_with_stdout(target, None)
}
/// Runs a non-durable direct child with one private inherited stdout descriptor.
///
/// # Errors
/// Returns an error when causal launch, exact waiting, or the command exit status fails.
pub fn run_transient_with_private_stdout(
&self,
target: &TargetCommand,
stdout: OwnedFd,
) -> io::Result<ProcessExit> {
self.run_transient_with_stdout(target, Some(stdout))
}
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
fn run_transient_with_stdout(
&self,
target: &TargetCommand,
stdout: Option<OwnedFd>,
) -> io::Result<ProcessExit> {
let _guard = self let _guard = self
.exclusion .exclusion
.lock() .lock()
@ -426,7 +459,7 @@ impl ChildBroker {
target.append_to(&mut command); target.append_to(&mut command);
command command
.stdin(wait.into_stdio()) .stdin(wait.into_stdio())
.stdout(Stdio::null()) .stdout(stdout.map_or_else(Stdio::null, Stdio::from))
.stderr(Stdio::null()); .stderr(Stdio::null());
let mut child = command.spawn()?; let mut child = command.spawn()?;
let process = match acquire_spawned_child(child.id()) { let process = match acquire_spawned_child(child.id()) {
@ -911,41 +944,87 @@ fn fail_after_pidfd_closed<T>(process: RetainedProcess, error: io::Error) -> io:
/// Refuses before PID lookup on boot mismatch and before action on every later mismatch. /// Refuses before PID lookup on boot mismatch and before action on every later mismatch.
#[allow(clippy::disallowed_methods)] #[allow(clippy::disallowed_methods)]
pub fn recover_process(identity: PidfsProcessIdentityV1) -> io::Result<RetainedProcess> { pub fn recover_process(identity: PidfsProcessIdentityV1) -> io::Result<RetainedProcess> {
recover_process_with(identity, read_boot_id, |host_pid| { match match_recovery_process(identity)? {
PidfsRecoveryMatch::BootMismatch => Err(io::Error::other(
"stored process authority belongs to another boot",
)),
PidfsRecoveryMatch::Matched(process) => Ok(process),
}
}
/// Reopens one stored process identity or reports boot mismatch before PID lookup.
///
/// # Errors
/// Refuses host-PID ambiguity, PID lookup, pidfs validation, identity, and negative-evidence
/// failures without returning an actionable descriptor.
#[allow(clippy::disallowed_methods)]
pub fn match_recovery_process(identity: PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch> {
match_recovery_process_with(
identity,
read_boot_id,
|host_pid| {
let raw_pid = i32::try_from(host_pid) let raw_pid = i32::try_from(host_pid)
.map_err(|_| io::Error::other("stored PID locator is out of range"))?; .map_err(|_| io::Error::other("stored PID locator is out of range"))?;
let pid = Pid::from_raw(raw_pid) let pid = Pid::from_raw(raw_pid)
.ok_or_else(|| io::Error::other("stored PID locator is invalid"))?; .ok_or_else(|| io::Error::other("stored PID locator is invalid"))?;
require_host_pid_namespace(host_pid)?; require_host_pid_namespace(host_pid)?;
rustix::process::pidfd_open(pid, PidfdFlags::empty()).map_err(Into::into) rustix::process::pidfd_open(pid, PidfdFlags::empty()).map_err(Into::into)
}) },
|pidfd, boot_id| authority_from_pidfd(pidfd, boot_id),
read_start_time,
)
} }
fn recover_process_with<Boot, Open>( /// Reconciles reboot-safe physical evidence or acts through the exactly reopened pidfd.
///
/// # Errors
/// Propagates reconciliation, pidfs matching, or same-descriptor action failure. The process
/// action is never called on boot mismatch or an unequal process identity.
pub fn recover_process_action<T>(
identity: PidfsProcessIdentityV1,
reconcile_boot: impl FnOnce() -> io::Result<T>,
action: impl FnOnce(&RetainedProcess) -> io::Result<T>,
) -> io::Result<RecoveryActionResult<T>> {
match match_recovery_process(identity)? {
PidfsRecoveryMatch::BootMismatch => {
reconcile_boot().map(RecoveryActionResult::BootReconciled)
}
PidfsRecoveryMatch::Matched(process) => {
action(&process).map(RecoveryActionResult::ProcessAction)
}
}
}
fn match_recovery_process_with<Boot, Open, Inspect, Start>(
identity: PidfsProcessIdentityV1, identity: PidfsProcessIdentityV1,
boot: Boot, boot: Boot,
open: Open, open: Open,
) -> io::Result<RetainedProcess> inspect: Inspect,
start: Start,
) -> io::Result<PidfsRecoveryMatch>
where where
Boot: FnOnce() -> io::Result<BootId>, Boot: FnOnce() -> io::Result<BootId>,
Open: FnOnce(u32) -> io::Result<OwnedFd>, Open: FnOnce(u32) -> io::Result<OwnedFd>,
Inspect: FnOnce(BorrowedFd<'_>, BootId) -> io::Result<PidfsProcessAuthorityV1>,
Start: FnOnce(u32) -> io::Result<u64>,
{ {
let current_boot = boot()?; let current_boot = boot()?;
if current_boot != identity.authority().boot_id { if current_boot != identity.authority().boot_id {
return Err(io::Error::other( return Ok(PidfsRecoveryMatch::BootMismatch);
"stored process authority belongs to another boot",
));
} }
let host_pid = identity.host_pid().get(); let host_pid = identity.host_pid().get();
let pidfd = open(host_pid)?; let pidfd = open(host_pid)?;
let observed = authority_from_pidfd(pidfd.as_fd(), current_boot)?; let observed = inspect(pidfd.as_fd(), current_boot)?;
if observed != identity.authority() { if observed != identity.authority() {
return Err(io::Error::other("pidfs process authority mismatch")); return Err(io::Error::other("pidfs process authority mismatch"));
} }
if read_start_time(host_pid)? != identity.start_time_ticks().get() { if start(host_pid)? != identity.start_time_ticks().get() {
return Err(io::Error::other("process start time mismatch")); return Err(io::Error::other("process start time mismatch"));
} }
Ok(RetainedProcess { pidfd, identity }) Ok(PidfsRecoveryMatch::Matched(RetainedProcess {
pidfd,
identity,
}))
} }
/// # Errors /// # Errors
@ -1491,16 +1570,136 @@ mod tests {
ProcessStartTimeTicks::new(1), ProcessStartTimeTicks::new(1),
); );
let opened = Cell::new(false); let opened = Cell::new(false);
let result = recover_process_with( let result = match_recovery_process_with(
identity, identity,
|| Ok(BootId::new([2; 16])), || Ok(BootId::new([2; 16])),
|_| { |_| {
opened.set(true); opened.set(true);
unreachable!() unreachable!()
}, },
|_, _| unreachable!(),
|_| unreachable!(),
);
assert!(matches!(result, Ok(PidfsRecoveryMatch::BootMismatch)));
assert!(!opened.get());
}
#[test]
fn recovery_refuses_each_post_open_mismatch_without_returning_authority() {
let authority = PidfsProcessAuthorityV1 {
boot_id: BootId::new([1; 16]),
pidfs_st_dev: PidfsDeviceId::new(7),
pidfs_st_ino: PidfsInodeId::new(9),
};
let identity = PidfsProcessIdentityV1::new(
authority,
HostPidLocator::new(41),
ProcessStartTimeTicks::new(43),
);
for (observed, start) in [
(
PidfsProcessAuthorityV1 {
pidfs_st_dev: PidfsDeviceId::new(8),
..authority
},
43,
),
(
PidfsProcessAuthorityV1 {
pidfs_st_ino: PidfsInodeId::new(10),
..authority
},
43,
),
(authority, 44),
] {
let events = Rc::new(std::cell::RefCell::new(Vec::new()));
let open_events = Rc::clone(&events);
let inspect_events = Rc::clone(&events);
let start_events = Rc::clone(&events);
let result = match_recovery_process_with(
identity,
|| Ok(authority.boot_id),
move |pid| {
open_events.borrow_mut().push(("open", u64::from(pid)));
Ok(File::open("/dev/null").unwrap().into())
},
move |fd, _| {
inspect_events
.borrow_mut()
.push(("inspect", u64::try_from(fd.as_raw_fd()).unwrap()));
Ok(observed)
},
move |pid| {
start_events.borrow_mut().push(("start", u64::from(pid)));
Ok(start)
},
); );
assert!(result.is_err()); assert!(result.is_err());
assert!(!opened.get()); assert_eq!(events.borrow()[0], ("open", 41));
assert_eq!(events.borrow()[1].0, "inspect");
if observed == authority {
assert_eq!(events.borrow()[2], ("start", 41));
} else {
assert_eq!(events.borrow().len(), 2);
}
}
for failure in ["open", "inspect", "start"] {
let result = match_recovery_process_with(
identity,
|| Ok(authority.boot_id),
|_| {
if failure == "open" {
Err(io::Error::from_raw_os_error(3))
} else {
Ok(File::open("/dev/null").unwrap().into())
}
},
|_, _| {
if failure == "inspect" {
Err(io::Error::other("injected descriptor inspection error"))
} else {
Ok(authority)
}
},
|_| {
if failure == "start" {
Err(io::Error::other("injected start observation error"))
} else {
Ok(43)
}
},
);
assert!(result.is_err(), "accepted injected {failure} failure");
}
}
#[test]
fn recovery_action_reconciles_boot_without_process_action() {
let mut old_boot = *read_boot_id().unwrap().as_bytes();
old_boot[0] ^= u8::MAX;
let identity = PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new(old_boot),
pidfs_st_dev: PidfsDeviceId::new(1),
pidfs_st_ino: PidfsInodeId::new(1),
},
HostPidLocator::new(u32::MAX),
ProcessStartTimeTicks::new(u64::MAX),
);
let action_called = Cell::new(false);
let result = recover_process_action(
identity,
|| Ok("reconciled"),
|_| {
action_called.set(true);
Ok("acted")
},
)
.unwrap();
assert_eq!(result, RecoveryActionResult::BootReconciled("reconciled"));
assert!(!action_called.get());
} }
#[test] #[test]

View file

@ -7,5 +7,7 @@ mod transport;
pub use layer::{ pub use layer::{
IdMap, IdMapRange, PreparationLimits, PreparedDestination, PreparedDestinationKind, IdMap, IdMapRange, PreparationLimits, PreparedDestination, PreparedDestinationKind,
}; };
pub use store::{PrepareRecovery, PrepareRequest, PreparedLowerStore, ReclaimOutcome}; pub use store::{
PrepareRecovery, PrepareRequest, PreparedArtifactGcResult, PreparedLowerStore, ReclaimOutcome,
};
pub use transport::{OciRegistryTransport, Platform, RegistryCredentials, TransportedImage}; pub use transport::{OciRegistryTransport, Platform, RegistryCredentials, TransportedImage};

View file

@ -55,6 +55,12 @@ pub enum ReclaimOutcome {
ReplacementPreserved, ReplacementPreserved,
} }
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PreparedArtifactGcResult {
pub prepared_artifact_id: PreparedArtifactId,
pub outcome: ReclaimOutcome,
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct PreparedLowerStore { pub struct PreparedLowerStore {
root: Arc<File>, root: Arc<File>,
@ -270,6 +276,55 @@ impl PreparedLowerStore {
} }
} }
/// Recovers the complete catalog snapshot of interrupted preparation operations.
///
/// # Errors
/// Stops at the first foreign, mismatched, unreadable, or changed operation without treating
/// it as owned absence.
pub fn recover_preparations(&self) -> Result<Vec<(PrepareOperationId, PrepareRecovery)>> {
let operations = self.catalog.snapshot()?.prepare_operations;
operations
.iter()
.map(|operation| {
self.recover_prepare(operation)
.map(|outcome| (operation.operation_id, outcome))
})
.collect()
}
/// Reclaims only the caller-selected exact catalog artifacts.
///
/// # Errors
/// Refuses an unknown target, duplicate target, any lease, catalog drift, or physical
/// ownership mismatch. Reclaiming rows resume their existing exact operation.
pub fn gc_artifacts(
&self,
selected: &[PreparedArtifactId],
) -> Result<Vec<PreparedArtifactGcResult>> {
let mut unique = std::collections::HashSet::new();
if selected.iter().any(|id| !unique.insert(*id)) {
bail!("prepared-artifact GC target is duplicated");
}
let snapshot = self.catalog.snapshot()?;
let mut results = Vec::with_capacity(selected.len());
for id in selected {
let artifact = snapshot
.prepared_artifacts
.iter()
.find(|artifact| artifact.prepared_artifact_id == *id)
.context("prepared-artifact GC target is absent")?;
let outcome = match artifact.state {
ArtifactState::Available => self.reclaim(artifact)?,
ArtifactState::Reclaiming => self.resume_reclaim(artifact)?,
};
results.push(PreparedArtifactGcResult {
prepared_artifact_id: *id,
outcome,
});
}
Ok(results)
}
/// Moves an exact zero-lease artifact to quarantine, deletes it, and closes reclaim intent. /// Moves an exact zero-lease artifact to quarantine, deletes it, and closes reclaim intent.
/// ///
/// # Errors /// # Errors

View file

@ -1135,6 +1135,43 @@ fn reclaim_refuses_forged_identity_and_leases() {
assert!(fixture.artifact_root(&artifact).exists()); assert!(fixture.artifact_root(&artifact).exists());
} }
#[test]
fn selected_gc_is_idempotent_only_at_the_durable_reclaim_boundary() {
let fixture = Fixture::new();
let artifact = fixture
.store
.prepare(&fixture.request(
71,
&[(
layer(&[Entry {
path: "payload",
kind: EntryType::Regular,
data: b"gc",
target: None,
mode: 0o644,
}]),
OCI_TAR,
)],
))
.unwrap();
let id = artifact.prepared_artifact_id;
assert!(fixture.store.gc_artifacts(&[id, id]).is_err());
let lease = lazaret_core::records::ArtifactLease {
prepared_artifact_id: id,
domain_id: lazaret_core::identity::DomainId::new([72; 32]),
generation_id: lazaret_core::identity::GenerationId::new(1),
};
fixture.catalog.acquire_lease(&lease).unwrap();
assert!(fixture.store.gc_artifacts(&[id]).is_err());
fixture.catalog.release_lease(&lease).unwrap();
let result = fixture.store.gc_artifacts(&[id]).unwrap();
assert_eq!(result[0].prepared_artifact_id, id);
assert_eq!(result[0].outcome, ReclaimOutcome::Removed);
assert!(fixture.store.gc_artifacts(&[id]).is_err());
}
#[test] #[test]
fn destination_layout_is_materialized_and_canonical_in_artifact_identity() { fn destination_layout_is_materialized_and_canonical_in_artifact_identity() {
let fixture = Fixture::new(); let fixture = Fixture::new();

View file

@ -6,28 +6,35 @@ use std::fs::File;
use std::io::{self, Write}; use std::io::{self, Write};
use std::os::fd::{AsFd, AsRawFd, OwnedFd}; use std::os::fd::{AsFd, AsRawFd, OwnedFd};
use std::os::unix::ffi::{OsStrExt, OsStringExt}; use std::os::unix::ffi::{OsStrExt, OsStringExt};
use std::path::{Path, PathBuf}; use std::path::{Component, Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::thread; use std::thread;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration}; use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration};
use lazaret_core::identity::{HostPath, MountIdentity, PreparedArtifactId}; use lazaret_core::identity::{
DeviceId, HostPath, InodeId, MountIdentity, NamespaceIdentity, PreparedArtifactId,
};
use lazaret_core::records::{ use lazaret_core::records::{
HostProcessOwnershipSlot, PersistedHostProcessOwnership, PreparedArtifact, HostProcessOwnershipSlot, PersistedHostProcessOwnership, PreparedArtifact,
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership, ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
}; };
use lazaret_linux::filesystem::mount_identity; use lazaret_linux::filesystem::{mount_identity, namespace_identity};
use lazaret_linux::ipc::{
private_seqpacket_pair, receive_descriptors, receive_packet, send_descriptors, send_packet,
};
use lazaret_linux::process::{ use lazaret_linux::process::{
ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, require_pidfs_magic,
}; };
use rustix::fs::{ use rustix::fs::{
AtFlags, FileType, Mode, OFlags, ResolveFlags, mkdirat, openat2, renameat, unlinkat, AtFlags, FileType, Mode, OFlags, ResolveFlags, mkdirat, openat2, renameat, unlinkat,
}; };
use rustix::io::dup; use rustix::io::dup;
use rustix::thread::{ThreadNameSpaceType, move_into_thread_name_spaces};
const MOUNT_TIMEOUT: Duration = Duration::from_secs(5); const MOUNT_TIMEOUT: Duration = Duration::from_secs(5);
const MOUNT_POLL_INTERVAL: Duration = Duration::from_millis(2); const MOUNT_POLL_INTERVAL: Duration = Duration::from_millis(2);
const PRIVATE_RECOVERY_ROLE: &str = "--lazaret-private-state-anchor-recovery";
static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0); static TEMPORARY_SEQUENCE: AtomicU64 = AtomicU64::new(0);
pub const PROTECTED_RUNTIME_FILES: [&str; 3] = ["resolv.conf", "hosts", "hostname"]; pub const PROTECTED_RUNTIME_FILES: [&str; 3] = ["resolv.conf", "hosts", "hostname"];
@ -181,6 +188,12 @@ impl<'configuration> StateAnchorLayout<'configuration> {
&self.merged &self.merged
} }
/// Returns the ordinary host path used as the merged mount label.
#[must_use]
pub fn merged_label(&self) -> &Path {
&self.merged_label
}
#[must_use] #[must_use]
pub fn runtime_files(&self) -> &Path { pub fn runtime_files(&self) -> &Path {
&self.runtime_files &self.runtime_files
@ -1007,6 +1020,259 @@ pub fn unmount_after_coordinator_loss(
Ok(()) Ok(())
} }
/// Enters an exact live holder's private namespaces and unmounts only its recorded State Anchor.
///
/// # Errors
/// Refuses descriptor transfer, pidfs, namespace, path, mount, helper, or acknowledgment drift.
pub fn recover_private_state_anchor_mount(
layout: &StateAnchorLayout<'_>,
holder: &RetainedProcess,
user_namespace: NamespaceIdentity,
mount_namespace: NamespaceIdentity,
expected_mount: MountIdentity,
) -> Result<MountIdentity, StateAnchorError> {
let merged = &layout.merged_label;
let (coordinator, helper) = private_seqpacket_pair()?;
send_descriptors(
&coordinator,
&[holder.as_fd(), layout.runtime_anchor.as_fd()],
)?;
let authority = holder.identity().authority();
let target = TargetCommand::new("/proc/self/exe")
.argument(PRIVATE_RECOVERY_ROLE)
.argument(authority.pidfs_st_dev.get().to_string())
.argument(authority.pidfs_st_ino.get().to_string())
.argument(user_namespace.device.get().to_string())
.argument(user_namespace.inode.get().to_string())
.argument(mount_namespace.device.get().to_string())
.argument(mount_namespace.inode.get().to_string())
.argument(expected_mount.mount_id.get().to_string())
.argument(expected_mount.device.get().to_string())
.argument(expected_mount.inode.get().to_string())
.argument(merged.as_os_str());
child_broker().run_transient_with_private_stdout(&target, helper)?;
let observed: MountIdentity = receive_packet(coordinator)?;
if observed == expected_mount {
Err(StateAnchorError::OwnershipMismatch)
} else {
Ok(observed)
}
}
/// Dispatches the bounded private-namespace State Anchor recovery helper.
#[must_use]
pub fn dispatch_private_state_anchor_recovery(arguments: &[OsString]) -> Option<io::Result<()>> {
if arguments
.first()
.is_none_or(|arg| arg != PRIVATE_RECOVERY_ROLE)
{
return None;
}
Some(run_private_state_anchor_recovery(arguments))
}
fn run_private_state_anchor_recovery(arguments: &[OsString]) -> io::Result<()> {
if arguments.len() != 11 {
return Err(io::Error::other(
"private State Anchor recovery arguments are malformed",
));
}
let values = arguments[1..10]
.iter()
.map(|value| {
value
.to_str()
.ok_or_else(|| io::Error::other("private recovery integer is not UTF-8"))?
.parse::<u64>()
.map_err(|_| io::Error::other("private recovery integer is invalid"))
})
.collect::<io::Result<Vec<_>>>()?;
let expected_pidfs = (values[0], values[1]);
let expected_user = NamespaceIdentity {
device: DeviceId::new(values[2]),
inode: InodeId::new(values[3]),
};
let expected_mount_namespace = NamespaceIdentity {
device: DeviceId::new(values[4]),
inode: InodeId::new(values[5]),
};
let expected_mount = MountIdentity {
mount_id: lazaret_core::identity::MountId::new(values[6]),
device: DeviceId::new(values[7]),
inode: InodeId::new(values[8]),
};
let merged = PathBuf::from(&arguments[10]);
let stdout = io::stdout();
let mut descriptors = receive_descriptors(&stdout, 2)?;
let runtime_anchor = descriptors
.pop()
.ok_or_else(|| io::Error::other("runtime anchor descriptor is absent"))?;
let pidfd = descriptors
.pop()
.ok_or_else(|| io::Error::other("pidfd descriptor is absent"))?;
let mut operations = KernelPrivateRecovery {
pidfd,
runtime_anchor,
expected_pidfs,
merged,
reopened_parent: None,
};
let observed = execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
)?;
send_packet(stdout, &observed)
}
trait PrivateRecoveryOperations {
fn validate_pidfs(&mut self) -> io::Result<()>;
fn enter_user_namespace(&mut self) -> io::Result<()>;
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity>;
fn enter_mount_namespace(&mut self) -> io::Result<()>;
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity>;
fn verify_mount(&mut self) -> io::Result<MountIdentity>;
fn unmount(&mut self) -> io::Result<()>;
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity>;
}
fn execute_private_recovery(
operations: &mut impl PrivateRecoveryOperations,
expected_user: NamespaceIdentity,
expected_mount_namespace: NamespaceIdentity,
expected_mount: MountIdentity,
) -> io::Result<MountIdentity> {
operations
.validate_pidfs()
.map_err(|error| io::Error::other(format!("validate pidfs: {error}")))?;
operations
.enter_user_namespace()
.map_err(|error| io::Error::other(format!("enter user namespace: {error}")))?;
if operations
.observe_user_namespace()
.map_err(|error| io::Error::other(format!("observe user namespace: {error}")))?
!= expected_user
{
return Err(io::Error::other("private recovery user namespace mismatch"));
}
operations
.enter_mount_namespace()
.map_err(|error| io::Error::other(format!("enter mount namespace: {error}")))?;
if operations
.observe_mount_namespace()
.map_err(|error| io::Error::other(format!("observe mount namespace: {error}")))?
!= expected_mount_namespace
{
return Err(io::Error::other(
"private recovery mount namespace mismatch",
));
}
if operations
.verify_mount()
.map_err(|error| io::Error::other(format!("verify mount: {error}")))?
!= expected_mount
{
return Err(io::Error::other("private recovery mount identity mismatch"));
}
operations
.unmount()
.map_err(|error| io::Error::other(format!("unmount: {error}")))?;
let observed = operations
.observe_after_unmount()
.map_err(|error| io::Error::other(format!("observe after unmount: {error}")))?;
if observed == expected_mount {
Err(io::Error::other("private recovery mount remains present"))
} else {
Ok(observed)
}
}
struct KernelPrivateRecovery {
pidfd: OwnedFd,
runtime_anchor: OwnedFd,
expected_pidfs: (u64, u64),
merged: PathBuf,
reopened_parent: Option<OwnedFd>,
}
impl PrivateRecoveryOperations for KernelPrivateRecovery {
fn validate_pidfs(&mut self) -> io::Result<()> {
require_pidfs_magic(rustix::fs::fstatfs(&self.pidfd)?.f_type as i64)?;
let stat = rustix::fs::fstat(&self.pidfd)?;
if (stat.st_dev, stat.st_ino) == self.expected_pidfs {
Ok(())
} else {
Err(io::Error::other(
"private recovery pidfs authority mismatch",
))
}
}
fn enter_user_namespace(&mut self) -> io::Result<()> {
move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::USER)?;
Ok(())
}
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
namespace_identity(Path::new("/proc/self/ns/user"))
}
fn enter_mount_namespace(&mut self) -> io::Result<()> {
move_into_thread_name_spaces(self.pidfd.as_fd(), ThreadNameSpaceType::MOUNT)?;
Ok(())
}
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
namespace_identity(Path::new("/proc/self/ns/mnt"))
}
fn verify_mount(&mut self) -> io::Result<MountIdentity> {
let parent_path = self
.merged
.parent()
.ok_or_else(|| io::Error::other("private recovery merged path has no parent"))?;
let reopened_parent = openat2(
rustix::fs::CWD,
parent_path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
)?;
let retained = rustix::fs::fstat(&self.runtime_anchor)?;
let reopened = rustix::fs::fstat(&reopened_parent)?;
if (retained.st_dev, retained.st_ino) != (reopened.st_dev, reopened.st_ino) {
return Err(io::Error::other("private recovery runtime anchor mismatch"));
}
let mounted = open_final_mount_directory(&reopened_parent, self.merged_name()?)?;
let identity = mount_identity(&mounted)?;
self.reopened_parent = Some(reopened_parent);
Ok(identity)
}
fn unmount(&mut self) -> io::Result<()> {
rustix::mount::unmount(&self.merged, rustix::mount::UnmountFlags::empty())?;
Ok(())
}
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
let parent = self
.reopened_parent
.as_ref()
.ok_or_else(|| io::Error::other("private recovery parent was not verified"))?;
mount_identity(&open_final_mount_directory(parent, self.merged_name()?)?)
}
}
impl KernelPrivateRecovery {
fn merged_name(&self) -> io::Result<&Path> {
self.merged
.file_name()
.map(Path::new)
.ok_or_else(|| io::Error::other("private recovery merged path has no name"))
}
}
/// Independently verifies that no mount is attached at the exact merged path. /// Independently verifies that no mount is attached at the exact merged path.
/// ///
/// # Errors /// # Errors
@ -1015,6 +1281,22 @@ pub fn require_mount_absent(layout: &StateAnchorLayout<'_>) -> Result<(), StateA
layout.require_unmounted() layout.require_unmounted()
} }
/// Freshly observes absence of one exact retired mount in the caller's current namespace.
///
/// # Errors
/// Refuses unreadable layout evidence or continued presence of the exact mount.
pub fn observe_mount_absence(
layout: &StateAnchorLayout<'_>,
expected: MountIdentity,
) -> Result<MountIdentity, StateAnchorError> {
let observed = layout.current_merged_identity()?;
if observed == expected {
Err(StateAnchorError::OwnershipMismatch)
} else {
Ok(observed)
}
}
fn create_directory(parent: impl AsFd, name: &str) -> Result<(), StateAnchorError> { fn create_directory(parent: impl AsFd, name: &str) -> Result<(), StateAnchorError> {
match mkdirat(parent.as_fd(), name, Mode::from_raw_mode(0o700)) { match mkdirat(parent.as_fd(), name, Mode::from_raw_mode(0o700)) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {} Ok(()) | Err(rustix::io::Errno::EXIST) => {}
@ -1045,6 +1327,16 @@ fn open_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchor
Ok(descriptor) Ok(descriptor)
} }
fn open_final_mount_directory(parent: impl AsFd, path: &Path) -> io::Result<OwnedFd> {
let mut components = path.components();
if !matches!(components.next(), Some(Component::Normal(_))) || components.next().is_some() {
return Err(io::Error::other(
"final mount path is not one normal component",
));
}
open_mounted_directory(parent, path).map_err(io::Error::other)
}
fn open_mounted_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchorError> { fn open_mounted_directory(parent: impl AsFd, path: &Path) -> Result<OwnedFd, StateAnchorError> {
let descriptor = openat2( let descriptor = openat2(
parent.as_fd(), parent.as_fd(),
@ -1331,3 +1623,200 @@ fn path_from_host(path: &HostPath) -> PathBuf {
fn host_path(path: &Path) -> HostPath { fn host_path(path: &Path) -> HostPath {
HostPath::new(path.as_os_str().as_bytes().to_vec()) HostPath::new(path.as_os_str().as_bytes().to_vec())
} }
#[cfg(test)]
mod recovery_tests {
use super::*;
use std::os::unix::fs::symlink;
struct FailpointRecovery {
fail_at: Option<usize>,
boundary: usize,
events: Vec<&'static str>,
user: NamespaceIdentity,
mount_namespace: NamespaceIdentity,
mount: MountIdentity,
observed: MountIdentity,
}
impl FailpointRecovery {
fn boundary(&mut self, name: &'static str) -> io::Result<()> {
self.boundary += 1;
self.events.push(name);
if self.fail_at == Some(self.boundary) {
Err(io::Error::other("injected recovery boundary failure"))
} else {
Ok(())
}
}
}
impl PrivateRecoveryOperations for FailpointRecovery {
fn validate_pidfs(&mut self) -> io::Result<()> {
self.boundary("pidfs")
}
fn enter_user_namespace(&mut self) -> io::Result<()> {
self.boundary("enter-user")
}
fn observe_user_namespace(&mut self) -> io::Result<NamespaceIdentity> {
self.boundary("observe-user")?;
Ok(self.user)
}
fn enter_mount_namespace(&mut self) -> io::Result<()> {
self.boundary("enter-mount")
}
fn observe_mount_namespace(&mut self) -> io::Result<NamespaceIdentity> {
self.boundary("observe-mount")?;
Ok(self.mount_namespace)
}
fn verify_mount(&mut self) -> io::Result<MountIdentity> {
self.boundary("verify-mount")?;
Ok(self.mount)
}
fn unmount(&mut self) -> io::Result<()> {
self.boundary("unmount")
}
fn observe_after_unmount(&mut self) -> io::Result<MountIdentity> {
self.boundary("observe-absence")?;
Ok(self.observed)
}
}
fn fixture(fail_at: Option<usize>) -> FailpointRecovery {
FailpointRecovery {
fail_at,
boundary: 0,
events: Vec::new(),
user: NamespaceIdentity {
device: DeviceId::new(1),
inode: InodeId::new(2),
},
mount_namespace: NamespaceIdentity {
device: DeviceId::new(3),
inode: InodeId::new(4),
},
mount: MountIdentity {
mount_id: lazaret_core::identity::MountId::new(5),
device: DeviceId::new(6),
inode: InodeId::new(7),
},
observed: MountIdentity {
mount_id: lazaret_core::identity::MountId::new(8),
device: DeviceId::new(6),
inode: InodeId::new(7),
},
}
}
#[test]
fn private_recovery_failpoints_refuse_before_every_later_boundary() {
for boundary in 1..=8 {
let mut operations = fixture(Some(boundary));
let expected_user = operations.user;
let expected_mount_namespace = operations.mount_namespace;
let expected_mount = operations.mount;
let result = execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
);
assert!(result.is_err());
assert_eq!(operations.events.len(), boundary);
}
let mut operations = fixture(None);
let expected_user = operations.user;
let expected_mount_namespace = operations.mount_namespace;
let expected_mount = operations.mount;
assert_eq!(
execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
)
.unwrap(),
operations.observed
);
assert_eq!(
operations.events,
[
"pidfs",
"enter-user",
"observe-user",
"enter-mount",
"observe-mount",
"verify-mount",
"unmount",
"observe-absence",
]
);
}
#[test]
fn final_mount_open_refuses_intermediate_and_link_traversal() {
let root = std::env::temp_dir().join(format!(
"lazaret-final-mount-open-{}-{}",
std::process::id(),
TEMPORARY_SEQUENCE.fetch_add(1, Ordering::Relaxed)
));
std::fs::create_dir_all(root.join("nested/final")).unwrap();
symlink("nested", root.join("link")).unwrap();
let parent = File::open(&root).unwrap();
let descriptor = File::open(root.join("nested")).unwrap();
symlink(
format!("/proc/self/fd/{}", descriptor.as_raw_fd()),
root.join("magic"),
)
.unwrap();
assert!(open_final_mount_directory(&parent, Path::new("nested/final")).is_err());
assert!(open_final_mount_directory(&parent, Path::new("link")).is_err());
assert!(open_final_mount_directory(&parent, Path::new("magic")).is_err());
assert!(open_final_mount_directory(&parent, Path::new("nested")).is_ok());
std::fs::remove_dir_all(root).unwrap();
}
#[test]
fn private_recovery_refuses_wrong_or_absent_final_mount_before_action() {
for observed_mount_id in [4, 8] {
let mut operations = fixture(None);
operations.mount.mount_id = lazaret_core::identity::MountId::new(observed_mount_id);
let expected_user = operations.user;
let expected_mount_namespace = operations.mount_namespace;
let expected_mount = MountIdentity {
mount_id: lazaret_core::identity::MountId::new(5),
device: DeviceId::new(6),
inode: InodeId::new(7),
};
assert!(
execute_private_recovery(
&mut operations,
expected_user,
expected_mount_namespace,
expected_mount,
)
.is_err()
);
assert_eq!(
operations.events,
[
"pidfs",
"enter-user",
"observe-user",
"enter-mount",
"observe-mount",
"verify-mount",
]
);
}
}
}

View file

@ -1,12 +1,13 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::ffi::OsString; use std::ffi::OsString;
use std::fs; use std::fs;
use std::io; use std::io::{self, BufRead, BufReader, Read, Write};
use std::os::unix::ffi::OsStrExt; use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::MetadataExt; use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::ExitCode; use std::process::ExitCode;
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use lazaret_core::configuration::{ use lazaret_core::configuration::{
ConfigurationLoadRequest, SandboxLayoutV1, load_trusted_configuration, ConfigurationLoadRequest, SandboxLayoutV1, load_trusted_configuration,
@ -16,12 +17,14 @@ use lazaret_core::records::{
ArtifactState, HostProcessOwnershipSlot, PreparedArtifact, ProcessOwnershipCommitAck, ArtifactState, HostProcessOwnershipSlot, PreparedArtifact, ProcessOwnershipCommitAck,
}; };
use lazaret_linux::process::{ use lazaret_linux::process::{
ProcessOwnershipContext, current_process_identity, dispatch_private_process_role, ProcessOwnershipContext, TargetCommand, child_broker, current_process_identity,
dispatch_private_process_role,
}; };
use lazaret_state_anchor::{ use lazaret_state_anchor::{
FuseOverlayfsMount, ProtectedRuntimeFiles, RuntimeStateAnchorAuthority, FuseOverlayfsMount, ProtectedRuntimeFiles, RuntimeStateAnchorAuthority,
RuntimeStateAnchorInherited, StateAnchorAcquisition, StateAnchorLaunchAuthority, RuntimeStateAnchorInherited, StateAnchorAcquisition, StateAnchorLaunchAuthority,
StateAnchorLayout, StateAnchorSandboxView, unmount_after_coordinator_loss, StateAnchorLayout, StateAnchorSandboxView, dispatch_private_state_anchor_recovery,
recover_private_state_anchor_mount, unmount_after_coordinator_loss,
}; };
use rustix::process::Signal; use rustix::process::Signal;
@ -29,6 +32,15 @@ static SEQUENCE: AtomicU64 = AtomicU64::new(0);
fn main() -> ExitCode { fn main() -> ExitCode {
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>(); let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
if arguments
.first()
.is_some_and(|value| value == "--private-recovery-holder")
{
return exit(run_private_recovery_holder(&arguments));
}
if let Some(result) = dispatch_private_state_anchor_recovery(&arguments) {
return exit(result);
}
if let Some(result) = dispatch_private_process_role(&arguments) { if let Some(result) = dispatch_private_process_role(&arguments) {
return exit(result); return exit(result);
} }
@ -55,8 +67,9 @@ fn run() -> io::Result<()> {
} }
Err(error) => return Err(error), Err(error) => return Err(error),
} }
let tree = TestTree::new()?; let tree = TestTree::new().map_err(|error| io::Error::other(format!("test tree: {error}")))?;
let configuration = configuration(&tree)?; let configuration = configuration(&tree)
.map_err(|error| io::Error::other(format!("configuration: {error}")))?;
let lower = tree.path.join("prepared-lower"); let lower = tree.path.join("prepared-lower");
fs::create_dir_all(lower.join("usr/bin"))?; fs::create_dir_all(lower.join("usr/bin"))?;
fs::create_dir_all(lower.join("etc"))?; fs::create_dir_all(lower.join("etc"))?;
@ -76,7 +89,10 @@ fn run() -> io::Result<()> {
inode: lazaret_core::identity::InodeId::new(metadata.ino()), inode: lazaret_core::identity::InodeId::new(metadata.ino()),
state: ArtifactState::Available, state: ArtifactState::Available,
}; };
let layout = StateAnchorLayout::create(&configuration).map_err(io::Error::other)?; let layout = StateAnchorLayout::create(&configuration)
.map_err(|error| io::Error::other(format!("State Anchor layout: {error}")))?;
exercise_private_recovery(&layout)
.map_err(|error| io::Error::other(format!("private recovery exercise: {error}")))?;
let launch = let launch =
StateAnchorLaunchAuthority::capture(&layout, &artifact).map_err(io::Error::other)?; StateAnchorLaunchAuthority::capture(&layout, &artifact).map_err(io::Error::other)?;
@ -197,6 +213,139 @@ fn run() -> io::Result<()> {
Ok(()) Ok(())
} }
#[allow(deprecated)]
fn run_private_recovery_holder(arguments: &[OsString]) -> io::Result<()> {
if arguments.len() != 2 {
return Err(io::Error::other("private holder arguments are malformed"));
}
let merged = PathBuf::from(&arguments[1]);
let uid = rustix::process::getuid().as_raw();
let gid = rustix::process::getgid().as_raw();
rustix::thread::unshare(rustix::thread::UnshareFlags::NEWUSER)
.map_err(|error| io::Error::other(format!("holder user namespace: {error}")))?;
fs::write("/proc/self/setgroups", "deny")
.map_err(|error| io::Error::other(format!("holder setgroups: {error}")))?;
fs::write("/proc/self/uid_map", format!("0 {uid} 1\n"))
.map_err(|error| io::Error::other(format!("holder uid map: {error}")))?;
fs::write("/proc/self/gid_map", format!("0 {gid} 1\n"))
.map_err(|error| io::Error::other(format!("holder gid map: {error}")))?;
rustix::thread::unshare(rustix::thread::UnshareFlags::NEWNS)
.map_err(|error| io::Error::other(format!("holder mount namespace: {error}")))?;
rustix::mount::mount_change(
"/",
rustix::mount::MountPropagationFlags::PRIVATE | rustix::mount::MountPropagationFlags::REC,
)
.map_err(|error| io::Error::other(format!("holder private propagation: {error}")))?;
rustix::mount::mount(
"tmpfs",
&merged,
"tmpfs",
rustix::mount::MountFlags::empty(),
c"size=1m",
)
.map_err(|error| io::Error::other(format!("holder synthetic mount: {error}")))?;
let user = lazaret_linux::filesystem::namespace_identity(Path::new("/proc/self/ns/user"))?;
let mount_namespace =
lazaret_linux::filesystem::namespace_identity(Path::new("/proc/self/ns/mnt"))?;
let mount = lazaret_linux::filesystem::current_mount_identity(&merged)?;
writeln!(
io::stdout(),
"{} {} {} {} {} {} {}",
user.device.get(),
user.inode.get(),
mount_namespace.device.get(),
mount_namespace.inode.get(),
mount.mount_id.get(),
mount.device.get(),
mount.inode.get(),
)?;
io::stdout().flush()?;
loop {
std::thread::park();
}
}
fn exercise_private_recovery(layout: &StateAnchorLayout<'_>) -> io::Result<()> {
let target = TargetCommand::new("/proc/self/exe")
.argument("--private-recovery-holder")
.argument(layout.merged_label());
let mut captured = child_broker()
.run_transient_captured(&target)
.map_err(|error| io::Error::other(format!("starting private holder: {error}")))?;
let values = {
let (stdout, stderr) = captured.streams();
let mut reader = BufReader::new(stdout);
let mut line = String::new();
let deadline = Instant::now() + Duration::from_secs(5);
loop {
match reader.read_line(&mut line) {
Ok(0) if Instant::now() < deadline => std::thread::yield_now(),
Ok(0) => {
let mut detail = String::new();
match stderr.read_to_string(&mut detail) {
Ok(_) => {}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {}
Err(error) => return Err(error),
}
return Err(io::Error::other(format!(
"private holder closed without evidence: {}",
detail.trim()
)));
}
Ok(_) => break,
Err(error)
if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline =>
{
std::thread::yield_now();
}
Err(error) => return Err(error),
}
}
line.split_whitespace()
.map(|value| {
value
.parse::<u64>()
.map_err(|_| io::Error::other("private holder output is malformed"))
})
.collect::<io::Result<Vec<_>>>()?
};
if values.len() != 7 {
return Err(io::Error::other("private holder evidence is incomplete"));
}
let holder = captured.into_process();
let user = lazaret_core::identity::NamespaceIdentity {
device: lazaret_core::identity::DeviceId::new(values[0]),
inode: lazaret_core::identity::InodeId::new(values[1]),
};
let mount_namespace = lazaret_core::identity::NamespaceIdentity {
device: lazaret_core::identity::DeviceId::new(values[2]),
inode: lazaret_core::identity::InodeId::new(values[3]),
};
let mount = lazaret_core::identity::MountIdentity {
mount_id: lazaret_core::identity::MountId::new(values[4]),
device: lazaret_core::identity::DeviceId::new(values[5]),
inode: lazaret_core::identity::InodeId::new(values[6]),
};
let wrong = lazaret_core::identity::MountIdentity {
mount_id: lazaret_core::identity::MountId::new(mount.mount_id.get() + 1),
..mount
};
if recover_private_state_anchor_mount(layout, &holder, user, mount_namespace, wrong).is_ok() {
return Err(io::Error::other("private recovery accepted wrong mount"));
}
let observed =
recover_private_state_anchor_mount(layout, &holder, user, mount_namespace, mount)
.map_err(io::Error::other)?;
if observed == mount {
return Err(io::Error::other("private recovery did not prove absence"));
}
holder.send_signal(Signal::KILL)?;
while !holder.has_exited()? {
std::thread::yield_now();
}
child_broker().reap(holder)
}
fn acquire<'layout, 'configuration>( fn acquire<'layout, 'configuration>(
layout: &'layout StateAnchorLayout<'configuration>, layout: &'layout StateAnchorLayout<'configuration>,
artifact: &PreparedArtifact, artifact: &PreparedArtifact,

View file

@ -1001,6 +1001,15 @@ classDiagram
optional path optional path
optional device optional device
optional inode optional inode
optional released_state_anchor
}
class StateAnchorReleaseEvidence {
<<serialized>>
daemon
user_namespace
mount_namespace
mount
} }
class StateAnchorOwnership { class StateAnchorOwnership {
@ -1055,6 +1064,11 @@ classDiagram
} }
ParticipantOwnership --> PreparedLowerOwnership ParticipantOwnership --> PreparedLowerOwnership
PreparedLowerOwnership --> StateAnchorReleaseEvidence : released_state_anchor
StateAnchorReleaseEvidence --> PersistedHostProcessOwnership : daemon
StateAnchorReleaseEvidence --> NamespaceIdentity : user_namespace
StateAnchorReleaseEvidence --> NamespaceIdentity : mount_namespace
StateAnchorReleaseEvidence --> MountIdentity : mount
ParticipantOwnership --> StateAnchorOwnership ParticipantOwnership --> StateAnchorOwnership
ParticipantOwnership --> GitRoutingOwnership ParticipantOwnership --> GitRoutingOwnership
ParticipantOwnership --> NetworkOwnership ParticipantOwnership --> NetworkOwnership
@ -1103,11 +1117,13 @@ and newly acquired child slots move only `Unassigned` to `Staged` to exact
matching current `Committed`. A current committed owner or supervisor may matching current `Committed`. A current committed owner or supervisor may
enter a newly created dependent-role snapshot directly and is then immutable. enter a newly created dependent-role snapshot directly and is then immutable.
Recorded evidence cannot be replaced or cleared. A transition to ReleaseIntent Recorded evidence cannot be replaced or cleared. A transition to ReleaseIntent
preserves all evidence; PreparedLower preserves all evidence. PreparedLower changes its lease phase to
changes only its lease phase to `ReleaseIntent`. ReleaseIntent becomes Clean `ReleaseIntent` and may atomically add the exact released State Anchor daemon,
only after independent absence proof and atomically clears every physical user namespace, mount namespace, and mount evidence needed to resume physical
field. A bare Intent may become Clean only after the same sealed runtime-absence release. That release evidence is immutable within ReleaseIntent. ReleaseIntent
check. becomes Clean only after independent absence proof and atomically clears every
physical field. A bare Intent may become Clean only after the same sealed
runtime-absence check.
The direct existing-process edges are Network RS, Helper RS, Sandbox HC, and The direct existing-process edges are Network RS, Helper RS, Sandbox HC, and
Session Terminal HC. State Anchor daemon, Git-routing daemon, Helper process, Session Terminal HC. State Anchor daemon, Git-routing daemon, Helper process,

View file

@ -159,6 +159,11 @@ PROCESS_LINT_ALLOWANCES = {
"append_parent_identity", "append_parent_identity",
"exec_target", "exec_target",
"run_transient", "run_transient",
"run_transient_with_stdout",
"run_transient_captured",
"spawn_bootstrap",
"captured_process",
"match_recovery_process",
}, },
"crates/lazaret-host-coordinator/tests/runtime-composition.rs": {"run_benchmark_guest"}, "crates/lazaret-host-coordinator/tests/runtime-composition.rs": {"run_benchmark_guest"},
"crates/lazaret-linux/tests/acquisition.rs": { "crates/lazaret-linux/tests/acquisition.rs": {
@ -176,7 +181,9 @@ PROCESS_LINT_ALLOWANCES = {
"concurrent_acquisitions_serialize", "concurrent_acquisitions_serialize",
}, },
"crates/lazaret-linux/tests/capability-drop-helper.rs": {"main"}, "crates/lazaret-linux/tests/capability-drop-helper.rs": {"main"},
"crates/lazaret-state-anchor/src/lib.rs": {"rootless_unmount"}, "crates/lazaret-prepared-lower/tests/prepared_lower.rs": {
"retained_fixed_child_bind_mount_is_refused_when_mount_setup_is_available"
},
} }
FORBIDDEN_PROCESS_SYSCALLS = tuple( FORBIDDEN_PROCESS_SYSCALLS = tuple(
re.compile(pattern) re.compile(pattern)