ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Implement exact bootstrap recovery and artifact GC

Browse source
This commit is contained in:
ms2k 2026-07-17 05:38:43 +09:00
commit d25942596f
5 changed files with 1631 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/lazaret-host-coordinator/src/lib.rs
View file

@ -9,6 +9,7 @@ pub mod lifecycle;
pub mod network;
pub mod production;
pub mod production_selector;
pub mod recovery;
pub mod runtime;
pub mod sandbox_init;
pub mod state_anchor;

7
crates/lazaret-host-coordinator/src/main.rs
View file

@ -18,6 +18,13 @@ fn main() {
}
return;
}
if let Some(result) = lazaret_state_anchor::dispatch_private_state_anchor_recovery(&arguments) {
if let Err(error) = result {
eprintln!("State Anchor recovery failed: {error}");
std::process::exit(1);
}
return;
}
if let Some(result) = lazaret_linux::process::dispatch_private_process_role(&arguments) {
if let Err(error) = result {
eprintln!("private process role failed: {error}");

297
crates/lazaret-host-coordinator/src/recovery.rs Normal file
View file

@ -0,0 +1,297 @@
use lazaret_core::identity::{
GenerationId, MountIdentity, NamespaceIdentity, PidfsProcessIdentityV1,
};
use lazaret_core::records::{
Participant, ParticipantOwnership, ParticipantRole, PersistedHostProcessOwnership,
};
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use crate::state_anchor::RuntimeOwnersAbsent;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct RuntimeRecoveryOwnership {
pub host_coordinator: Option<PidfsProcessIdentityV1>,
pub runtime_supervisor: Option<PidfsProcessIdentityV1>,
pub attached_runsc: Option<PidfsProcessIdentityV1>,
pub connectivity_helper: Option<PidfsProcessIdentityV1>,
pub state_anchor_daemon: Option<PidfsProcessIdentityV1>,
pub git_routing_daemon: Option<PidfsProcessIdentityV1>,
pub state_anchor_user_namespace: Option<NamespaceIdentity>,
pub state_anchor_mount_namespace: Option<NamespaceIdentity>,
pub state_anchor_mount: Option<MountIdentity>,
}
/// Extracts one generation's exact process and private-mount recovery authority.
///
/// # Errors
/// Refuses unequal copies of the constitutive HC or RS identities, and incomplete State Anchor
/// namespace/mount evidence. Staged identities remain cleanup authority.
pub fn runtime_recovery_ownership(
participants: &[Participant],
generation_id: GenerationId,
) -> Result<RuntimeRecoveryOwnership, &'static str> {
let mut result = RuntimeRecoveryOwnership::default();
for participant in participants
.iter()
.filter(|participant| participant.generation_id() == generation_id)
{
let Some(ownership) = participant.ownership() else {
continue;
};
match ownership {
ParticipantOwnership::Sandbox(sandbox) => {
merge_process(
&mut result.host_coordinator,
&sandbox.host_coordinator,
"Sandbox HC identity mismatch",
)?;
merge_process(
&mut result.runtime_supervisor,
&sandbox.runtime_supervisor,
"Sandbox RS identity mismatch",
)?;
merge_process(
&mut result.attached_runsc,
&sandbox.attached_runsc,
"attached runsc identity mismatch",
)?;
}
ParticipantOwnership::Network(network) => merge_process(
&mut result.runtime_supervisor,
&network.runtime_supervisor,
"Network RS identity mismatch",
)?,
ParticipantOwnership::Helper(helper) => {
merge_process(
&mut result.connectivity_helper,
&helper.connectivity_helper,
"connectivity-helper identity mismatch",
)?;
merge_process(
&mut result.runtime_supervisor,
&helper.runtime_supervisor,
"Helper RS identity mismatch",
)?;
}
ParticipantOwnership::StateAnchor(anchor) => {
merge_process(
&mut result.state_anchor_daemon,
&anchor.daemon,
"State Anchor daemon identity mismatch",
)?;
let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
if evidence == (None, None, None) {
continue;
}
let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
return Err("State Anchor recovery evidence is incomplete");
};
merge_value(
&mut result.state_anchor_user_namespace,
user,
"State Anchor user namespace mismatch",
)?;
merge_value(
&mut result.state_anchor_mount_namespace,
mount_namespace,
"State Anchor mount namespace mismatch",
)?;
merge_value(
&mut result.state_anchor_mount,
mount,
"State Anchor mount identity mismatch",
)?;
}
ParticipantOwnership::GitRouting(routing) => merge_process(
&mut result.git_routing_daemon,
&routing.daemon,
"Git-routing daemon identity mismatch",
)?,
ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
}
}
Ok(result)
}
fn merge_process(
target: &mut Option<PidfsProcessIdentityV1>,
observed: &PersistedHostProcessOwnership,
mismatch: &'static str,
) -> Result<(), &'static str> {
let Ok(observed) = observed.authoritative_pidfs() else {
return Ok(());
};
merge_value(target, *observed, mismatch)
}
fn merge_value<T: Copy + Eq>(
target: &mut Option<T>,
observed: T,
mismatch: &'static str,
) -> Result<(), &'static str> {
match target {
None => *target = Some(observed),
Some(current) if *current == observed => {}
Some(_) => return Err(mismatch),
}
Ok(())
}
#[must_use]
pub fn recovery_role_order() -> [ParticipantRole; 7] {
[
ParticipantRole::Publication,
ParticipantRole::Sandbox,
ParticipantRole::Helper,
ParticipantRole::Network,
ParticipantRole::GitRouting,
ParticipantRole::StateAnchor,
ParticipantRole::PreparedLower,
]
}
/// Proves the constitutive RS and attached runsc are absent before private-mount reconciliation.
///
/// # Errors
/// Refuses a live or stopped exact authority and every unreadable or ambiguous observation.
pub fn prove_runtime_owners_absent(
generation_id: GenerationId,
ownership: &RuntimeRecoveryOwnership,
) -> Result<RuntimeOwnersAbsent, std::io::Error> {
for identity in [ownership.attached_runsc, ownership.runtime_supervisor]
.into_iter()
.flatten()
{
match match_recovery_process(identity) {
Ok(PidfsRecoveryMatch::BootMismatch) => {}
Ok(PidfsRecoveryMatch::Matched(process)) if process.has_exited()? => {}
Ok(PidfsRecoveryMatch::Matched(_)) => {
return Err(std::io::Error::other(
"constitutive runtime authority remains present",
));
}
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {}
Err(error) => return Err(error),
}
}
Ok(RuntimeOwnersAbsent::for_generation(generation_id))
}
#[cfg(test)]
mod tests {
use lazaret_core::identity::{
BootId, BundleDigest, ContainerId, DeviceId, HostPath, HostPidLocator, MountId,
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks,
};
use lazaret_core::records::{
HelperOwnership, ParticipantState, SandboxOwnership, StateAnchorOwnership,
};
use super::*;
fn process(value: u64) -> PidfsProcessIdentityV1 {
PidfsProcessIdentityV1::new(
PidfsProcessAuthorityV1 {
boot_id: BootId::new([1; 16]),
pidfs_st_dev: PidfsDeviceId::new(value),
pidfs_st_ino: PidfsInodeId::new(value),
},
HostPidLocator::new(u32::try_from(value).unwrap()),
ProcessStartTimeTicks::new(value),
)
}
fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant {
Participant::snapshot(
GenerationId::new(1),
role,
ParticipantState::Intent,
Some(ownership),
)
.unwrap()
}
#[test]
fn inventory_accepts_staged_cleanup_authority_and_rejects_cross_role_rs_mismatch() {
let rs = process(2);
let sandbox = participant(
ParticipantRole::Sandbox,
ParticipantOwnership::Sandbox(SandboxOwnership {
container_id: Some(ContainerId::new([3; 32])),
runtime_root: Some(HostPath::new(b"/runtime".to_vec())),
bundle_digest: Some(BundleDigest::new([4; 32])),
host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
runtime_supervisor: PersistedHostProcessOwnership::staged(rs),
attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
}),
);
let helper = |runtime_supervisor| {
participant(
ParticipantRole::Helper,
ParticipantOwnership::Helper(HelperOwnership {
network_sidecar_id: None,
connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
runtime_supervisor: PersistedHostProcessOwnership::committed(
runtime_supervisor,
),
}),
)
};
let inventory =
runtime_recovery_ownership(&[sandbox.clone(), helper(rs)], GenerationId::new(1))
.unwrap();
assert_eq!(inventory.runtime_supervisor, Some(rs));
assert_eq!(inventory.attached_runsc, Some(process(3)));
assert_eq!(inventory.connectivity_helper, Some(process(4)));
assert!(
runtime_recovery_ownership(&[sandbox, helper(process(99))], GenerationId::new(1),)
.is_err()
);
}
#[test]
fn inventory_requires_complete_private_mount_evidence() {
let incomplete = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: Some(HostPath::new(b"upper".to_vec())),
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: Some(NamespaceIdentity {
device: DeviceId::new(6),
inode: lazaret_core::identity::InodeId::new(7),
}),
mount_namespace: None,
mount: None,
}),
);
assert!(runtime_recovery_ownership(&[incomplete], GenerationId::new(1)).is_err());
let complete = participant(
ParticipantRole::StateAnchor,
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: Some(HostPath::new(b"upper".to_vec())),
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process(5)),
user_namespace: Some(NamespaceIdentity {
device: DeviceId::new(6),
inode: lazaret_core::identity::InodeId::new(7),
}),
mount_namespace: Some(NamespaceIdentity {
device: DeviceId::new(8),
inode: lazaret_core::identity::InodeId::new(9),
}),
mount: Some(MountIdentity {
mount_id: MountId::new(10),
device: DeviceId::new(11),
inode: lazaret_core::identity::InodeId::new(12),
}),
}),
);
assert!(runtime_recovery_ownership(&[complete], GenerationId::new(1)).is_ok());
}
}

147
crates/lazaret-host-coordinator/src/state_anchor.rs
View file

@ -10,8 +10,11 @@ use lazaret_core::records::{
ArtifactLease, HostProcessOwnershipSlot, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PersistedHostProcessOwnership, PreparedArtifact, PreparedLowerOwnership,
ProcessOwnershipCommitAck, StagedProcessOwnership, StateAnchorOwnership,
StateAnchorReleaseEvidence,
};
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
use lazaret_state_anchor::{StateAnchorError, StateAnchorLayout, VerifiedStateAnchorReadiness};
use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_mount};
use lazaret_storage_catalog::{Catalog, CatalogError};
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
@ -103,6 +106,17 @@ impl StateAnchorReleaseAuthority {
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct RuntimeOwnersAbsent {
generation_id: GenerationId,
}
impl RuntimeOwnersAbsent {
pub(crate) const fn for_generation(generation_id: GenerationId) -> Self {
Self { generation_id }
}
}
pub struct StateAnchorCoordinator<'a> {
domain_store: &'a HostCoordinatorDomainStore,
catalog: &'a Catalog,
@ -140,6 +154,7 @@ impl<'a> StateAnchorCoordinator<'a> {
path: Some(artifact.path.clone()),
device: Some(artifact.device),
inode: Some(artifact.inode),
released_state_anchor: None,
};
let expected_prepared_intent = participant(
generation_id,
@ -364,6 +379,52 @@ impl<'a> StateAnchorCoordinator<'a> {
Ok(authority)
}
/// Recovers the exact private State Anchor after all constitutive runtime owners are absent.
///
/// # Errors
/// Refuses generation drift, pidfs/namespace/mount mismatch, unreadable absence, or helper
/// failure. Boot mismatch and `ESRCH` reconcile physical absence without process action only
/// under the caller-provided ordered runtime-owner absence witness.
pub fn recover_after_runtime_owner_loss(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
owners_absent: RuntimeOwnersAbsent,
) -> Result<
(StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
StateAnchorCoordinatorError,
> {
if owners_absent.generation_id != generation_id {
return Err(StateAnchorCoordinatorError::InvalidState(
"runtime-owner absence belongs to another generation",
));
}
let authority = self.begin_release(generation_id, layout)?;
let observed = match match_recovery_process(authority.daemon) {
Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
layout,
&holder,
authority.user_namespace,
authority.mount_namespace,
authority.mount,
)?,
Ok(PidfsRecoveryMatch::BootMismatch) => observe_mount_absence(layout, authority.mount)?,
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
observe_mount_absence(layout, authority.mount)?
}
Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
};
Ok((
authority.clone(),
StateAnchorAbsenceObservation {
user_namespace: authority.user_namespace,
mount_namespace: authority.mount_namespace,
mount: authority.mount,
observed,
},
))
}
/// Completes Domain Clean, exact catalog deletion, and prepared-lower Clean after unmount.
///
/// # Errors
@ -404,7 +465,15 @@ impl<'a> StateAnchorCoordinator<'a> {
authority.generation_id,
ParticipantRole::StateAnchor,
)?),
DomainStoreMutation::PutParticipant(prepared_release_intent(&prepared)?),
DomainStoreMutation::PutParticipant(prepared_release_intent(
&prepared,
StateAnchorReleaseEvidence {
daemon: authority.daemon,
user_namespace: authority.user_namespace,
mount_namespace: authority.mount_namespace,
mount: authority.mount,
},
)?),
])?;
}
(ParticipantState::Clean, ParticipantState::ReleaseIntent) => {}
@ -422,6 +491,80 @@ impl<'a> StateAnchorCoordinator<'a> {
Ok(())
}
/// Resumes lease release after the atomic State Anchor Clean checkpoint was committed.
///
/// # Errors
/// Refuses missing retained release evidence, generation or layout drift, a mismatched lease,
/// or failure of the fresh catalog-lock absence observation.
pub fn resume_release_after_checkpoint<Observe>(
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
mut observe_again: Observe,
) -> Result<(), StateAnchorCoordinatorError>
where
Observe: FnMut() -> Result<StateAnchorAbsenceObservation, StateAnchorCoordinatorError>,
{
let snapshot = self.domain_store.snapshot()?;
let anchor = role(
&snapshot.participants,
generation_id,
ParticipantRole::StateAnchor,
)?;
let prepared = role(
&snapshot.participants,
generation_id,
ParticipantRole::PreparedLower,
)?;
if anchor.state() != ParticipantState::Clean
|| prepared.state() != ParticipantState::ReleaseIntent
{
return Err(StateAnchorCoordinatorError::InvalidState(
"durable release checkpoint is absent",
));
}
let Some(ParticipantOwnership::PreparedLower(ownership)) = prepared.ownership() else {
return Err(StateAnchorCoordinatorError::InvalidState(
"prepared-lower ownership is invalid",
));
};
let evidence =
ownership
.released_state_anchor
.ok_or(StateAnchorCoordinatorError::InvalidState(
"prepared-lower release has no exact State Anchor evidence",
))?;
let artifact =
ownership
.prepared_artifact_id
.ok_or(StateAnchorCoordinatorError::InvalidState(
"prepared-lower artifact is absent",
))?;
let expected_layout = layout.ownership_intent(artifact);
let merged = expected_layout.merged_path.as_ref().ok_or(
StateAnchorCoordinatorError::InvalidState("State Anchor merged path is absent"),
)?;
let authority = StateAnchorReleaseAuthority {
generation_id,
lease: ArtifactLease {
prepared_artifact_id: artifact,
domain_id: snapshot.domain.domain_id,
generation_id,
},
merged_label: path_from_record(merged).to_path_buf(),
mount: evidence.mount,
daemon: evidence.daemon,
user_namespace: evidence.user_namespace,
mount_namespace: evidence.mount_namespace,
};
self.release_catalog_lease_if_present(&authority, &mut observe_again)?;
self.domain_store
.apply(&[DomainStoreMutation::CleanParticipantAfterAbsence(
clean_participant(generation_id, ParticipantRole::PreparedLower)?,
)])?;
Ok(())
}
fn require_staged_identity(
&self,
staged: StagedProcessOwnership,
@ -754,10 +897,12 @@ fn clean_participant(
fn prepared_release_intent(
prepared: &Participant,
released_state_anchor: StateAnchorReleaseEvidence,
) -> Result<Participant, StateAnchorCoordinatorError> {
let ownership = match prepared.ownership() {
Some(ParticipantOwnership::PreparedLower(ownership)) => PreparedLowerOwnership {
lease_state: Some(ParticipantState::ReleaseIntent),
released_state_anchor: Some(released_state_anchor),
..ownership.clone()
},
_ => {

22
crates/lazaret-host-coordinator/src/state_anchor/tests.rs
View file

@ -232,20 +232,31 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
DomainStoreMutation::CleanParticipantAfterAbsence(
clean_participant(generation_id, ParticipantRole::StateAnchor).unwrap(),
),
DomainStoreMutation::PutParticipant(prepared_release_intent(prepared).unwrap()),
DomainStoreMutation::PutParticipant(
prepared_release_intent(
prepared,
StateAnchorReleaseEvidence {
daemon: authority.daemon(),
user_namespace: authority.user_namespace(),
mount_namespace: authority.mount_namespace(),
mount: authority.mount(),
},
)
.unwrap(),
),
])
.unwrap();
for invalid in invalid_observations {
assert!(
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || Ok(invalid))
.resume_release_after_checkpoint(generation_id, &layout, || Ok(invalid))
.is_err()
);
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
}
assert!(
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || {
.resume_release_after_checkpoint(generation_id, &layout, || {
Err(StateAnchorCoordinatorError::InvalidState(
"State Anchor absence evidence is unreadable",
))
@ -255,7 +266,7 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
assert!(
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || {
.resume_release_after_checkpoint(generation_id, &layout, || {
Err(StateAnchorCoordinatorError::InvalidState(
"second State Anchor observation was skipped",
))
@ -265,7 +276,7 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
let mut observations = 0;
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || {
.resume_release_after_checkpoint(generation_id, &layout, || {
observations += 1;
Ok(valid)
})
@ -552,6 +563,7 @@ impl AcquisitionFixture {
path: Some(self.artifact.path.clone()),
device: Some(self.artifact.device),
inode: Some(self.artifact.inode),
released_state_anchor: None,
}),
)
.unwrap()