ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Integrate exact production recovery before cold start

Browse source
This commit is contained in:
ms2k 2026-07-17 06:15:18 +09:00
commit f21a847991
6 changed files with 1537 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

5
crates/lazaret-host-coordinator/Cargo.toml
View file

@ -14,6 +14,11 @@ name = "runtime-composition"
path = "tests/runtime-composition.rs"
harness = false
[[test]]
name = "recovery-process"
path = "tests/recovery-process.rs"
harness = false
[dependencies]
ed25519-dalek.workspace = true
getrandom.workspace = true

3
crates/lazaret-host-coordinator/src/production_selector.rs
View file

@ -30,6 +30,7 @@ use rustix::fs::{Mode, OFlags, ResolveFlags, openat2};
use crate::domain_store::HostCoordinatorDomainStore;
use crate::production::ProductionColdStartInputs;
use crate::recovery::recover_production_before_cold_start;
use crate::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE;
const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
@ -150,6 +151,8 @@ async fn select_production_cold_start_for_platform(
let prepared =
PreparedLowerStore::under_storage_root(configuration.storage_root(), catalog.clone())
.map_err(failure)?;
recover_production_before_cold_start(configuration, &store, &catalog, &prepared)
.map_err(failure)?;
let image = OciRegistryTransport
.pull_confined(
&configuration.fingerprint_record().image,

1499
crates/lazaret-host-coordinator/src/recovery.rs
View file

File diff suppressed because it is too large Load diff

69
crates/lazaret-host-coordinator/src/state_anchor.rs
View file

@ -18,6 +18,7 @@ use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_m
use lazaret_storage_catalog::{Catalog, CatalogError};
use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
use crate::recovery::RuntimeOwnersAbsent;
use crate::runtime::StateAnchorAbsenceObservation;
#[derive(Debug)]
@ -106,17 +107,6 @@ impl StateAnchorReleaseAuthority {
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct RuntimeOwnersAbsent {
generation_id: GenerationId,
}
impl RuntimeOwnersAbsent {
pub(crate) const fn for_generation(generation_id: GenerationId) -> Self {
Self { generation_id }
}
}
pub struct StateAnchorCoordinator<'a> {
domain_store: &'a HostCoordinatorDomainStore,
catalog: &'a Catalog,
@ -364,6 +354,11 @@ impl<'a> StateAnchorCoordinator<'a> {
"State Anchor layout changed",
));
}
if self.catalog_lease_checkpoint(&authority.lease)? != CatalogLeaseCheckpoint::Exact {
return Err(StateAnchorCoordinatorError::InvalidState(
"LeaseHeld prepared lower lacks its exact catalog lease",
));
}
self.domain_store
.apply(&[DomainStoreMutation::PutParticipant(participant(
generation_id,
@ -389,16 +384,11 @@ impl<'a> StateAnchorCoordinator<'a> {
&self,
generation_id: GenerationId,
layout: &StateAnchorLayout<'_>,
owners_absent: RuntimeOwnersAbsent,
owners_absent: Option<&RuntimeOwnersAbsent>,
) -> Result<
(StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
StateAnchorCoordinatorError,
> {
if owners_absent.generation_id != generation_id {
return Err(StateAnchorCoordinatorError::InvalidState(
"runtime-owner absence belongs to another generation",
));
}
let authority = self.begin_release(generation_id, layout)?;
let observed = match match_recovery_process(authority.daemon) {
Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
@ -408,8 +398,12 @@ impl<'a> StateAnchorCoordinator<'a> {
authority.mount_namespace,
authority.mount,
)?,
Ok(PidfsRecoveryMatch::BootMismatch) => observe_mount_absence(layout, authority.mount)?,
Ok(PidfsRecoveryMatch::BootMismatch) => {
self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)?
}
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
self.require_absence_witness(owners_absent, &authority)?;
observe_mount_absence(layout, authority.mount)?
}
Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
@ -425,6 +419,39 @@ impl<'a> StateAnchorCoordinator<'a> {
))
}
fn require_absence_witness(
&self,
witness: Option<&RuntimeOwnersAbsent>,
authority: &StateAnchorReleaseAuthority,
) -> Result<(), StateAnchorCoordinatorError> {
let witness = witness.ok_or(StateAnchorCoordinatorError::InvalidState(
"complete runtime-owner absence proof is required",
))?;
let snapshot = self.domain_store.snapshot()?;
let generation = snapshot
.generations
.iter()
.find(|generation| generation.generation_id == authority.generation_id)
.ok_or(StateAnchorCoordinatorError::InvalidState(
"recovery generation is absent",
))?;
if !witness
.ownership()
.binding_matches(snapshot.domain.domain_id, generation)
|| !witness.ownership().state_anchor_matches(
authority.daemon,
authority.user_namespace,
authority.mount_namespace,
authority.mount,
)
{
return Err(StateAnchorCoordinatorError::InvalidState(
"runtime-owner absence proof mismatches durable authority",
));
}
Ok(())
}
/// Completes Domain Clean, exact catalog deletion, and prepared-lower Clean after unmount.
///
/// # Errors
@ -460,6 +487,12 @@ impl<'a> StateAnchorCoordinator<'a> {
"release authority does not match durable state",
));
}
if self.catalog_lease_checkpoint(&authority.lease)? != CatalogLeaseCheckpoint::Exact
{
return Err(StateAnchorCoordinatorError::InvalidState(
"LeaseHeld prepared lower lacks its exact catalog lease",
));
}
self.domain_store.apply(&[
DomainStoreMutation::CleanParticipantAfterAbsence(clean_participant(
authority.generation_id,

13
crates/lazaret-host-coordinator/src/state_anchor/tests.rs
View file

@ -183,6 +183,19 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
);
assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
let valid = absence_observation(&authority);
catalog.release_lease(&lease).unwrap();
assert!(
coordinator
.complete_release_after_unmount(&authority, &layout, valid, || Ok(valid))
.is_err()
);
assert_states(
&domain_store,
ParticipantState::LeaseHeld,
ParticipantState::ReleaseIntent,
);
assert!(catalog.snapshot().unwrap().artifact_leases.is_empty());
catalog.acquire_lease_checked(&artifact, &lease).unwrap();
let invalid_observations = [
StateAnchorAbsenceObservation {
observed: authority.mount(),

81
crates/lazaret-host-coordinator/tests/recovery-process.rs Normal file
View file

@ -0,0 +1,81 @@
use std::io;
use std::process::ExitCode;
use lazaret_core::identity::{
ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, NetworkSidecarId, SandboxId,
};
use lazaret_core::records::{
DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PersistedHostProcessOwnership, StateAnchorOwnership,
};
use lazaret_host_coordinator::recovery::{prove_runtime_owners_absent, runtime_recovery_ownership};
use lazaret_linux::process::{TargetCommand, child_broker, dispatch_private_process_role};
use rustix::process::Signal;
fn main() -> ExitCode {
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
if let Some(result) = dispatch_private_process_role(&arguments) {
return exit(result);
}
exit(run())
}
fn exit(result: io::Result<()>) -> ExitCode {
match result {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
eprintln!("recovery process test failed: {error}");
ExitCode::FAILURE
}
}
}
fn run() -> io::Result<()> {
verify_exact_action(None)?;
verify_exact_action(Some(Signal::STOP))
}
fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
let captured =
child_broker().run_transient_captured(&TargetCommand::new("/bin/sleep").argument("60"))?;
if let Some(signal) = signal {
captured.process().send_signal(signal)?;
}
let process = captured.into_process();
let generation = generation();
let participants = [
Participant::bare_intent(generation.generation_id, ParticipantRole::PreparedLower),
Participant::snapshot(
generation.generation_id,
ParticipantRole::StateAnchor,
ParticipantState::Intent,
Some(ParticipantOwnership::StateAnchor(StateAnchorOwnership {
lower_artifact_id: None,
upper_path: None,
work_path: None,
merged_path: None,
daemon: PersistedHostProcessOwnership::staged(process.identity()),
user_namespace: None,
mount_namespace: None,
mount: None,
})),
)
.map_err(|_| io::Error::other("State Anchor participant is invalid"))?,
];
let ownership =
runtime_recovery_ownership(&participants, &generation).map_err(io::Error::other)?;
prove_runtime_owners_absent(&ownership)?;
child_broker().reap(process)
}
fn generation() -> Generation {
Generation {
domain_id: DomainId::new([1; 32]),
generation_id: GenerationId::new(1),
configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
coordinator_id: CoordinatorId::new([3; 32]),
network_sidecar_id: NetworkSidecarId::new([4; 32]),
sandbox_id: SandboxId::new([5; 32]),
durable_decision: DurableDecision::Preparing,
}
}