Integrate exact production recovery before cold start

crates/lazaret-core/src/records/registry.rs

@@ -19,9 +19,9 @@ use super::{
     SessionClose, SessionExit, SessionExitOutcome, SessionRegistrationRequest,
     SessionRegistrationResponse, SessionRequestCapability, SessionTerminal, SessionTerminalEvent,
     SessionTerminalMessage, SessionTerminalState, SignedProtocolRecord, StagedProcessOwnership,
-    StateAnchorOwnership, StorageCatalog, StoreMetadata, TerminalInput, TerminalOutput,
+    StateAnchorOwnership, StateAnchorReleaseEvidence, StorageCatalog, StoreMetadata, TerminalInput,
-    TerminalResize, TerminalSignal, TerminalSignalKind, TlsCredential, TlsRole, TrustManifest,
+    TerminalOutput, TerminalResize, TerminalSignal, TerminalSignalKind, TlsCredential, TlsRole,
-    TrustedHostConfigurationV1,
+    TrustManifest, TrustedHostConfigurationV1,
 };
 
 pub trait SchemaType {
@@ -107,6 +107,7 @@ schema_types!(
     "NamespaceIdentity" => NamespaceIdentity,
     "MountIdentity" => MountIdentity,
     "PreparedLowerOwnership" => PreparedLowerOwnership,
+    "StateAnchorReleaseEvidence" => StateAnchorReleaseEvidence,
     "StateAnchorOwnership" => StateAnchorOwnership,
     "GitRoutingOwnership" => GitRoutingOwnership,
     "NetworkOwnership" => NetworkOwnership,

crates/lazaret-core/tests/schema_records/encoding.rs

@@ -95,6 +95,18 @@ fn every_schema_shape_round_trips_and_rejects_malformed_or_unknown_versions() {
         ParticipantOwnership::Network(Box::new(network_ownership())),
         &mut seen,
     );
+    verify_shape(
+        StateAnchorReleaseEvidence {
+            daemon: process(72),
+            user_namespace: namespace(),
+            mount_namespace: NamespaceIdentity {
+                device: DeviceId::new(73),
+                inode: InodeId::new(74),
+            },
+            mount: mount(),
+        },
+        &mut seen,
+    );
     verify_shape(session.clone(), &mut seen);
     verify_shape(terminal.clone(), &mut seen);
     verify_shape(SessionTerminalState::ReleaseIntent, &mut seen);

crates/lazaret-host-coordinator/Cargo.toml

@@ -14,6 +14,11 @@ name = "runtime-composition"
 path = "tests/runtime-composition.rs"
 harness = false
 
+[[test]]
+name = "recovery-process"
+path = "tests/recovery-process.rs"
+harness = false
+
 [dependencies]
 ed25519-dalek.workspace = true
 getrandom.workspace = true

crates/lazaret-host-coordinator/src/production_selector.rs

@@ -30,6 +30,7 @@ use rustix::fs::{Mode, OFlags, ResolveFlags, openat2};
 
 use crate::domain_store::HostCoordinatorDomainStore;
 use crate::production::ProductionColdStartInputs;
+use crate::recovery::recover_production_before_cold_start;
 use crate::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE;
 
 const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
@@ -150,6 +151,8 @@ async fn select_production_cold_start_for_platform(
     let prepared =
         PreparedLowerStore::under_storage_root(configuration.storage_root(), catalog.clone())
             .map_err(failure)?;
+    recover_production_before_cold_start(configuration, &store, &catalog, &prepared)
+        .map_err(failure)?;
     let image = OciRegistryTransport
         .pull_confined(
             &configuration.fingerprint_record().image,

crates/lazaret-host-coordinator/src/state_anchor.rs

@@ -18,6 +18,7 @@ use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_m
 use lazaret_storage_catalog::{Catalog, CatalogError};
 
 use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
+use crate::recovery::RuntimeOwnersAbsent;
 use crate::runtime::StateAnchorAbsenceObservation;
 
 #[derive(Debug)]
@@ -106,17 +107,6 @@ impl StateAnchorReleaseAuthority {
     }
 }
 
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub struct RuntimeOwnersAbsent {
-    generation_id: GenerationId,
-}
-
-impl RuntimeOwnersAbsent {
-    pub(crate) const fn for_generation(generation_id: GenerationId) -> Self {
-        Self { generation_id }
-    }
-}
-
 pub struct StateAnchorCoordinator<'a> {
     domain_store: &'a HostCoordinatorDomainStore,
     catalog: &'a Catalog,
@@ -364,6 +354,11 @@ impl<'a> StateAnchorCoordinator<'a> {
                 "State Anchor layout changed",
             ));
         }
+        if self.catalog_lease_checkpoint(&authority.lease)? != CatalogLeaseCheckpoint::Exact {
+            return Err(StateAnchorCoordinatorError::InvalidState(
+                "LeaseHeld prepared lower lacks its exact catalog lease",
+            ));
+        }
         self.domain_store
             .apply(&[DomainStoreMutation::PutParticipant(participant(
                 generation_id,
@@ -389,16 +384,11 @@ impl<'a> StateAnchorCoordinator<'a> {
         &self,
         generation_id: GenerationId,
         layout: &StateAnchorLayout<'_>,
-        owners_absent: RuntimeOwnersAbsent,
+        owners_absent: Option<&RuntimeOwnersAbsent>,
     ) -> Result<
         (StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
         StateAnchorCoordinatorError,
     > {
-        if owners_absent.generation_id != generation_id {
-            return Err(StateAnchorCoordinatorError::InvalidState(
-                "runtime-owner absence belongs to another generation",
-            ));
-        }
         let authority = self.begin_release(generation_id, layout)?;
         let observed = match match_recovery_process(authority.daemon) {
             Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
@@ -408,8 +398,12 @@ impl<'a> StateAnchorCoordinator<'a> {
                 authority.mount_namespace,
                 authority.mount,
             )?,
-            Ok(PidfsRecoveryMatch::BootMismatch) => observe_mount_absence(layout, authority.mount)?,
+            Ok(PidfsRecoveryMatch::BootMismatch) => {
+                self.require_absence_witness(owners_absent, &authority)?;
+                observe_mount_absence(layout, authority.mount)?
+            }
             Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
+                self.require_absence_witness(owners_absent, &authority)?;
                 observe_mount_absence(layout, authority.mount)?
             }
             Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
@@ -425,6 +419,39 @@ impl<'a> StateAnchorCoordinator<'a> {
         ))
     }
 
+    fn require_absence_witness(
+        &self,
+        witness: Option<&RuntimeOwnersAbsent>,
+        authority: &StateAnchorReleaseAuthority,
+    ) -> Result<(), StateAnchorCoordinatorError> {
+        let witness = witness.ok_or(StateAnchorCoordinatorError::InvalidState(
+            "complete runtime-owner absence proof is required",
+        ))?;
+        let snapshot = self.domain_store.snapshot()?;
+        let generation = snapshot
+            .generations
+            .iter()
+            .find(|generation| generation.generation_id == authority.generation_id)
+            .ok_or(StateAnchorCoordinatorError::InvalidState(
+                "recovery generation is absent",
+            ))?;
+        if !witness
+            .ownership()
+            .binding_matches(snapshot.domain.domain_id, generation)
+            || !witness.ownership().state_anchor_matches(
+                authority.daemon,
+                authority.user_namespace,
+                authority.mount_namespace,
+                authority.mount,
+            )
+        {
+            return Err(StateAnchorCoordinatorError::InvalidState(
+                "runtime-owner absence proof mismatches durable authority",
+            ));
+        }
+        Ok(())
+    }
+
     /// Completes Domain Clean, exact catalog deletion, and prepared-lower Clean after unmount.
     ///
     /// # Errors
@@ -460,6 +487,12 @@ impl<'a> StateAnchorCoordinator<'a> {
                         "release authority does not match durable state",
                     ));
                 }
+                if self.catalog_lease_checkpoint(&authority.lease)? != CatalogLeaseCheckpoint::Exact
+                {
+                    return Err(StateAnchorCoordinatorError::InvalidState(
+                        "LeaseHeld prepared lower lacks its exact catalog lease",
+                    ));
+                }
                 self.domain_store.apply(&[
                     DomainStoreMutation::CleanParticipantAfterAbsence(clean_participant(
                         authority.generation_id,

crates/lazaret-host-coordinator/src/state_anchor/tests.rs

@@ -183,6 +183,19 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
     );
     assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
     let valid = absence_observation(&authority);
+    catalog.release_lease(&lease).unwrap();
+    assert!(
+        coordinator
+            .complete_release_after_unmount(&authority, &layout, valid, || Ok(valid))
+            .is_err()
+    );
+    assert_states(
+        &domain_store,
+        ParticipantState::LeaseHeld,
+        ParticipantState::ReleaseIntent,
+    );
+    assert!(catalog.snapshot().unwrap().artifact_leases.is_empty());
+    catalog.acquire_lease_checked(&artifact, &lease).unwrap();
     let invalid_observations = [
         StateAnchorAbsenceObservation {
             observed: authority.mount(),

crates/lazaret-host-coordinator/tests/recovery-process.rs

@@ -0,0 +1,81 @@
+use std::io;
+use std::process::ExitCode;
+
+use lazaret_core::identity::{
+    ConfigurationFingerprint, CoordinatorId, DomainId, GenerationId, NetworkSidecarId, SandboxId,
+};
+use lazaret_core::records::{
+    DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
+    ParticipantState, PersistedHostProcessOwnership, StateAnchorOwnership,
+};
+use lazaret_host_coordinator::recovery::{prove_runtime_owners_absent, runtime_recovery_ownership};
+use lazaret_linux::process::{TargetCommand, child_broker, dispatch_private_process_role};
+use rustix::process::Signal;
+
+fn main() -> ExitCode {
+    let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
+    if let Some(result) = dispatch_private_process_role(&arguments) {
+        return exit(result);
+    }
+    exit(run())
+}
+
+fn exit(result: io::Result<()>) -> ExitCode {
+    match result {
+        Ok(()) => ExitCode::SUCCESS,
+        Err(error) => {
+            eprintln!("recovery process test failed: {error}");
+            ExitCode::FAILURE
+        }
+    }
+}
+
+fn run() -> io::Result<()> {
+    verify_exact_action(None)?;
+    verify_exact_action(Some(Signal::STOP))
+}
+
+fn verify_exact_action(signal: Option<Signal>) -> io::Result<()> {
+    let captured =
+        child_broker().run_transient_captured(&TargetCommand::new("/bin/sleep").argument("60"))?;
+    if let Some(signal) = signal {
+        captured.process().send_signal(signal)?;
+    }
+    let process = captured.into_process();
+    let generation = generation();
+    let participants = [
+        Participant::bare_intent(generation.generation_id, ParticipantRole::PreparedLower),
+        Participant::snapshot(
+            generation.generation_id,
+            ParticipantRole::StateAnchor,
+            ParticipantState::Intent,
+            Some(ParticipantOwnership::StateAnchor(StateAnchorOwnership {
+                lower_artifact_id: None,
+                upper_path: None,
+                work_path: None,
+                merged_path: None,
+                daemon: PersistedHostProcessOwnership::staged(process.identity()),
+                user_namespace: None,
+                mount_namespace: None,
+                mount: None,
+            })),
+        )
+        .map_err(|_| io::Error::other("State Anchor participant is invalid"))?,
+    ];
+    let ownership =
+        runtime_recovery_ownership(&participants, &generation).map_err(io::Error::other)?;
+    prove_runtime_owners_absent(&ownership)?;
+    child_broker().reap(process)
+}
+
+fn generation() -> Generation {
+    Generation {
+        domain_id: DomainId::new([1; 32]),
+        generation_id: GenerationId::new(1),
+        configuration_fingerprint: ConfigurationFingerprint::new([2; 32]),
+        coordinator_id: CoordinatorId::new([3; 32]),
+        network_sidecar_id: NetworkSidecarId::new([4; 32]),
+        sandbox_id: SandboxId::new([5; 32]),
+        durable_decision: DurableDecision::Preparing,
+    }
+}

tests/tooling/test_verify.py

@@ -275,7 +275,8 @@ class VerificationTests(unittest.TestCase):
                 "--filterset",
                 (
                     "not binary(capability-drop-helper) and not binary(acquisition) "
-                    "and not binary(runtime-composition) and not binary(network-runtime)"
+                    "and not binary(recovery-process) and not binary(runtime-composition) "
+                    "and not binary(network-runtime)"
                 ),
             ],
             [
@@ -288,6 +289,16 @@ class VerificationTests(unittest.TestCase):
                 "acquisition",
                 "--locked",
             ],
+            [
+                "cargo",
+                "test",
+                "--offline",
+                "-p",
+                "lazaret-host-coordinator",
+                "--test",
+                "recovery-process",
+                "--locked",
+            ],
             [
                 "cargo",
                 "test",

tools/lazaret_verify.py

@@ -29,7 +29,8 @@ NEXTEST_COMMAND = (
     "--filterset",
     (
         "not binary(capability-drop-helper) and not binary(acquisition) "
-        "and not binary(runtime-composition) and not binary(network-runtime)"
+        "and not binary(recovery-process) and not binary(runtime-composition) "
+        "and not binary(network-runtime)"
     ),
 )
 DOCTEST_COMMAND = (
@@ -61,6 +62,16 @@ ACQUISITION_TEST_COMMAND = (
     "acquisition",
     "--locked",
 )
+RECOVERY_PROCESS_TEST_COMMAND = (
+    "cargo",
+    "test",
+    "--offline",
+    "-p",
+    "lazaret-host-coordinator",
+    "--test",
+    "recovery-process",
+    "--locked",
+)
 RUNTIME_COMPOSITION_TEST_COMMAND = (
     "cargo",
     "test",
@@ -554,6 +565,7 @@ def refresh_advisories() -> None:
 def run_rust_tests() -> None:
     run(list(NEXTEST_COMMAND))
     run(list(ACQUISITION_TEST_COMMAND))
+    run(list(RECOVERY_PROCESS_TEST_COMMAND))
     run(list(CAPABILITY_TEST_COMMAND))
     run_physical(list(RUNTIME_COMPOSITION_TEST_COMMAND))
     run_physical(list(NETWORK_RUNTIME_TEST_COMMAND))