Integrate exact production recovery before cold start

crates/lazaret-host-coordinator/src/production_selector.rs

@@ -30,6 +30,7 @@ use rustix::fs::{Mode, OFlags, ResolveFlags, openat2};
 
 use crate::domain_store::HostCoordinatorDomainStore;
 use crate::production::ProductionColdStartInputs;
+use crate::recovery::recover_production_before_cold_start;
 use crate::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE;
 
 const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
@@ -150,6 +151,8 @@ async fn select_production_cold_start_for_platform(
     let prepared =
         PreparedLowerStore::under_storage_root(configuration.storage_root(), catalog.clone())
             .map_err(failure)?;
+    recover_production_before_cold_start(configuration, &store, &catalog, &prepared)
+        .map_err(failure)?;
     let image = OciRegistryTransport
         .pull_confined(
             &configuration.fingerprint_record().image,

crates/lazaret-host-coordinator/src/state_anchor.rs

@@ -18,6 +18,7 @@ use lazaret_state_anchor::{observe_mount_absence, recover_private_state_anchor_m
 use lazaret_storage_catalog::{Catalog, CatalogError};
 
 use crate::domain_store::{DomainStoreError, DomainStoreMutation, HostCoordinatorDomainStore};
+use crate::recovery::RuntimeOwnersAbsent;
 use crate::runtime::StateAnchorAbsenceObservation;
 
 #[derive(Debug)]
@@ -106,17 +107,6 @@ impl StateAnchorReleaseAuthority {
     }
 }
 
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub struct RuntimeOwnersAbsent {
-    generation_id: GenerationId,
-}
-
-impl RuntimeOwnersAbsent {
-    pub(crate) const fn for_generation(generation_id: GenerationId) -> Self {
-        Self { generation_id }
-    }
-}
-
 pub struct StateAnchorCoordinator<'a> {
     domain_store: &'a HostCoordinatorDomainStore,
     catalog: &'a Catalog,
@@ -364,6 +354,11 @@ impl<'a> StateAnchorCoordinator<'a> {
                 "State Anchor layout changed",
             ));
         }
+        if self.catalog_lease_checkpoint(&authority.lease)? != CatalogLeaseCheckpoint::Exact {
+            return Err(StateAnchorCoordinatorError::InvalidState(
+                "LeaseHeld prepared lower lacks its exact catalog lease",
+            ));
+        }
         self.domain_store
             .apply(&[DomainStoreMutation::PutParticipant(participant(
                 generation_id,
@@ -389,16 +384,11 @@ impl<'a> StateAnchorCoordinator<'a> {
         &self,
         generation_id: GenerationId,
         layout: &StateAnchorLayout<'_>,
-        owners_absent: RuntimeOwnersAbsent,
+        owners_absent: Option<&RuntimeOwnersAbsent>,
     ) -> Result<
         (StateAnchorReleaseAuthority, StateAnchorAbsenceObservation),
         StateAnchorCoordinatorError,
     > {
-        if owners_absent.generation_id != generation_id {
-            return Err(StateAnchorCoordinatorError::InvalidState(
-                "runtime-owner absence belongs to another generation",
-            ));
-        }
         let authority = self.begin_release(generation_id, layout)?;
         let observed = match match_recovery_process(authority.daemon) {
             Ok(PidfsRecoveryMatch::Matched(holder)) => recover_private_state_anchor_mount(
@@ -408,8 +398,12 @@ impl<'a> StateAnchorCoordinator<'a> {
                 authority.mount_namespace,
                 authority.mount,
             )?,
-            Ok(PidfsRecoveryMatch::BootMismatch) => observe_mount_absence(layout, authority.mount)?,
+            Ok(PidfsRecoveryMatch::BootMismatch) => {
+                self.require_absence_witness(owners_absent, &authority)?;
+                observe_mount_absence(layout, authority.mount)?
+            }
             Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {
+                self.require_absence_witness(owners_absent, &authority)?;
                 observe_mount_absence(layout, authority.mount)?
             }
             Err(error) => return Err(StateAnchorCoordinatorError::Physical(error.into())),
@@ -425,6 +419,39 @@ impl<'a> StateAnchorCoordinator<'a> {
         ))
     }
 
+    fn require_absence_witness(
+        &self,
+        witness: Option<&RuntimeOwnersAbsent>,
+        authority: &StateAnchorReleaseAuthority,
+    ) -> Result<(), StateAnchorCoordinatorError> {
+        let witness = witness.ok_or(StateAnchorCoordinatorError::InvalidState(
+            "complete runtime-owner absence proof is required",
+        ))?;
+        let snapshot = self.domain_store.snapshot()?;
+        let generation = snapshot
+            .generations
+            .iter()
+            .find(|generation| generation.generation_id == authority.generation_id)
+            .ok_or(StateAnchorCoordinatorError::InvalidState(
+                "recovery generation is absent",
+            ))?;
+        if !witness
+            .ownership()
+            .binding_matches(snapshot.domain.domain_id, generation)
+            || !witness.ownership().state_anchor_matches(
+                authority.daemon,
+                authority.user_namespace,
+                authority.mount_namespace,
+                authority.mount,
+            )
+        {
+            return Err(StateAnchorCoordinatorError::InvalidState(
+                "runtime-owner absence proof mismatches durable authority",
+            ));
+        }
+        Ok(())
+    }
+
     /// Completes Domain Clean, exact catalog deletion, and prepared-lower Clean after unmount.
     ///
     /// # Errors
@@ -460,6 +487,12 @@ impl<'a> StateAnchorCoordinator<'a> {
                         "release authority does not match durable state",
                     ));
                 }
+                if self.catalog_lease_checkpoint(&authority.lease)? != CatalogLeaseCheckpoint::Exact
+                {
+                    return Err(StateAnchorCoordinatorError::InvalidState(
+                        "LeaseHeld prepared lower lacks its exact catalog lease",
+                    ));
+                }
                 self.domain_store.apply(&[
                     DomainStoreMutation::CleanParticipantAfterAbsence(clean_participant(
                         authority.generation_id,

crates/lazaret-host-coordinator/src/state_anchor/tests.rs

@@ -183,6 +183,19 @@ fn durable_phases_resume_release_without_collapsing_lease_order() {
     );
     assert_eq!(catalog.snapshot().unwrap().artifact_leases, vec![lease]);
     let valid = absence_observation(&authority);
+    catalog.release_lease(&lease).unwrap();
+    assert!(
+        coordinator
+            .complete_release_after_unmount(&authority, &layout, valid, || Ok(valid))
+            .is_err()
+    );
+    assert_states(
+        &domain_store,
+        ParticipantState::LeaseHeld,
+        ParticipantState::ReleaseIntent,
+    );
+    assert!(catalog.snapshot().unwrap().artifact_leases.is_empty());
+    catalog.acquire_lease_checked(&artifact, &lease).unwrap();
     let invalid_observations = [
         StateAnchorAbsenceObservation {
             observed: authority.mount(),