Bind prepared artifact selection against reclaim

crates/lazaret-host-coordinator/src/bootstrap.rs

@@ -11,6 +11,7 @@ use lazaret_core::records::{
     Generation, GenerationIdentity, Participant, ParticipantOwnership, ParticipantRole,
     ParticipantState, PreparedArtifact,
 };
+use lazaret_linux::filesystem::SharedFileLock;
 use lazaret_linux::process::{ProcessOwnershipContext, current_process_identity};
 use lazaret_runtime_supervisor::sandbox::SandboxBundle;
 use lazaret_runtime_supervisor::supervision::RuntimeSupervisorLaunch;
@@ -65,6 +66,7 @@ pub struct BootstrapPlan<'a> {
     pub configuration: &'a ResolvedDomainConfiguration,
     pub generation_identity: GenerationIdentity,
     pub prepared_artifact: &'a PreparedArtifact,
+    pub prepared_artifact_selection: Option<SharedFileLock>,
     pub state_anchor_layout: &'a StateAnchorLayout<'a>,
     pub runtime_launch: RuntimeSupervisorLaunch,
     pub sandbox_bundle: &'a SandboxBundle,
@@ -281,15 +283,21 @@ impl<'a> CoordinatorLifecycleAdapter<'a> {
         Ok(())
     }
 
-    fn lease_prepared_lower(&self) -> Result<(), LifecycleServiceError> {
+    fn lease_prepared_lower(&mut self) -> Result<(), LifecycleServiceError> {
         self.state_anchor()
             .begin_acquisition(
                 self.plan.generation_identity.generation_id,
                 self.plan.prepared_artifact,
                 self.plan.state_anchor_layout,
             )
-            .map(|_| ())
+            .map_err(failure)?;
-            .map_err(failure)
+        drop(
+            self.plan
+                .prepared_artifact_selection
+                .take()
+                .ok_or_else(|| LifecycleServiceError("prepared artifact claim is absent".into()))?,
+        );
+        Ok(())
     }
 
     fn mount_state_anchor(&mut self) -> Result<(), LifecycleServiceError> {

crates/lazaret-host-coordinator/src/client_bootstrap.rs

@@ -132,7 +132,7 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
         .enable_all()
         .build()
         .map_err(failure)?;
-    let selection = runtime
+    let mut selection = runtime
         .block_on(select_runtime(
             &configuration,
             Arc::clone(&store),

crates/lazaret-host-coordinator/src/cold_start.rs

@@ -16,6 +16,7 @@ use lazaret_core::records::{
     Generation, GenerationIdentity, HostProcessOwnershipSlot, PreparedArtifact, Session,
 };
 use lazaret_ignored_workspace::git_common_directory;
+use lazaret_linux::filesystem::SharedFileLock;
 use lazaret_linux::filesystem::{RegularFileMailbox, mount_identity, namespace_identity};
 use lazaret_linux::process::{
     ProcessOwnershipContext, TargetCommand, child_broker, current_process_identity,
@@ -76,6 +77,7 @@ pub struct ColdStartInputs<'a> {
     pub generation: Generation,
     pub prepared_artifact: &'a PreparedArtifact,
     pub prepared_artifact_authority: &'a File,
+    pub prepared_artifact_selection: SharedFileLock,
     pub prepared_artifact_label: &'a Path,
     pub root_process: RootProcess,
     pub container_id: ContainerId,
@@ -135,6 +137,7 @@ pub struct ColdStart<'a> {
     generation_identity: GenerationIdentity,
     prepared_artifact: &'a PreparedArtifact,
     state_anchor_layout: StateAnchorLayout<'a>,
+    prepared_artifact_selection: Option<SharedFileLock>,
     runtime_launch: Option<RuntimeSupervisorLaunch>,
     sandbox_bundle: SandboxBundle,
     network_policy: lazaret_core::network::CanonicalDerivedNetworkPolicy,
@@ -256,6 +259,7 @@ impl<'a> ColdStart<'a> {
             generation_identity,
             prepared_artifact: inputs.prepared_artifact,
             state_anchor_layout,
+            prepared_artifact_selection: Some(inputs.prepared_artifact_selection),
             runtime_launch: Some(runtime_launch),
             sandbox_bundle,
             network_policy,
@@ -371,6 +375,7 @@ impl<'a> ColdStart<'a> {
             configuration: self.configuration,
             generation_identity: self.generation_identity.clone(),
             prepared_artifact: self.prepared_artifact,
+            prepared_artifact_selection: self.prepared_artifact_selection.take(),
             state_anchor_layout: &self.state_anchor_layout,
             runtime_launch: self
                 .runtime_launch

crates/lazaret-host-coordinator/src/runtime_selector.rs

@@ -11,6 +11,7 @@ use lazaret_core::identity::{
     PrepareOperationId, SandboxId,
 };
 use lazaret_core::records::{DurableDecision, Generation, PreparedArtifact, VolumeKind};
+use lazaret_linux::filesystem::SharedFileLock;
 use lazaret_linux::platform::{PlatformObservation, check_platform};
 use lazaret_linux::user_namespace::UserNamespaceMap;
 use lazaret_prepared_lower::{
@@ -32,6 +33,8 @@ use crate::prepared_worker::prepare_in_user_namespace;
 use crate::recovery::recover_before_cold_start;
 use crate::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE;
 
+const PREPARED_SELECTION_ATTEMPTS: usize = 16;
+
 #[derive(Debug)]
 pub enum RuntimeSelectionError {
     PreparedLower(PreparedLowerError),
@@ -78,6 +81,7 @@ pub struct RuntimeSelection {
     catalog: Catalog,
     artifact: PreparedArtifact,
     artifact_authority: File,
+    artifact_selection: Option<SharedFileLock>,
     artifact_label: PathBuf,
     generation: Generation,
     container_id: ContainerId,
@@ -92,7 +96,7 @@ impl RuntimeSelection {
     /// # Errors
     /// Refuses a configuration other than the exact one used for selection.
     pub fn cold_start_inputs<'a>(
-        &'a self,
+        &'a mut self,
         configuration: &'a ResolvedDomainConfiguration,
     ) -> Result<ColdStartInputs<'a>, RuntimeSelectionError> {
         if configuration.domain_id() != self.domain_id
@@ -107,6 +111,10 @@ impl RuntimeSelection {
             generation: self.generation.clone(),
             prepared_artifact: &self.artifact,
             prepared_artifact_authority: &self.artifact_authority,
+            prepared_artifact_selection: self
+                .artifact_selection
+                .take()
+                .ok_or_else(|| failure("selected artifact claim was already consumed"))?,
             prepared_artifact_label: &self.artifact_label,
             root_process: RootProcess::new(
                 SANDBOX_INIT_DESTINATION,
@@ -174,6 +182,8 @@ async fn select_runtime_for_platform(
         )
         .await
         .map_err(RuntimeSelectionError::PreparedLower)?;
+    let mut selection = None;
+    for _ in 0..PREPARED_SELECTION_ATTEMPTS {
         let artifact = prepare_in_user_namespace(
             configuration,
             &image,
@@ -182,9 +192,17 @@ async fn select_runtime_for_platform(
             &user_namespace_map,
         )
         .map_err(|error| failure(format!("mapped preparation worker failed: {error}")))?;
-    let artifact_authority = prepared
+        if let Some(selected) = prepared
-        .open_artifact_authority(&artifact)
+            .select_artifact(&artifact)
-        .map_err(RuntimeSelectionError::PreparedLower)?;
+            .map_err(RuntimeSelectionError::PreparedLower)?
+        {
+            selection = Some(selected);
+            break;
+        }
+    }
+    let selection = selection
+        .ok_or_else(|| failure("prepared artifact was repeatedly reclaimed before selection"))?;
+    let (artifact, artifact_authority, artifact_selection) = selection.into_parts();
     let artifact_label = PathBuf::from(std::ffi::OsString::from_vec(
         configuration
             .storage_root()
@@ -218,6 +236,7 @@ async fn select_runtime_for_platform(
         catalog,
         artifact,
         artifact_authority,
+        artifact_selection: Some(artifact_selection),
         artifact_label,
         generation,
         container_id: ContainerId::new(random_id()?),

crates/lazaret-host-coordinator/tests/cold_start.rs

@@ -13,7 +13,7 @@ use lazaret_core::identity::{
 use lazaret_core::records::{ArtifactState, Domain, DurableDecision, Generation, PreparedArtifact};
 use lazaret_host_coordinator::cold_start::{ColdStart, ColdStartInputs};
 use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore;
-use lazaret_linux::filesystem::try_exclusive_flock;
+use lazaret_linux::filesystem::{try_exclusive_flock, try_shared_flock};
 use lazaret_linux::user_namespace::{
     REQUIRED_SUBORDINATE_IDS, UserNamespaceIdRange, UserNamespaceMap,
 };
@@ -267,6 +267,10 @@ impl ColdStartFixture {
             generation: self.generation.clone(),
             prepared_artifact: &self.artifact,
             prepared_artifact_authority: &self.artifact_authority,
+            prepared_artifact_selection: try_shared_flock(
+                File::open(&self.prepared_lower).unwrap(),
+            )
+            .unwrap(),
             prepared_artifact_label: Path::new(
                 std::str::from_utf8(self.artifact.path.as_bytes()).unwrap(),
             ),

crates/lazaret-linux/src/filesystem.rs

@@ -351,6 +351,13 @@ pub fn try_exclusive_flock(file: File) -> io::Result<ExclusiveFileLock> {
     Ok(ExclusiveFileLock(file))
 }
 
+/// # Errors
+/// Returns an error when the nonblocking shared lock is unavailable.
+pub fn try_shared_flock(file: File) -> io::Result<SharedFileLock> {
+    flock(file.as_fd(), FlockOperation::NonBlockingLockShared)?;
+    Ok(SharedFileLock(file))
+}
+
 #[derive(Debug)]
 pub struct ExclusiveFileLock(File);
 
@@ -360,6 +367,15 @@ impl AsFd for ExclusiveFileLock {
     }
 }
 
+#[derive(Debug)]
+pub struct SharedFileLock(File);
+
+impl AsFd for SharedFileLock {
+    fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> {
+        self.0.as_fd()
+    }
+}
+
 /// # Errors
 /// Returns an error unless the replacement and containing directory are synced.
 pub fn atomic_replace(path: &Path, bytes: &[u8]) -> io::Result<()> {
@@ -459,7 +475,7 @@ pub fn require_mount_identity(path: &Path, expected: MountIdentity) -> io::Resul
 
 #[cfg(test)]
 mod tests {
-    use std::fs::OpenOptions;
+    use std::fs::{File, OpenOptions};
     use std::io::{Read, Write};
     use std::os::unix::fs::symlink;
 
@@ -468,6 +484,7 @@ mod tests {
     use super::{
         RegularFileMailbox, atomic_replace, current_mount_identity, namespace_identity,
         require_mount_identity, require_write_sealed, sealed_memfd, try_exclusive_flock,
+        try_shared_flock,
     };
 
     fn test_directory(name: &str) -> std::path::PathBuf {
@@ -552,6 +569,21 @@ mod tests {
         std::fs::remove_dir_all(directory).unwrap();
     }
 
+    #[test]
+    fn shared_flock_excludes_reclaim_until_every_claim_closes() {
+        let directory = test_directory("shared-flock");
+        let path = directory.join("artifact");
+        std::fs::create_dir(&path).unwrap();
+        let first = try_shared_flock(File::open(&path).unwrap()).unwrap();
+        let second = try_shared_flock(File::open(&path).unwrap()).unwrap();
+        assert!(try_exclusive_flock(File::open(&path).unwrap()).is_err());
+        drop(first);
+        assert!(try_exclusive_flock(File::open(&path).unwrap()).is_err());
+        drop(second);
+        assert!(try_exclusive_flock(File::open(&path).unwrap()).is_ok());
+        std::fs::remove_dir_all(directory).unwrap();
+    }
+
     #[test]
     fn namespace_identity_is_stable_for_same_namespace() {
         let first = namespace_identity(std::path::Path::new("/proc/self/ns/mnt")).unwrap();

crates/lazaret-prepared-lower/src/lib.rs

@@ -10,6 +10,7 @@ pub use layer::{
     IdMap, IdMapRange, PreparationLimits, PreparedDestination, PreparedDestinationKind,
 };
 pub use store::{
-    PrepareRecovery, PrepareRequest, PreparedArtifactGcResult, PreparedLowerStore, ReclaimOutcome,
+    PrepareRecovery, PrepareRequest, PreparedArtifactGcResult, PreparedArtifactSelection,
+    PreparedLowerStore, ReclaimOutcome,
 };
 pub use transport::{OciRegistryTransport, Platform, RegistryCredentials, TransportedImage};

crates/lazaret-prepared-lower/src/store.rs

@@ -14,7 +14,9 @@ use lazaret_core::identity::{
     SelectedManifestDigest, SourceIdentity,
 };
 use lazaret_core::records::{ArtifactState, PrepareOperation, PrepareStageState, PreparedArtifact};
-use lazaret_linux::filesystem::{ExclusiveFileLock, try_exclusive_flock};
+use lazaret_linux::filesystem::{
+    ExclusiveFileLock, SharedFileLock, try_exclusive_flock, try_shared_flock,
+};
 use lazaret_storage_catalog::{Catalog, CatalogError, CatalogRow};
 use rustix::fs::{
     AtFlags, Mode, OFlags, RenameFlags, ResolveFlags, mkdirat, openat2, renameat, renameat_with,
@@ -56,6 +58,7 @@ pub enum PrepareRecovery {
 pub enum ReclaimOutcome {
     Removed,
     ReplacementPreserved,
+    RetainedBySelection,
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -64,6 +67,25 @@ pub struct PreparedArtifactGcResult {
     pub outcome: ReclaimOutcome,
 }
 
+#[derive(Debug)]
+pub struct PreparedArtifactSelection {
+    artifact: PreparedArtifact,
+    authority: File,
+    claim: SharedFileLock,
+}
+
+impl PreparedArtifactSelection {
+    #[must_use]
+    pub fn artifact(&self) -> &PreparedArtifact {
+        &self.artifact
+    }
+
+    #[must_use]
+    pub fn into_parts(self) -> (PreparedArtifact, File, SharedFileLock) {
+        (self.artifact, self.authority, self.claim)
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct PreparedLowerStore {
     root: Arc<File>,
@@ -178,6 +200,52 @@ impl PreparedLowerStore {
             .context("prepared artifact path is absent")
     }
 
+    /// Selects the current equivalent artifact and retains a claim that excludes reclaim.
+    ///
+    /// # Errors
+    /// Refuses semantic identity drift, catalog inconsistency, or physical identity drift.
+    pub fn select_artifact(
+        &self,
+        expected: &PreparedArtifact,
+    ) -> PreparedLowerResult<Option<PreparedArtifactSelection>> {
+        let physical_lock = self.physical_lock()?;
+        self.select_artifact_inner(expected, &physical_lock)
+            .map_err(PreparedLowerError::confinement)
+    }
+
+    fn select_artifact_inner(
+        &self,
+        expected: &PreparedArtifact,
+        _physical_lock: &ExclusiveFileLock,
+    ) -> Result<Option<PreparedArtifactSelection>> {
+        let Some(artifact) = self.available_artifact(expected.prepared_artifact_id)? else {
+            return Ok(None);
+        };
+        require_equivalent_artifact(expected, &artifact)?;
+        let path = artifact_path(artifact.prepared_artifact_id);
+        let authority = self
+            .open_relative(&path)?
+            .context("selected artifact path is absent")?;
+        require_identity(&authority, artifact.device, artifact.inode)?;
+        let claim_file = self
+            .open_relative(&path)?
+            .context("selected artifact claim path is absent")?;
+        require_identity(&claim_file, artifact.device, artifact.inode)?;
+        let claim = try_shared_flock(claim_file)?;
+        if self
+            .catalog_artifact(artifact.prepared_artifact_id)?
+            .as_ref()
+            != Some(&artifact)
+        {
+            bail!("selected artifact catalog ownership changed");
+        }
+        Ok(Some(PreparedArtifactSelection {
+            artifact,
+            authority,
+            claim,
+        }))
+    }
+
     /// Materializes and atomically publishes one verified immutable prepared lower.
     ///
     /// # Errors
@@ -498,11 +566,26 @@ impl PreparedLowerStore {
     /// Refuses leases, forged catalog rows, physical identity mismatch, foreign paths, and
     /// filesystem or catalog failures.
     pub fn reclaim(&self, expected: &PreparedArtifact) -> PreparedLowerResult<ReclaimOutcome> {
+        let _physical_lock = self.physical_lock()?;
+        let artifact = self
+            .open_relative(&artifact_path(expected.prepared_artifact_id))
+            .map_err(PreparedLowerError::reclaim)?
+            .ok_or_else(|| PreparedLowerError::reclaim(anyhow::anyhow!("artifact is absent")))?;
+        require_identity(&artifact, expected.device, expected.inode)
+            .map_err(PreparedLowerError::reclaim)?;
+        let _claim = match try_exclusive_flock(artifact) {
+            Ok(claim) => claim,
+            Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
+                return Ok(ReclaimOutcome::RetainedBySelection);
+            }
+            Err(error) => return Err(PreparedLowerError::reclaim(error)),
+        };
         let reclaiming = self
             .catalog
             .begin_reclaim(expected)
             .map_err(PreparedLowerError::catalog)?;
-        self.resume_reclaim(&reclaiming)
+        self.resume_reclaim_inner(&reclaiming)
+            .map_err(PreparedLowerError::reclaim)
     }
 
     /// Resumes an exact durable reclaim after interruption.
@@ -516,10 +599,28 @@ impl PreparedLowerStore {
         reclaiming: &PreparedArtifact,
     ) -> PreparedLowerResult<ReclaimOutcome> {
         let _physical_lock = self.physical_lock()?;
+        let _claim = self
+            .lock_reclaiming_artifact(reclaiming)
+            .map_err(PreparedLowerError::reclaim)?;
         self.resume_reclaim_inner(reclaiming)
             .map_err(PreparedLowerError::reclaim)
     }
 
+    fn lock_reclaiming_artifact(&self, reclaiming: &PreparedArtifact) -> Result<ExclusiveFileLock> {
+        for path in [
+            artifact_path(reclaiming.prepared_artifact_id),
+            quarantine_path(reclaiming),
+        ] {
+            let Some(file) = self.open_relative(&path)? else {
+                continue;
+            };
+            if file_identity(&file)? == (reclaiming.device, reclaiming.inode) {
+                return Ok(try_exclusive_flock(file)?);
+            }
+        }
+        bail!("reclaiming artifact and quarantine are both absent")
+    }
+
     fn resume_reclaim_inner(&self, reclaiming: &PreparedArtifact) -> Result<ReclaimOutcome> {
         if reclaiming.state != ArtifactState::Reclaiming {
             bail!("artifact is not reclaiming");
@@ -903,6 +1004,24 @@ fn artifact_from_operation(
     }
 }
 
+fn require_equivalent_artifact(
+    expected: &PreparedArtifact,
+    observed: &PreparedArtifact,
+) -> Result<()> {
+    if expected.state != ArtifactState::Available
+        || expected.prepared_artifact_id != observed.prepared_artifact_id
+        || expected.source_identity != observed.source_identity
+        || expected.selected_manifest_digest != observed.selected_manifest_digest
+        || expected.policy_id != observed.policy_id
+        || expected.id_map_digest != observed.id_map_digest
+        || expected.path != observed.path
+        || observed.state != ArtifactState::Available
+    {
+        bail!("selected prepared artifact identity inputs differ");
+    }
+    Ok(())
+}
+
 fn file_identity(file: &File) -> Result<(DeviceId, InodeId)> {
     let stat = rustix::fs::fstat(file)?;
     Ok((DeviceId::new(stat.st_dev), InodeId::new(stat.st_ino)))