ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Bind prepared artifact selection against reclaim

Browse source
This commit is contained in:
ms2k 2026-07-22 01:51:46 +09:00
commit 1bd1d395d9
8 changed files with 209 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

14
crates/lazaret-host-coordinator/src/bootstrap.rs
View file

@ -11,6 +11,7 @@ use lazaret_core::records::{
Generation, GenerationIdentity, Participant, ParticipantOwnership, ParticipantRole,
ParticipantState, PreparedArtifact,
};
use lazaret_linux::filesystem::SharedFileLock;
use lazaret_linux::process::{ProcessOwnershipContext, current_process_identity};
use lazaret_runtime_supervisor::sandbox::SandboxBundle;
use lazaret_runtime_supervisor::supervision::RuntimeSupervisorLaunch;
@ -65,6 +66,7 @@ pub struct BootstrapPlan<'a> {
pub configuration: &'a ResolvedDomainConfiguration,
pub generation_identity: GenerationIdentity,
pub prepared_artifact: &'a PreparedArtifact,
pub prepared_artifact_selection: Option<SharedFileLock>,
pub state_anchor_layout: &'a StateAnchorLayout<'a>,
pub runtime_launch: RuntimeSupervisorLaunch,
pub sandbox_bundle: &'a SandboxBundle,
@ -281,15 +283,21 @@ impl<'a> CoordinatorLifecycleAdapter<'a> {
Ok(())
}
fn lease_prepared_lower(&self) -> Result<(), LifecycleServiceError> {
fn lease_prepared_lower(&mut self) -> Result<(), LifecycleServiceError> {
self.state_anchor()
.begin_acquisition(
self.plan.generation_identity.generation_id,
self.plan.prepared_artifact,
self.plan.state_anchor_layout,
)
.map(|_| ())
.map_err(failure)
.map_err(failure)?;
drop(
self.plan
.prepared_artifact_selection
.take()
.ok_or_else(|| LifecycleServiceError("prepared artifact claim is absent".into()))?,
);
Ok(())
}
fn mount_state_anchor(&mut self) -> Result<(), LifecycleServiceError> {

2
crates/lazaret-host-coordinator/src/client_bootstrap.rs
View file

@ -132,7 +132,7 @@ fn run_private_client_bootstrap(arguments: &[OsString]) -> Result<(), PrivateCli
.enable_all()
.build()
.map_err(failure)?;
let selection = runtime
let mut selection = runtime
.block_on(select_runtime(
&configuration,
Arc::clone(&store),

5
crates/lazaret-host-coordinator/src/cold_start.rs
View file

@ -16,6 +16,7 @@ use lazaret_core::records::{
Generation, GenerationIdentity, HostProcessOwnershipSlot, PreparedArtifact, Session,
};
use lazaret_ignored_workspace::git_common_directory;
use lazaret_linux::filesystem::SharedFileLock;
use lazaret_linux::filesystem::{RegularFileMailbox, mount_identity, namespace_identity};
use lazaret_linux::process::{
ProcessOwnershipContext, TargetCommand, child_broker, current_process_identity,
@ -76,6 +77,7 @@ pub struct ColdStartInputs<'a> {
pub generation: Generation,
pub prepared_artifact: &'a PreparedArtifact,
pub prepared_artifact_authority: &'a File,
pub prepared_artifact_selection: SharedFileLock,
pub prepared_artifact_label: &'a Path,
pub root_process: RootProcess,
pub container_id: ContainerId,
@ -135,6 +137,7 @@ pub struct ColdStart<'a> {
generation_identity: GenerationIdentity,
prepared_artifact: &'a PreparedArtifact,
state_anchor_layout: StateAnchorLayout<'a>,
prepared_artifact_selection: Option<SharedFileLock>,
runtime_launch: Option<RuntimeSupervisorLaunch>,
sandbox_bundle: SandboxBundle,
network_policy: lazaret_core::network::CanonicalDerivedNetworkPolicy,
@ -256,6 +259,7 @@ impl<'a> ColdStart<'a> {
generation_identity,
prepared_artifact: inputs.prepared_artifact,
state_anchor_layout,
prepared_artifact_selection: Some(inputs.prepared_artifact_selection),
runtime_launch: Some(runtime_launch),
sandbox_bundle,
network_policy,
@ -371,6 +375,7 @@ impl<'a> ColdStart<'a> {
configuration: self.configuration,
generation_identity: self.generation_identity.clone(),
prepared_artifact: self.prepared_artifact,
prepared_artifact_selection: self.prepared_artifact_selection.take(),
state_anchor_layout: &self.state_anchor_layout,
runtime_launch: self
.runtime_launch

43
crates/lazaret-host-coordinator/src/runtime_selector.rs
View file

@ -11,6 +11,7 @@ use lazaret_core::identity::{
PrepareOperationId, SandboxId,
};
use lazaret_core::records::{DurableDecision, Generation, PreparedArtifact, VolumeKind};
use lazaret_linux::filesystem::SharedFileLock;
use lazaret_linux::platform::{PlatformObservation, check_platform};
use lazaret_linux::user_namespace::UserNamespaceMap;
use lazaret_prepared_lower::{
@ -32,6 +33,8 @@ use crate::prepared_worker::prepare_in_user_namespace;
use crate::recovery::recover_before_cold_start;
use crate::sandbox_init::PRIVATE_SANDBOX_INIT_ROLE;
const PREPARED_SELECTION_ATTEMPTS: usize = 16;
#[derive(Debug)]
pub enum RuntimeSelectionError {
PreparedLower(PreparedLowerError),
@ -78,6 +81,7 @@ pub struct RuntimeSelection {
catalog: Catalog,
artifact: PreparedArtifact,
artifact_authority: File,
artifact_selection: Option<SharedFileLock>,
artifact_label: PathBuf,
generation: Generation,
container_id: ContainerId,
@ -92,7 +96,7 @@ impl RuntimeSelection {
/// # Errors
/// Refuses a configuration other than the exact one used for selection.
pub fn cold_start_inputs<'a>(
&'a self,
&'a mut self,
configuration: &'a ResolvedDomainConfiguration,
) -> Result<ColdStartInputs<'a>, RuntimeSelectionError> {
if configuration.domain_id() != self.domain_id
@ -107,6 +111,10 @@ impl RuntimeSelection {
generation: self.generation.clone(),
prepared_artifact: &self.artifact,
prepared_artifact_authority: &self.artifact_authority,
prepared_artifact_selection: self
.artifact_selection
.take()
.ok_or_else(|| failure("selected artifact claim was already consumed"))?,
prepared_artifact_label: &self.artifact_label,
root_process: RootProcess::new(
SANDBOX_INIT_DESTINATION,
@ -174,17 +182,27 @@ async fn select_runtime_for_platform(
)
.await
.map_err(RuntimeSelectionError::PreparedLower)?;
let artifact = prepare_in_user_namespace(
configuration,
&image,
PrepareOperationId::new(random_id()?),
prepared_destinations(configuration),
&user_namespace_map,
)
.map_err(|error| failure(format!("mapped preparation worker failed: {error}")))?;
let artifact_authority = prepared
.open_artifact_authority(&artifact)
.map_err(RuntimeSelectionError::PreparedLower)?;
let mut selection = None;
for _ in 0..PREPARED_SELECTION_ATTEMPTS {
let artifact = prepare_in_user_namespace(
configuration,
&image,
PrepareOperationId::new(random_id()?),
prepared_destinations(configuration),
&user_namespace_map,
)
.map_err(|error| failure(format!("mapped preparation worker failed: {error}")))?;
if let Some(selected) = prepared
.select_artifact(&artifact)
.map_err(RuntimeSelectionError::PreparedLower)?
{
selection = Some(selected);
break;
}
}
let selection = selection
.ok_or_else(|| failure("prepared artifact was repeatedly reclaimed before selection"))?;
let (artifact, artifact_authority, artifact_selection) = selection.into_parts();
let artifact_label = PathBuf::from(std::ffi::OsString::from_vec(
configuration
.storage_root()
@ -218,6 +236,7 @@ async fn select_runtime_for_platform(
catalog,
artifact,
artifact_authority,
artifact_selection: Some(artifact_selection),
artifact_label,
generation,
container_id: ContainerId::new(random_id()?),

6
crates/lazaret-host-coordinator/tests/cold_start.rs
View file

@ -13,7 +13,7 @@ use lazaret_core::identity::{
use lazaret_core::records::{ArtifactState, Domain, DurableDecision, Generation, PreparedArtifact};
use lazaret_host_coordinator::cold_start::{ColdStart, ColdStartInputs};
use lazaret_host_coordinator::domain_store::HostCoordinatorDomainStore;
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_linux::filesystem::{try_exclusive_flock, try_shared_flock};
use lazaret_linux::user_namespace::{
REQUIRED_SUBORDINATE_IDS, UserNamespaceIdRange, UserNamespaceMap,
};
@ -267,6 +267,10 @@ impl ColdStartFixture {
generation: self.generation.clone(),
prepared_artifact: &self.artifact,
prepared_artifact_authority: &self.artifact_authority,
prepared_artifact_selection: try_shared_flock(
File::open(&self.prepared_lower).unwrap(),
)
.unwrap(),
prepared_artifact_label: Path::new(
std::str::from_utf8(self.artifact.path.as_bytes()).unwrap(),
),

34
crates/lazaret-linux/src/filesystem.rs
View file

@ -351,6 +351,13 @@ pub fn try_exclusive_flock(file: File) -> io::Result<ExclusiveFileLock> {
Ok(ExclusiveFileLock(file))
}
/// # Errors
/// Returns an error when the nonblocking shared lock is unavailable.
pub fn try_shared_flock(file: File) -> io::Result<SharedFileLock> {
flock(file.as_fd(), FlockOperation::NonBlockingLockShared)?;
Ok(SharedFileLock(file))
}
#[derive(Debug)]
pub struct ExclusiveFileLock(File);
@ -360,6 +367,15 @@ impl AsFd for ExclusiveFileLock {
}
}
#[derive(Debug)]
pub struct SharedFileLock(File);
impl AsFd for SharedFileLock {
fn as_fd(&self) -> std::os::fd::BorrowedFd<'_> {
self.0.as_fd()
}
}
/// # Errors
/// Returns an error unless the replacement and containing directory are synced.
pub fn atomic_replace(path: &Path, bytes: &[u8]) -> io::Result<()> {
@ -459,7 +475,7 @@ pub fn require_mount_identity(path: &Path, expected: MountIdentity) -> io::Resul
#[cfg(test)]
mod tests {
use std::fs::OpenOptions;
use std::fs::{File, OpenOptions};
use std::io::{Read, Write};
use std::os::unix::fs::symlink;
@ -468,6 +484,7 @@ mod tests {
use super::{
RegularFileMailbox, atomic_replace, current_mount_identity, namespace_identity,
require_mount_identity, require_write_sealed, sealed_memfd, try_exclusive_flock,
try_shared_flock,
};
fn test_directory(name: &str) -> std::path::PathBuf {
@ -552,6 +569,21 @@ mod tests {
std::fs::remove_dir_all(directory).unwrap();
}
#[test]
fn shared_flock_excludes_reclaim_until_every_claim_closes() {
let directory = test_directory("shared-flock");
let path = directory.join("artifact");
std::fs::create_dir(&path).unwrap();
let first = try_shared_flock(File::open(&path).unwrap()).unwrap();
let second = try_shared_flock(File::open(&path).unwrap()).unwrap();
assert!(try_exclusive_flock(File::open(&path).unwrap()).is_err());
drop(first);
assert!(try_exclusive_flock(File::open(&path).unwrap()).is_err());
drop(second);
assert!(try_exclusive_flock(File::open(&path).unwrap()).is_ok());
std::fs::remove_dir_all(directory).unwrap();
}
#[test]
fn namespace_identity_is_stable_for_same_namespace() {
let first = namespace_identity(std::path::Path::new("/proc/self/ns/mnt")).unwrap();

3
crates/lazaret-prepared-lower/src/lib.rs
View file

@ -10,6 +10,7 @@ pub use layer::{
IdMap, IdMapRange, PreparationLimits, PreparedDestination, PreparedDestinationKind,
};
pub use store::{
PrepareRecovery, PrepareRequest, PreparedArtifactGcResult, PreparedLowerStore, ReclaimOutcome,
PrepareRecovery, PrepareRequest, PreparedArtifactGcResult, PreparedArtifactSelection,
PreparedLowerStore, ReclaimOutcome,
};
pub use transport::{OciRegistryTransport, Platform, RegistryCredentials, TransportedImage};

123
crates/lazaret-prepared-lower/src/store.rs
View file

@ -14,7 +14,9 @@ use lazaret_core::identity::{
SelectedManifestDigest, SourceIdentity,
};
use lazaret_core::records::{ArtifactState, PrepareOperation, PrepareStageState, PreparedArtifact};
use lazaret_linux::filesystem::{ExclusiveFileLock, try_exclusive_flock};
use lazaret_linux::filesystem::{
ExclusiveFileLock, SharedFileLock, try_exclusive_flock, try_shared_flock,
};
use lazaret_storage_catalog::{Catalog, CatalogError, CatalogRow};
use rustix::fs::{
AtFlags, Mode, OFlags, RenameFlags, ResolveFlags, mkdirat, openat2, renameat, renameat_with,
@ -56,6 +58,7 @@ pub enum PrepareRecovery {
pub enum ReclaimOutcome {
Removed,
ReplacementPreserved,
RetainedBySelection,
}
#[derive(Clone, Debug, Eq, PartialEq)]
@ -64,6 +67,25 @@ pub struct PreparedArtifactGcResult {
pub outcome: ReclaimOutcome,
}
#[derive(Debug)]
pub struct PreparedArtifactSelection {
artifact: PreparedArtifact,
authority: File,
claim: SharedFileLock,
}
impl PreparedArtifactSelection {
#[must_use]
pub fn artifact(&self) -> &PreparedArtifact {
&self.artifact
}
#[must_use]
pub fn into_parts(self) -> (PreparedArtifact, File, SharedFileLock) {
(self.artifact, self.authority, self.claim)
}
}
#[derive(Clone, Debug)]
pub struct PreparedLowerStore {
root: Arc<File>,
@ -178,6 +200,52 @@ impl PreparedLowerStore {
.context("prepared artifact path is absent")
}
/// Selects the current equivalent artifact and retains a claim that excludes reclaim.
///
/// # Errors
/// Refuses semantic identity drift, catalog inconsistency, or physical identity drift.
pub fn select_artifact(
&self,
expected: &PreparedArtifact,
) -> PreparedLowerResult<Option<PreparedArtifactSelection>> {
let physical_lock = self.physical_lock()?;
self.select_artifact_inner(expected, &physical_lock)
.map_err(PreparedLowerError::confinement)
}
fn select_artifact_inner(
&self,
expected: &PreparedArtifact,
_physical_lock: &ExclusiveFileLock,
) -> Result<Option<PreparedArtifactSelection>> {
let Some(artifact) = self.available_artifact(expected.prepared_artifact_id)? else {
return Ok(None);
};
require_equivalent_artifact(expected, &artifact)?;
let path = artifact_path(artifact.prepared_artifact_id);
let authority = self
.open_relative(&path)?
.context("selected artifact path is absent")?;
require_identity(&authority, artifact.device, artifact.inode)?;
let claim_file = self
.open_relative(&path)?
.context("selected artifact claim path is absent")?;
require_identity(&claim_file, artifact.device, artifact.inode)?;
let claim = try_shared_flock(claim_file)?;
if self
.catalog_artifact(artifact.prepared_artifact_id)?
.as_ref()
!= Some(&artifact)
{
bail!("selected artifact catalog ownership changed");
}
Ok(Some(PreparedArtifactSelection {
artifact,
authority,
claim,
}))
}
/// Materializes and atomically publishes one verified immutable prepared lower.
///
/// # Errors
@ -498,11 +566,26 @@ impl PreparedLowerStore {
/// Refuses leases, forged catalog rows, physical identity mismatch, foreign paths, and
/// filesystem or catalog failures.
pub fn reclaim(&self, expected: &PreparedArtifact) -> PreparedLowerResult<ReclaimOutcome> {
let _physical_lock = self.physical_lock()?;
let artifact = self
.open_relative(&artifact_path(expected.prepared_artifact_id))
.map_err(PreparedLowerError::reclaim)?
.ok_or_else(|| PreparedLowerError::reclaim(anyhow::anyhow!("artifact is absent")))?;
require_identity(&artifact, expected.device, expected.inode)
.map_err(PreparedLowerError::reclaim)?;
let _claim = match try_exclusive_flock(artifact) {
Ok(claim) => claim,
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
return Ok(ReclaimOutcome::RetainedBySelection);
}
Err(error) => return Err(PreparedLowerError::reclaim(error)),
};
let reclaiming = self
.catalog
.begin_reclaim(expected)
.map_err(PreparedLowerError::catalog)?;
self.resume_reclaim(&reclaiming)
self.resume_reclaim_inner(&reclaiming)
.map_err(PreparedLowerError::reclaim)
}
/// Resumes an exact durable reclaim after interruption.
@ -516,10 +599,28 @@ impl PreparedLowerStore {
reclaiming: &PreparedArtifact,
) -> PreparedLowerResult<ReclaimOutcome> {
let _physical_lock = self.physical_lock()?;
let _claim = self
.lock_reclaiming_artifact(reclaiming)
.map_err(PreparedLowerError::reclaim)?;
self.resume_reclaim_inner(reclaiming)
.map_err(PreparedLowerError::reclaim)
}
fn lock_reclaiming_artifact(&self, reclaiming: &PreparedArtifact) -> Result<ExclusiveFileLock> {
for path in [
artifact_path(reclaiming.prepared_artifact_id),
quarantine_path(reclaiming),
] {
let Some(file) = self.open_relative(&path)? else {
continue;
};
if file_identity(&file)? == (reclaiming.device, reclaiming.inode) {
return Ok(try_exclusive_flock(file)?);
}
}
bail!("reclaiming artifact and quarantine are both absent")
}
fn resume_reclaim_inner(&self, reclaiming: &PreparedArtifact) -> Result<ReclaimOutcome> {
if reclaiming.state != ArtifactState::Reclaiming {
bail!("artifact is not reclaiming");
@ -903,6 +1004,24 @@ fn artifact_from_operation(
}
}
fn require_equivalent_artifact(
expected: &PreparedArtifact,
observed: &PreparedArtifact,
) -> Result<()> {
if expected.state != ArtifactState::Available
|| expected.prepared_artifact_id != observed.prepared_artifact_id
|| expected.source_identity != observed.source_identity
|| expected.selected_manifest_digest != observed.selected_manifest_digest
|| expected.policy_id != observed.policy_id
|| expected.id_map_digest != observed.id_map_digest
|| expected.path != observed.path
|| observed.state != ArtifactState::Available
{
bail!("selected prepared artifact identity inputs differ");
}
Ok(())
}
fn file_identity(file: &File) -> Result<(DeviceId, InodeId)> {
let stat = rustix::fs::fstat(file)?;
Ok((DeviceId::new(stat.st_dev), InodeId::new(stat.st_ino)))