ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Preserve live prepared-lower materializations during recovery

Browse source
This commit is contained in:
ms2k 2026-07-21 23:47:29 +09:00
commit 85f0db08d7
1 changed files with 205 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

135
crates/lazaret-prepared-lower/src/store.rs
View file

@ -47,6 +47,7 @@ pub struct PrepareRequest {
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PrepareRecovery {
ClearedUnmaterializedIntent,
PreservedLiveOwnedStage,
RemovedOwnedStage,
PublishedCompletedStage,
}
@ -267,47 +268,67 @@ impl PreparedLowerStore {
)));
}
let identities = Identities::new(request).map_err(PreparedLowerError::layer_validation)?;
if let Some(existing) = self
.available_artifact(identities.artifact_id)
.map_err(PreparedLowerError::catalog)?
{
identities
.require_equivalent(&existing)
.map_err(PreparedLowerError::catalog)?;
self.require_artifact_identity(&existing)
.map_err(PreparedLowerError::confinement)?;
return Ok(existing);
}
let stage_relative = stage_path(request.operation_id);
let intent = identities.intent(request.operation_id, &stage_relative);
self.catalog
.record_prepare(&intent)
.map_err(PreparedLowerError::catalog)?;
let stage = match self.create_stage(&stage_relative) {
Ok(stage) => stage,
Err(error) => {
let _ = self.catalog.remove_prepare(&intent);
return Err(PreparedLowerError::materialization(error));
}
};
let (device, inode) =
file_identity(&stage).map_err(PreparedLowerError::operating_system)?;
let owned = PrepareOperation {
stage_state: PrepareStageState::Owned,
device: Some(device),
inode: Some(inode),
..intent.clone()
};
if let Err(error) = self.catalog.bind_prepare_stage(&intent, &owned) {
if self
.remove_exact_path(&stage_relative, device, inode)
.is_ok()
let (stage_relative, stage, owned, device, inode, _stage_lock) = {
let _physical_lock = self.physical_lock()?;
if let Some(existing) = self
.available_artifact(identities.artifact_id)
.map_err(PreparedLowerError::catalog)?
{
let _ = self.catalog.remove_prepare(&intent);
identities
.require_equivalent(&existing)
.map_err(PreparedLowerError::catalog)?;
self.require_artifact_identity(&existing)
.map_err(PreparedLowerError::confinement)?;
return Ok(existing);
}
return Err(PreparedLowerError::catalog(error));
}
let stage_relative = stage_path(request.operation_id);
let intent = identities.intent(request.operation_id, &stage_relative);
self.catalog
.record_prepare(&intent)
.map_err(PreparedLowerError::catalog)?;
let stage = match self.create_stage(&stage_relative) {
Ok(stage) => stage,
Err(error) => {
let _ = self.catalog.remove_prepare(&intent);
return Err(PreparedLowerError::materialization(error));
}
};
let (device, inode) =
file_identity(&stage).map_err(PreparedLowerError::operating_system)?;
let stage_lock = match try_exclusive_flock(
stage
.try_clone()
.map_err(PreparedLowerError::operating_system)?,
) {
Ok(lock) => lock,
Err(error) => {
if self
.remove_exact_path(&stage_relative, device, inode)
.is_ok()
{
let _ = self.catalog.remove_prepare(&intent);
}
return Err(PreparedLowerError::locking(error));
}
};
let owned = PrepareOperation {
stage_state: PrepareStageState::Owned,
device: Some(device),
inode: Some(inode),
..intent.clone()
};
if let Err(error) = self.catalog.bind_prepare_stage(&intent, &owned) {
if self
.remove_exact_path(&stage_relative, device, inode)
.is_ok()
{
let _ = self.catalog.remove_prepare(&intent);
}
return Err(PreparedLowerError::catalog(error));
}
(stage_relative, stage, owned, device, inode, stage_lock)
};
let materialization_map = if materialize_logical_owners {
logical_id_map(&request.id_map)
} else {
@ -320,9 +341,10 @@ impl PreparedLowerStore {
})
.and_then(|()| sync_tree(&stage))
{
if self
.remove_exact_path(&stage_relative, device, inode)
.is_ok()
if let Ok(_physical_lock) = self.physical_lock()
&& self
.remove_exact_path(&stage_relative, device, inode)
.is_ok()
{
let _ = self.catalog.remove_prepare(&owned);
}
@ -343,11 +365,16 @@ impl PreparedLowerStore {
&self,
operation: &PrepareOperation,
) -> PreparedLowerResult<PrepareRecovery> {
self.recover_prepare_inner(operation)
let physical_lock = self.physical_lock()?;
self.recover_prepare_inner(operation, &physical_lock)
.map_err(PreparedLowerError::recovery)
}
fn recover_prepare_inner(&self, operation: &PrepareOperation) -> Result<PrepareRecovery> {
fn recover_prepare_inner(
&self,
operation: &PrepareOperation,
_physical_lock: &ExclusiveFileLock,
) -> Result<PrepareRecovery> {
let expected_stage = stage_path(operation.operation_id);
if operation.stage_path.as_bytes() != path_bytes(&expected_stage) {
bail!("prepare operation stage path is not canonical");
@ -363,7 +390,15 @@ impl PreparedLowerStore {
PrepareStageState::Owned => {
let device = operation.device.context("owned prepare lacks device")?;
let inode = operation.inode.context("owned prepare lacks inode")?;
if self.open_relative(&expected_stage)?.is_some() {
if let Some(stage) = self.open_relative(&expected_stage)? {
require_identity(&stage, device, inode)?;
let _stage_lock = match try_exclusive_flock(stage) {
Ok(lock) => lock,
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
return Ok(PrepareRecovery::PreservedLiveOwnedStage);
}
Err(error) => return Err(error.into()),
};
self.remove_exact_path(&expected_stage, device, inode)?;
self.catalog.remove_prepare(operation)?;
return Ok(PrepareRecovery::RemovedOwnedStage);
@ -391,16 +426,20 @@ impl PreparedLowerStore {
pub fn recover_preparations(
&self,
) -> PreparedLowerResult<Vec<(PrepareOperationId, PrepareRecovery)>> {
self.recover_preparations_inner()
let physical_lock = self.physical_lock()?;
self.recover_preparations_inner(&physical_lock)
.map_err(PreparedLowerError::recovery)
}
fn recover_preparations_inner(&self) -> Result<Vec<(PrepareOperationId, PrepareRecovery)>> {
fn recover_preparations_inner(
&self,
physical_lock: &ExclusiveFileLock,
) -> Result<Vec<(PrepareOperationId, PrepareRecovery)>> {
let operations = self.catalog.snapshot()?.prepare_operations;
operations
.iter()
.map(|operation| {
self.recover_prepare_inner(operation)
self.recover_prepare_inner(operation, physical_lock)
.map(|outcome| (operation.operation_id, outcome))
})
.collect()