ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Close hostile recovery and concurrency gaps

Browse source
This commit is contained in:
ms2k 2026-07-18 05:47:15 +09:00
commit 3f9b1b3543
1 changed files with 306 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

49
crates/lazaret-host-coordinator/src/recovery.rs
View file

@ -630,7 +630,9 @@ where
{
recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
let snapshot = store.snapshot().map_err(|error| error.to_string())?;
let snapshot = store
.snapshot()
.map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
if snapshot.domain.domain_id != configuration.domain_id() {
return Err("recovery Domain differs from resolved configuration".into());
}
@ -651,7 +653,13 @@ where
&mut failpoint,
&mut open,
&mut delete,
)?;
)
.map_err(|error| {
format!(
"generation {} recovery failed: {error}",
generation.generation_id.get()
)
})?;
}
failpoint(
ProductionRecoveryBoundary::PrepareOperations,
@ -725,26 +733,34 @@ where
if !session_updates.is_empty() {
store
.apply(&session_updates)
.map_err(|error| error.to_string())?;
.map_err(|error| format!("Session failure update failed: {error}"))?;
}
let participants = store
.snapshot()
.map_err(|error| error.to_string())?
.map_err(|error| format!("Participant snapshot failed: {error}"))?
.participants;
let ownership = runtime_recovery_ownership(&participants, generation)?;
let mut owners = seal_runtime_owners(&ownership, open).map_err(|error| error.to_string())?;
let ownership = runtime_recovery_ownership(&participants, generation)
.map_err(|error| format!("runtime ownership inventory failed: {error}"))?;
let mut owners = seal_runtime_owners(&ownership, open)
.map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
recover_publication(configuration, store, generation_id)?;
recover_publication(configuration, store, generation_id)
.map_err(|error| format!("Publication recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
reconcile_runtime_before_state_anchor(&mut owners).map_err(|error| error.to_string())?;
require_sandbox_absent(configuration, store, generation_id, delete)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)?;
reconcile_runtime_before_state_anchor(&mut owners)
.map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
require_sandbox_absent(configuration, store, generation_id, delete)
.map_err(|error| format!("Sandbox absence failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
.map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Network)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
.map_err(|error| format!("Helper cleanup failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Network)
.map_err(|error| format!("Network cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
require_mount_absent(
@ -753,13 +769,16 @@ where
"Git-routing",
)
.map_err(|error| error.to_string())?;
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)?;
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
.map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
recover_state_anchor(configuration, store, catalog, generation, &mut owners)?;
recover_state_anchor(configuration, store, catalog, generation, &mut owners)
.map_err(|error| format!("State Anchor recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
recover_partial_prepared_lower(configuration, store, catalog, generation_id)?;
recover_partial_prepared_lower(configuration, store, catalog, generation_id)
.map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
let snapshot = store.snapshot().map_err(|error| error.to_string())?;
if snapshot.participants.iter().any(|participant| {