ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Remove phase vocabulary from Lazaret interfaces

Browse source
This commit is contained in:
ms2k 2026-07-18 22:35:48 +09:00
commit e7dd95aea6
1 changed files with 547 additions and 579 deletions
Download patch file Download diff file Expand all files Collapse all files

117
crates/lazaret-host-coordinator/src/recovery.rs
View file

@ -565,7 +565,7 @@ fn require_mount_absent(
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ProductionRecoveryBoundary {
pub(crate) enum RecoveryBoundary {
Snapshot,
Publication,
Sandbox,
@ -578,22 +578,16 @@ pub(crate) enum ProductionRecoveryBoundary {
FinishGeneration,
}
pub(crate) fn recover_production_before_cold_start(
pub(crate) fn recover_before_cold_start(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
prepared: &PreparedLowerStore,
) -> Result<(), String> {
recover_production_before_cold_start_with(
configuration,
store,
catalog,
prepared,
|_, _| Ok(()),
)
recover_before_cold_start_with(configuration, store, catalog, prepared, |_, _| Ok(()))
}
fn recover_production_before_cold_start_with<Failpoint>(
fn recover_before_cold_start_with<Failpoint>(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
@ -601,9 +595,9 @@ fn recover_production_before_cold_start_with<Failpoint>(
failpoint: Failpoint,
) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
{
recover_production_before_cold_start_with_opener(
recover_before_cold_start_with_opener(
configuration,
store,
catalog,
@ -614,7 +608,7 @@ where
)
}
fn recover_production_before_cold_start_with_opener<Failpoint, Open, Delete>(
fn recover_before_cold_start_with_opener<Failpoint, Open, Delete>(
configuration: &ResolvedDomainConfiguration,
store: &HostCoordinatorDomainStore,
catalog: &Catalog,
@ -624,12 +618,12 @@ fn recover_production_before_cold_start_with_opener<Failpoint, Open, Delete>(
mut delete: Delete,
) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
failpoint(RecoveryBoundary::Snapshot, GenerationId::new(0))?;
let snapshot = store
.snapshot()
.map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
@ -661,14 +655,11 @@ where
)
})?;
}
failpoint(
ProductionRecoveryBoundary::PrepareOperations,
GenerationId::new(0),
)?;
failpoint(RecoveryBoundary::PrepareOperations, GenerationId::new(0))?;
prepared
.recover_preparations()
.map_err(|error| error.to_string())?;
failpoint(ProductionRecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
failpoint(RecoveryBoundary::ArtifactGc, GenerationId::new(0))?;
let catalog_snapshot = catalog.snapshot().map_err(|error| error.to_string())?;
let selected = catalog_snapshot
.prepared_artifacts
@ -697,7 +688,7 @@ fn recover_generation<Failpoint, Open, Delete>(
delete: &mut Delete,
) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
Open: FnMut(PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch>,
Delete: FnMut(&std::os::fd::OwnedFd, &str) -> Result<(), String>,
{
@ -744,11 +735,11 @@ where
let mut owners = seal_runtime_owners(&ownership, open)
.map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
failpoint(RecoveryBoundary::Publication, generation_id)?;
recover_publication(configuration, store, generation_id)
.map_err(|error| format!("Publication recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
failpoint(RecoveryBoundary::Sandbox, generation_id)?;
reconcile_runtime_before_state_anchor(&mut owners)
.map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
require_sandbox_absent(configuration, store, generation_id, delete)
@ -756,13 +747,13 @@ where
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
.map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
failpoint(RecoveryBoundary::HelperAndNetwork, generation_id)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
.map_err(|error| format!("Helper cleanup failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Network)
.map_err(|error| format!("Network cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
failpoint(RecoveryBoundary::GitRouting, generation_id)?;
require_mount_absent(
ownership.git_routing_path.as_ref(),
ownership.git_routing_mount,
@ -772,11 +763,11 @@ where
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
.map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
failpoint(RecoveryBoundary::StateAnchor, generation_id)?;
recover_state_anchor(configuration, store, catalog, generation, &mut owners)
.map_err(|error| format!("State Anchor recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
failpoint(RecoveryBoundary::PreparedLower, generation_id)?;
recover_partial_prepared_lower(configuration, store, catalog, generation_id)
.map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
@ -787,7 +778,7 @@ where
}) {
return Err("generation retains non-Clean participants after recovery".into());
}
failpoint(ProductionRecoveryBoundary::FinishGeneration, generation_id)?;
failpoint(RecoveryBoundary::FinishGeneration, generation_id)?;
let mut clean = generation.clone();
clean.durable_decision = DurableDecision::Clean;
store
@ -1341,7 +1332,7 @@ mod tests {
SandboxOwnership, StateAnchorOwnership,
};
use lazaret_linux::filesystem::try_exclusive_flock;
use lazaret_runtime_supervisor::sandbox::production_sandbox_layout;
use lazaret_runtime_supervisor::sandbox::default_sandbox_layout;
use super::*;
@ -1504,8 +1495,8 @@ mod tests {
}
#[test]
fn production_preflight_recovers_full_runtime_then_is_idempotent() {
let fixture = ProductionRecoveryFixture::new();
fn recovery_preflight_recovers_full_runtime_then_is_idempotent() {
let fixture = RecoveryFixture::new();
let mut boundaries = Vec::new();
fixture
.recover_with(|boundary, generation| {
@ -1516,7 +1507,7 @@ mod tests {
assert!(
boundaries
.iter()
.any(|(boundary, _)| { *boundary == ProductionRecoveryBoundary::StateAnchor })
.any(|(boundary, _)| { *boundary == RecoveryBoundary::StateAnchor })
);
assert_eq!(
fixture.store.snapshot().unwrap().generations[0].durable_decision,
@ -1552,7 +1543,7 @@ mod tests {
.exists()
);
recover_production_before_cold_start(
recover_before_cold_start(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
@ -1570,12 +1561,12 @@ mod tests {
#[test]
fn bare_publication_intent_requires_every_fixed_label_to_be_absent() {
let absent = ProductionRecoveryFixture::new();
let absent = RecoveryFixture::new();
absent.prepare_bare_publication(PublicationFixtureBoundary::TrulyAbsent);
recover_publication(&absent.configuration, &absent.store, GenerationId::new(2)).unwrap();
assert_eq!(absent.publication(2).state(), ParticipantState::Clean);
let identity = ProductionRecoveryFixture::new();
let identity = RecoveryFixture::new();
identity.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedIdentity);
assert!(
recover_publication(
@ -1594,7 +1585,7 @@ mod tests {
assert_eq!(identity.publication(2).state(), ParticipantState::Intent);
assert!(identity.publication(2).ownership().is_none());
let socket = ProductionRecoveryFixture::new();
let socket = RecoveryFixture::new();
socket.prepare_bare_publication(PublicationFixtureBoundary::AfterStagedSocket);
assert!(
recover_publication(&socket.configuration, &socket.store, GenerationId::new(2))
@ -1617,27 +1608,27 @@ mod tests {
}
#[test]
fn production_preflight_retries_every_recovery_boundary() {
fn recovery_preflight_retries_every_recovery_boundary() {
for failed in [
ProductionRecoveryBoundary::Snapshot,
ProductionRecoveryBoundary::Publication,
ProductionRecoveryBoundary::Sandbox,
ProductionRecoveryBoundary::HelperAndNetwork,
ProductionRecoveryBoundary::GitRouting,
ProductionRecoveryBoundary::StateAnchor,
ProductionRecoveryBoundary::PreparedLower,
ProductionRecoveryBoundary::PrepareOperations,
ProductionRecoveryBoundary::ArtifactGc,
ProductionRecoveryBoundary::FinishGeneration,
RecoveryBoundary::Snapshot,
RecoveryBoundary::Publication,
RecoveryBoundary::Sandbox,
RecoveryBoundary::HelperAndNetwork,
RecoveryBoundary::GitRouting,
RecoveryBoundary::StateAnchor,
RecoveryBoundary::PreparedLower,
RecoveryBoundary::PrepareOperations,
RecoveryBoundary::ArtifactGc,
RecoveryBoundary::FinishGeneration,
] {
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let mut injected = false;
assert!(
fixture
.recover_with(|boundary, _| {
if boundary == failed && !injected {
injected = true;
Err("injected production recovery boundary".into())
Err("injected recovery boundary".into())
} else {
Ok(())
}
@ -1654,10 +1645,10 @@ mod tests {
}
#[test]
fn production_preflight_opens_every_exact_owner_once() {
let fixture = ProductionRecoveryFixture::new();
fn recovery_preflight_opens_every_exact_owner_once() {
let fixture = RecoveryFixture::new();
let mut opened = BTreeMap::new();
recover_production_before_cold_start_with_opener(
recover_before_cold_start_with_opener(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
@ -1678,7 +1669,7 @@ mod tests {
#[test]
fn publication_replacement_and_runsc_root_replacement_refuse_untouched() {
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1701,7 +1692,7 @@ mod tests {
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert!(socket.exists());
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1720,7 +1711,7 @@ mod tests {
.is_symlink()
);
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1734,7 +1725,7 @@ mod tests {
assert!(fixture.recover_with(|_, _| Ok(())).is_err());
assert_eq!(fs::read(&identity).unwrap(), b"replacement");
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1747,7 +1738,7 @@ mod tests {
fs::rename(&runsc, &old_runsc).unwrap();
fs::create_dir(&runsc).unwrap();
let mut deletes = 0;
let result = recover_production_before_cold_start_with_opener(
let result = recover_before_cold_start_with_opener(
&fixture.configuration,
&fixture.store,
&fixture.catalog,
@ -1763,7 +1754,7 @@ mod tests {
assert_eq!(deletes, 0);
assert!(runsc.exists());
let fixture = ProductionRecoveryFixture::new();
let fixture = RecoveryFixture::new();
let runtime_domain = Path::new(std::ffi::OsStr::from_bytes(
fixture
.configuration
@ -1783,7 +1774,7 @@ mod tests {
);
}
struct ProductionRecoveryFixture {
struct RecoveryFixture {
_tree: tempfile::TempDir,
configuration: ResolvedDomainConfiguration,
store: HostCoordinatorDomainStore,
@ -1791,12 +1782,12 @@ mod tests {
prepared: PreparedLowerStore,
}
impl ProductionRecoveryFixture {
impl RecoveryFixture {
fn recover_with<Failpoint>(&self, failpoint: Failpoint) -> Result<(), String>
where
Failpoint: FnMut(ProductionRecoveryBoundary, GenerationId) -> Result<(), String>,
Failpoint: FnMut(RecoveryBoundary, GenerationId) -> Result<(), String>,
{
recover_production_before_cold_start_with_opener(
recover_before_cold_start_with_opener(
&self.configuration,
&self.store,
&self.catalog,
@ -2221,7 +2212,7 @@ mod tests {
xdg_data_home: None,
xdg_runtime_dir: runtime,
environment: BTreeMap::new(),
sandbox_layout: production_sandbox_layout().unwrap(),
sandbox_layout: default_sandbox_layout().unwrap(),
})
.unwrap()
}