Implement exact bootstrap recovery and artifact GC
This commit is contained in:
parent
0f4845d4f8
commit
d25942596f
1 changed files with 1631 additions and 48 deletions
297
crates/lazaret-host-coordinator/src/recovery.rs
Normal file
297
crates/lazaret-host-coordinator/src/recovery.rs
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
use lazaret_core::identity::{
|
||||
GenerationId, MountIdentity, NamespaceIdentity, PidfsProcessIdentityV1,
|
||||
};
|
||||
use lazaret_core::records::{
|
||||
Participant, ParticipantOwnership, ParticipantRole, PersistedHostProcessOwnership,
|
||||
};
|
||||
use lazaret_linux::process::{PidfsRecoveryMatch, match_recovery_process};
|
||||
|
||||
use crate::state_anchor::RuntimeOwnersAbsent;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub struct RuntimeRecoveryOwnership {
|
||||
pub host_coordinator: Option<PidfsProcessIdentityV1>,
|
||||
pub runtime_supervisor: Option<PidfsProcessIdentityV1>,
|
||||
pub attached_runsc: Option<PidfsProcessIdentityV1>,
|
||||
pub connectivity_helper: Option<PidfsProcessIdentityV1>,
|
||||
pub state_anchor_daemon: Option<PidfsProcessIdentityV1>,
|
||||
pub git_routing_daemon: Option<PidfsProcessIdentityV1>,
|
||||
pub state_anchor_user_namespace: Option<NamespaceIdentity>,
|
||||
pub state_anchor_mount_namespace: Option<NamespaceIdentity>,
|
||||
pub state_anchor_mount: Option<MountIdentity>,
|
||||
}
|
||||
|
||||
/// Extracts one generation's exact process and private-mount recovery authority.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses unequal copies of the constitutive HC or RS identities, and incomplete State Anchor
|
||||
/// namespace/mount evidence. Staged identities remain cleanup authority.
|
||||
pub fn runtime_recovery_ownership(
|
||||
participants: &[Participant],
|
||||
generation_id: GenerationId,
|
||||
) -> Result<RuntimeRecoveryOwnership, &'static str> {
|
||||
let mut result = RuntimeRecoveryOwnership::default();
|
||||
for participant in participants
|
||||
.iter()
|
||||
.filter(|participant| participant.generation_id() == generation_id)
|
||||
{
|
||||
let Some(ownership) = participant.ownership() else {
|
||||
continue;
|
||||
};
|
||||
match ownership {
|
||||
ParticipantOwnership::Sandbox(sandbox) => {
|
||||
merge_process(
|
||||
&mut result.host_coordinator,
|
||||
&sandbox.host_coordinator,
|
||||
"Sandbox HC identity mismatch",
|
||||
)?;
|
||||
merge_process(
|
||||
&mut result.runtime_supervisor,
|
||||
&sandbox.runtime_supervisor,
|
||||
"Sandbox RS identity mismatch",
|
||||
)?;
|
||||
merge_process(
|
||||
&mut result.attached_runsc,
|
||||
&sandbox.attached_runsc,
|
||||
"attached runsc identity mismatch",
|
||||
)?;
|
||||
}
|
||||
ParticipantOwnership::Network(network) => merge_process(
|
||||
&mut result.runtime_supervisor,
|
||||
&network.runtime_supervisor,
|
||||
"Network RS identity mismatch",
|
||||
)?,
|
||||
ParticipantOwnership::Helper(helper) => {
|
||||
merge_process(
|
||||
&mut result.connectivity_helper,
|
||||
&helper.connectivity_helper,
|
||||
"connectivity-helper identity mismatch",
|
||||
)?;
|
||||
merge_process(
|
||||
&mut result.runtime_supervisor,
|
||||
&helper.runtime_supervisor,
|
||||
"Helper RS identity mismatch",
|
||||
)?;
|
||||
}
|
||||
ParticipantOwnership::StateAnchor(anchor) => {
|
||||
merge_process(
|
||||
&mut result.state_anchor_daemon,
|
||||
&anchor.daemon,
|
||||
"State Anchor daemon identity mismatch",
|
||||
)?;
|
||||
let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
|
||||
if evidence == (None, None, None) {
|
||||
continue;
|
||||
}
|
||||
let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
|
||||
return Err("State Anchor recovery evidence is incomplete");
|
||||
};
|
||||
merge_value(
|
||||
&mut result.state_anchor_user_namespace,
|
||||
user,
|
||||
"State Anchor user namespace mismatch",
|
||||
)?;
|
||||
merge_value(
|
||||
&mut result.state_anchor_mount_namespace,
|
||||
mount_namespace,
|
||||
"State Anchor mount namespace mismatch",
|
||||
)?;
|
||||
merge_value(
|
||||
&mut result.state_anchor_mount,
|
||||
mount,
|
||||
"State Anchor mount identity mismatch",
|
||||
)?;
|
||||
}
|
||||
ParticipantOwnership::GitRouting(routing) => merge_process(
|
||||
&mut result.git_routing_daemon,
|
||||
&routing.daemon,
|
||||
"Git-routing daemon identity mismatch",
|
||||
)?,
|
||||
ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn merge_process(
|
||||
target: &mut Option<PidfsProcessIdentityV1>,
|
||||
observed: &PersistedHostProcessOwnership,
|
||||
mismatch: &'static str,
|
||||
) -> Result<(), &'static str> {
|
||||
let Ok(observed) = observed.authoritative_pidfs() else {
|
||||
return Ok(());
|
||||
};
|
||||
merge_value(target, *observed, mismatch)
|
||||
}
|
||||
|
||||
fn merge_value<T: Copy + Eq>(
|
||||
target: &mut Option<T>,
|
||||
observed: T,
|
||||
mismatch: &'static str,
|
||||
) -> Result<(), &'static str> {
|
||||
match target {
|
||||
None => *target = Some(observed),
|
||||
Some(current) if *current == observed => {}
|
||||
Some(_) => return Err(mismatch),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn recovery_role_order() -> [ParticipantRole; 7] {
|
||||
[
|
||||
ParticipantRole::Publication,
|
||||
ParticipantRole::Sandbox,
|
||||
ParticipantRole::Helper,
|
||||
ParticipantRole::Network,
|
||||
ParticipantRole::GitRouting,
|
||||
ParticipantRole::StateAnchor,
|
||||
ParticipantRole::PreparedLower,
|
||||
]
|
||||
}
|
||||
|
||||
/// Proves the constitutive RS and attached runsc are absent before private-mount reconciliation.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses a live or stopped exact authority and every unreadable or ambiguous observation.
|
||||
pub fn prove_runtime_owners_absent(
|
||||
generation_id: GenerationId,
|
||||
ownership: &RuntimeRecoveryOwnership,
|
||||
) -> Result<RuntimeOwnersAbsent, std::io::Error> {
|
||||
for identity in [ownership.attached_runsc, ownership.runtime_supervisor]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
{
|
||||
match match_recovery_process(identity) {
|
||||
Ok(PidfsRecoveryMatch::BootMismatch) => {}
|
||||
Ok(PidfsRecoveryMatch::Matched(process)) if process.has_exited()? => {}
|
||||
Ok(PidfsRecoveryMatch::Matched(_)) => {
|
||||
return Err(std::io::Error::other(
|
||||
"constitutive runtime authority remains present",
|
||||
));
|
||||
}
|
||||
Err(error) if error.raw_os_error() == Some(rustix::io::Errno::SRCH.raw_os_error()) => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Ok(RuntimeOwnersAbsent::for_generation(generation_id))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use lazaret_core::identity::{
|
||||
BootId, BundleDigest, ContainerId, DeviceId, HostPath, HostPidLocator, MountId,
|
||||
PidfsDeviceId, PidfsInodeId, PidfsProcessAuthorityV1, ProcessStartTimeTicks,
|
||||
};
|
||||
use lazaret_core::records::{
|
||||
HelperOwnership, ParticipantState, SandboxOwnership, StateAnchorOwnership,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn process(value: u64) -> PidfsProcessIdentityV1 {
|
||||
PidfsProcessIdentityV1::new(
|
||||
PidfsProcessAuthorityV1 {
|
||||
boot_id: BootId::new([1; 16]),
|
||||
pidfs_st_dev: PidfsDeviceId::new(value),
|
||||
pidfs_st_ino: PidfsInodeId::new(value),
|
||||
},
|
||||
HostPidLocator::new(u32::try_from(value).unwrap()),
|
||||
ProcessStartTimeTicks::new(value),
|
||||
)
|
||||
}
|
||||
|
||||
fn participant(role: ParticipantRole, ownership: ParticipantOwnership) -> Participant {
|
||||
Participant::snapshot(
|
||||
GenerationId::new(1),
|
||||
role,
|
||||
ParticipantState::Intent,
|
||||
Some(ownership),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inventory_accepts_staged_cleanup_authority_and_rejects_cross_role_rs_mismatch() {
|
||||
let rs = process(2);
|
||||
let sandbox = participant(
|
||||
ParticipantRole::Sandbox,
|
||||
ParticipantOwnership::Sandbox(SandboxOwnership {
|
||||
container_id: Some(ContainerId::new([3; 32])),
|
||||
runtime_root: Some(HostPath::new(b"/runtime".to_vec())),
|
||||
bundle_digest: Some(BundleDigest::new([4; 32])),
|
||||
host_coordinator: PersistedHostProcessOwnership::committed(process(1)),
|
||||
runtime_supervisor: PersistedHostProcessOwnership::staged(rs),
|
||||
attached_runsc: PersistedHostProcessOwnership::staged(process(3)),
|
||||
}),
|
||||
);
|
||||
let helper = |runtime_supervisor| {
|
||||
participant(
|
||||
ParticipantRole::Helper,
|
||||
ParticipantOwnership::Helper(HelperOwnership {
|
||||
network_sidecar_id: None,
|
||||
connectivity_helper: PersistedHostProcessOwnership::staged(process(4)),
|
||||
runtime_supervisor: PersistedHostProcessOwnership::committed(
|
||||
runtime_supervisor,
|
||||
),
|
||||
}),
|
||||
)
|
||||
};
|
||||
let inventory =
|
||||
runtime_recovery_ownership(&[sandbox.clone(), helper(rs)], GenerationId::new(1))
|
||||
.unwrap();
|
||||
assert_eq!(inventory.runtime_supervisor, Some(rs));
|
||||
assert_eq!(inventory.attached_runsc, Some(process(3)));
|
||||
assert_eq!(inventory.connectivity_helper, Some(process(4)));
|
||||
assert!(
|
||||
runtime_recovery_ownership(&[sandbox, helper(process(99))], GenerationId::new(1),)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inventory_requires_complete_private_mount_evidence() {
|
||||
let incomplete = participant(
|
||||
ParticipantRole::StateAnchor,
|
||||
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
|
||||
lower_artifact_id: None,
|
||||
upper_path: Some(HostPath::new(b"upper".to_vec())),
|
||||
work_path: None,
|
||||
merged_path: None,
|
||||
daemon: PersistedHostProcessOwnership::staged(process(5)),
|
||||
user_namespace: Some(NamespaceIdentity {
|
||||
device: DeviceId::new(6),
|
||||
inode: lazaret_core::identity::InodeId::new(7),
|
||||
}),
|
||||
mount_namespace: None,
|
||||
mount: None,
|
||||
}),
|
||||
);
|
||||
assert!(runtime_recovery_ownership(&[incomplete], GenerationId::new(1)).is_err());
|
||||
|
||||
let complete = participant(
|
||||
ParticipantRole::StateAnchor,
|
||||
ParticipantOwnership::StateAnchor(StateAnchorOwnership {
|
||||
lower_artifact_id: None,
|
||||
upper_path: Some(HostPath::new(b"upper".to_vec())),
|
||||
work_path: None,
|
||||
merged_path: None,
|
||||
daemon: PersistedHostProcessOwnership::staged(process(5)),
|
||||
user_namespace: Some(NamespaceIdentity {
|
||||
device: DeviceId::new(6),
|
||||
inode: lazaret_core::identity::InodeId::new(7),
|
||||
}),
|
||||
mount_namespace: Some(NamespaceIdentity {
|
||||
device: DeviceId::new(8),
|
||||
inode: lazaret_core::identity::InodeId::new(9),
|
||||
}),
|
||||
mount: Some(MountIdentity {
|
||||
mount_id: MountId::new(10),
|
||||
device: DeviceId::new(11),
|
||||
inode: lazaret_core::identity::InodeId::new(12),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert!(runtime_recovery_ownership(&[complete], GenerationId::new(1)).is_ok());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue