472 lines
17 KiB
Rust
472 lines
17 KiB
Rust
use std::io;
|
|
use std::os::fd::AsRawFd;
|
|
use std::os::unix::ffi::OsStrExt;
|
|
use std::path::Path;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use lazaret_core::configuration::{ConfinedPathType, ResolvedDomainConfiguration};
|
|
use lazaret_core::identity::{
|
|
ConfigurationFingerprint, DomainId, GenerationId, HostPath, MountIdentity, NamespaceIdentity,
|
|
PidfsProcessIdentityV1,
|
|
};
|
|
use lazaret_core::records::{
|
|
ArtifactLease, DurableDecision, Generation, Participant, ParticipantOwnership, ParticipantRole,
|
|
ParticipantState, PersistedHostProcessOwnership, SessionPhase,
|
|
};
|
|
use lazaret_linux::filesystem::{current_mount_identity, mount_identity};
|
|
use lazaret_linux::process::{
|
|
PidfsRecoveryMatch, RetainedProcess, TargetCommand, child_broker, match_recovery_process,
|
|
};
|
|
use lazaret_prepared_lower::PreparedLowerStore;
|
|
use lazaret_state_anchor::StateAnchorLayout;
|
|
use lazaret_storage_catalog::{Catalog, CatalogError};
|
|
use rustix::fs::{AtFlags, Mode, OFlags, fstat, openat, statat, unlinkat};
|
|
use rustix::io::{FdFlags, fcntl_setfd};
|
|
use rustix::process::Signal;
|
|
|
|
use crate::control::{
|
|
CONTROL_SOCKET_NAME, PUBLICATION_IDENTITY_NAME, STAGED_CONTROL_SOCKET_NAME,
|
|
STAGED_PUBLICATION_IDENTITY_NAME,
|
|
};
|
|
use crate::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
|
use crate::runtime::StateAnchorAbsenceObservation;
|
|
use crate::state_anchor::{
|
|
StateAnchorCoordinator, StateAnchorCoordinatorError, StateAnchorRecoveryOwner,
|
|
};
|
|
use crate::state_maintenance::recover_state_reset;
|
|
|
|
const PROCESS_EXIT_TIMEOUT: Duration = Duration::from_secs(5);
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
enum RuntimeProcessRole {
|
|
HostCoordinator,
|
|
RuntimeSupervisor,
|
|
AttachedRunsc,
|
|
ConnectivityHelper,
|
|
StateAnchorDaemon,
|
|
GitRoutingDaemon,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
struct RuntimeProcessOwner {
|
|
role: RuntimeProcessRole,
|
|
identity: PidfsProcessIdentityV1,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct RuntimeRecoveryOwnership {
|
|
domain_id: DomainId,
|
|
generation_id: GenerationId,
|
|
configuration_fingerprint: ConfigurationFingerprint,
|
|
participants_present: usize,
|
|
processes: Vec<RuntimeProcessOwner>,
|
|
state_anchor_user_namespace: Option<NamespaceIdentity>,
|
|
state_anchor_mount_namespace: Option<NamespaceIdentity>,
|
|
state_anchor_mount: Option<MountIdentity>,
|
|
state_anchor_path: Option<HostPath>,
|
|
git_routing_mount: Option<MountIdentity>,
|
|
git_routing_path: Option<HostPath>,
|
|
}
|
|
|
|
impl RuntimeRecoveryOwnership {
|
|
fn process(&self, role: RuntimeProcessRole) -> Option<PidfsProcessIdentityV1> {
|
|
self.processes
|
|
.iter()
|
|
.find(|owner| owner.role == role)
|
|
.map(|owner| owner.identity)
|
|
}
|
|
|
|
pub(crate) fn binding_matches(&self, domain_id: DomainId, generation: &Generation) -> bool {
|
|
self.domain_id == domain_id
|
|
&& self.generation_id == generation.generation_id
|
|
&& self.configuration_fingerprint == generation.configuration_fingerprint
|
|
}
|
|
|
|
pub(crate) fn state_anchor_matches(
|
|
&self,
|
|
daemon: PidfsProcessIdentityV1,
|
|
user_namespace: NamespaceIdentity,
|
|
mount_namespace: NamespaceIdentity,
|
|
mount: MountIdentity,
|
|
) -> bool {
|
|
self.process(RuntimeProcessRole::StateAnchorDaemon) == Some(daemon)
|
|
&& self.state_anchor_user_namespace == Some(user_namespace)
|
|
&& self.state_anchor_mount_namespace == Some(mount_namespace)
|
|
&& self.state_anchor_mount == Some(mount)
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
|
pub struct RuntimeOwnersAbsent {
|
|
ownership: RuntimeRecoveryOwnership,
|
|
}
|
|
|
|
enum SealedProcessMatch {
|
|
BootMismatch,
|
|
Esrch,
|
|
Matched(RetainedProcess),
|
|
}
|
|
|
|
struct SealedProcessOwner {
|
|
role: RuntimeProcessRole,
|
|
matched: SealedProcessMatch,
|
|
reconciled: bool,
|
|
}
|
|
|
|
struct SealedRuntimeOwners {
|
|
ownership: RuntimeRecoveryOwnership,
|
|
processes: Vec<SealedProcessOwner>,
|
|
}
|
|
|
|
impl SealedRuntimeOwners {
|
|
fn process(&self, role: RuntimeProcessRole) -> Option<&SealedProcessMatch> {
|
|
self.processes
|
|
.iter()
|
|
.find(|owner| owner.role == role)
|
|
.map(|owner| &owner.matched)
|
|
}
|
|
|
|
fn reconcile(&mut self, role: RuntimeProcessRole) -> io::Result<()> {
|
|
let Some(owner) = self.processes.iter_mut().find(|owner| owner.role == role) else {
|
|
return Ok(());
|
|
};
|
|
if owner.reconciled {
|
|
return Ok(());
|
|
}
|
|
match &owner.matched {
|
|
SealedProcessMatch::BootMismatch | SealedProcessMatch::Esrch => {}
|
|
SealedProcessMatch::Matched(process) if process.has_exited()? => {}
|
|
SealedProcessMatch::Matched(_) if role == RuntimeProcessRole::HostCoordinator => {
|
|
return Err(io::Error::other(
|
|
"recorded Host Coordinator remains present",
|
|
));
|
|
}
|
|
SealedProcessMatch::Matched(process) => {
|
|
process.send_signal(Signal::KILL)?;
|
|
let deadline = Instant::now() + PROCESS_EXIT_TIMEOUT;
|
|
while !process.has_exited()? {
|
|
if Instant::now() >= deadline {
|
|
return Err(io::Error::other(
|
|
"exact runtime owner did not exit after pidfd action",
|
|
));
|
|
}
|
|
std::thread::yield_now();
|
|
}
|
|
}
|
|
}
|
|
owner.reconciled = true;
|
|
Ok(())
|
|
}
|
|
|
|
fn prove_absent(&mut self) -> Result<RuntimeOwnersAbsent, io::Error> {
|
|
for role in [
|
|
RuntimeProcessRole::AttachedRunsc,
|
|
RuntimeProcessRole::ConnectivityHelper,
|
|
RuntimeProcessRole::GitRoutingDaemon,
|
|
RuntimeProcessRole::StateAnchorDaemon,
|
|
RuntimeProcessRole::RuntimeSupervisor,
|
|
RuntimeProcessRole::HostCoordinator,
|
|
] {
|
|
self.reconcile(role)?;
|
|
}
|
|
require_mount_absent(
|
|
self.ownership.state_anchor_path.as_ref(),
|
|
self.ownership.state_anchor_mount,
|
|
"State Anchor",
|
|
)?;
|
|
require_mount_absent(
|
|
self.ownership.git_routing_path.as_ref(),
|
|
self.ownership.git_routing_mount,
|
|
"Git-routing",
|
|
)?;
|
|
Ok(RuntimeOwnersAbsent {
|
|
ownership: self.ownership.clone(),
|
|
})
|
|
}
|
|
}
|
|
|
|
impl RuntimeOwnersAbsent {
|
|
pub(crate) fn ownership(&self) -> &RuntimeRecoveryOwnership {
|
|
&self.ownership
|
|
}
|
|
}
|
|
|
|
/// Extracts one generation's complete exact process and private-mount recovery authority.
|
|
///
|
|
/// # Errors
|
|
/// Refuses duplicate roles, missing role dependencies, unequal owner copies, and every incomplete
|
|
/// process, namespace, mount, or path relation required by a recorded participant phase.
|
|
#[allow(clippy::too_many_lines)]
|
|
pub fn runtime_recovery_ownership(
|
|
participants: &[Participant],
|
|
generation: &Generation,
|
|
) -> Result<RuntimeRecoveryOwnership, &'static str> {
|
|
let mut ownership = RuntimeRecoveryOwnership {
|
|
domain_id: generation.domain_id,
|
|
generation_id: generation.generation_id,
|
|
configuration_fingerprint: generation.configuration_fingerprint,
|
|
participants_present: 0,
|
|
processes: Vec::new(),
|
|
state_anchor_user_namespace: None,
|
|
state_anchor_mount_namespace: None,
|
|
state_anchor_mount: None,
|
|
state_anchor_path: None,
|
|
git_routing_mount: None,
|
|
git_routing_path: None,
|
|
};
|
|
let mut states = [None; 7];
|
|
for participant in participants
|
|
.iter()
|
|
.filter(|participant| participant.generation_id() == generation.generation_id)
|
|
{
|
|
let index = role_index(participant.role());
|
|
if states[index].replace(participant.state()).is_some() {
|
|
return Err("recovery participant role is duplicated");
|
|
}
|
|
ownership.participants_present += 1;
|
|
let Some(recorded) = participant.ownership() else {
|
|
if participant.state() != ParticipantState::Intent {
|
|
return Err("non-Intent recovery participant has no ownership");
|
|
}
|
|
continue;
|
|
};
|
|
match recorded {
|
|
ParticipantOwnership::Sandbox(sandbox) => {
|
|
merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::HostCoordinator,
|
|
&sandbox.host_coordinator,
|
|
"Sandbox HC identity mismatch",
|
|
)?;
|
|
merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::RuntimeSupervisor,
|
|
&sandbox.runtime_supervisor,
|
|
"Sandbox RS identity mismatch",
|
|
)?;
|
|
merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::AttachedRunsc,
|
|
&sandbox.attached_runsc,
|
|
"attached runsc identity mismatch",
|
|
)?;
|
|
}
|
|
ParticipantOwnership::Network(network) => merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::RuntimeSupervisor,
|
|
&network.runtime_supervisor,
|
|
"Network RS identity mismatch",
|
|
)?,
|
|
ParticipantOwnership::Helper(helper) => {
|
|
merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::ConnectivityHelper,
|
|
&helper.connectivity_helper,
|
|
"connectivity-helper identity mismatch",
|
|
)?;
|
|
merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::RuntimeSupervisor,
|
|
&helper.runtime_supervisor,
|
|
"Helper RS identity mismatch",
|
|
)?;
|
|
}
|
|
ParticipantOwnership::StateAnchor(anchor) => {
|
|
merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::StateAnchorDaemon,
|
|
&anchor.daemon,
|
|
"State Anchor daemon identity mismatch",
|
|
)?;
|
|
let evidence = (anchor.user_namespace, anchor.mount_namespace, anchor.mount);
|
|
if evidence != (None, None, None) {
|
|
let (Some(user), Some(mount_namespace), Some(mount)) = evidence else {
|
|
return Err("State Anchor recovery evidence is incomplete");
|
|
};
|
|
if ownership
|
|
.process(RuntimeProcessRole::StateAnchorDaemon)
|
|
.is_none()
|
|
{
|
|
return Err("State Anchor mount has no daemon authority");
|
|
}
|
|
ownership.state_anchor_user_namespace = Some(user);
|
|
ownership.state_anchor_mount_namespace = Some(mount_namespace);
|
|
ownership.state_anchor_mount = Some(mount);
|
|
ownership.state_anchor_path.clone_from(&anchor.merged_path);
|
|
if ownership.state_anchor_path.is_none() {
|
|
return Err("State Anchor mount path is absent");
|
|
}
|
|
}
|
|
}
|
|
ParticipantOwnership::GitRouting(routing) => {
|
|
merge_process(
|
|
&mut ownership,
|
|
RuntimeProcessRole::GitRoutingDaemon,
|
|
&routing.daemon,
|
|
"Git-routing daemon identity mismatch",
|
|
)?;
|
|
if let Some(mount) = routing.mount {
|
|
if ownership
|
|
.process(RuntimeProcessRole::GitRoutingDaemon)
|
|
.is_none()
|
|
{
|
|
return Err("Git-routing mount has no daemon authority");
|
|
}
|
|
ownership.git_routing_mount = Some(mount);
|
|
ownership.git_routing_path.clone_from(&routing.mountpoint);
|
|
if ownership.git_routing_path.is_none() {
|
|
return Err("Git-routing mount path is absent");
|
|
}
|
|
}
|
|
}
|
|
ParticipantOwnership::PreparedLower(_) | ParticipantOwnership::Publication(_) => {}
|
|
}
|
|
}
|
|
require_role_dependencies(states)?;
|
|
require_phase_topology(participants, generation.generation_id, &ownership, states)?;
|
|
Ok(ownership)
|
|
}
|
|
|
|
fn merge_process(
|
|
ownership: &mut RuntimeRecoveryOwnership,
|
|
role: RuntimeProcessRole,
|
|
observed: &PersistedHostProcessOwnership,
|
|
mismatch: &'static str,
|
|
) -> Result<(), &'static str> {
|
|
let Ok(identity) = observed.authoritative_pidfs() else {
|
|
return Ok(());
|
|
};
|
|
match ownership.process(role) {
|
|
None => {
|
|
if ownership
|
|
.processes
|
|
.iter()
|
|
.any(|owner| owner.identity == *identity)
|
|
{
|
|
return Err("one process identity is assigned to distinct runtime roles");
|
|
}
|
|
ownership.processes.push(RuntimeProcessOwner {
|
|
role,
|
|
identity: *identity,
|
|
});
|
|
}
|
|
Some(current) if current == *identity => {}
|
|
Some(_) => return Err(mismatch),
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn require_role_dependencies(states: [Option<ParticipantState>; 7]) -> Result<(), &'static str> {
|
|
for (role, required) in [
|
|
(
|
|
ParticipantRole::StateAnchor,
|
|
&[ParticipantRole::PreparedLower, ParticipantRole::Sandbox][..],
|
|
),
|
|
(
|
|
ParticipantRole::Helper,
|
|
&[ParticipantRole::PreparedLower, ParticipantRole::StateAnchor],
|
|
),
|
|
(
|
|
ParticipantRole::Network,
|
|
&[
|
|
ParticipantRole::PreparedLower,
|
|
ParticipantRole::StateAnchor,
|
|
ParticipantRole::Helper,
|
|
],
|
|
),
|
|
(
|
|
ParticipantRole::Publication,
|
|
&[
|
|
ParticipantRole::PreparedLower,
|
|
ParticipantRole::StateAnchor,
|
|
ParticipantRole::Helper,
|
|
ParticipantRole::Network,
|
|
ParticipantRole::Sandbox,
|
|
],
|
|
),
|
|
] {
|
|
if states[role_index(role)].is_some()
|
|
&& required
|
|
.iter()
|
|
.any(|required| states[role_index(*required)].is_none())
|
|
{
|
|
return Err("recovery participant dependency is missing");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn require_phase_topology(
|
|
participants: &[Participant],
|
|
generation_id: GenerationId,
|
|
ownership: &RuntimeRecoveryOwnership,
|
|
states: [Option<ParticipantState>; 7],
|
|
) -> Result<(), &'static str> {
|
|
let sandbox = participants.iter().find(|participant| {
|
|
participant.generation_id() == generation_id
|
|
&& participant.role() == ParticipantRole::Sandbox
|
|
});
|
|
if let Some(participant) = sandbox {
|
|
let Some(ParticipantOwnership::Sandbox(sandbox)) = participant.ownership() else {
|
|
return Err("Sandbox durable intent lacks its immutable launch authority");
|
|
};
|
|
if participant.state() != ParticipantState::Clean
|
|
&& (sandbox.container_id.is_none()
|
|
|| sandbox.runtime_root.is_none()
|
|
|| sandbox.runtime_root_identity.is_none()
|
|
|| sandbox.bundle_digest.is_none()
|
|
|| sandbox.host_coordinator.authoritative_pidfs().is_err())
|
|
{
|
|
return Err("Sandbox durable launch authority is incomplete");
|
|
}
|
|
if participant.state() == ParticipantState::Committed
|
|
&& (sandbox.runtime_supervisor.authoritative_pidfs().is_err()
|
|
|| sandbox.attached_runsc.authoritative_pidfs().is_err())
|
|
{
|
|
return Err("committed Sandbox process authority is incomplete");
|
|
}
|
|
}
|
|
let later_runtime_role = [
|
|
ParticipantRole::StateAnchor,
|
|
ParticipantRole::Helper,
|
|
ParticipantRole::Network,
|
|
ParticipantRole::GitRouting,
|
|
ParticipantRole::Publication,
|
|
]
|
|
.into_iter()
|
|
.any(|role| states[role_index(role)].is_some());
|
|
let sandbox_clean =
|
|
states[role_index(ParticipantRole::Sandbox)] == Some(ParticipantState::Clean);
|
|
if later_runtime_role
|
|
&& !sandbox_clean
|
|
&& ownership
|
|
.process(RuntimeProcessRole::RuntimeSupervisor)
|
|
.is_none()
|
|
{
|
|
return Err("later runtime phase lacks its Runtime Supervisor authority");
|
|
}
|
|
if states[role_index(ParticipantRole::StateAnchor)].is_some()
|
|
&& !sandbox_clean
|
|
&& ownership
|
|
.process(RuntimeProcessRole::HostCoordinator)
|
|
.is_none()
|
|
{
|
|
return Err("State Anchor phase lacks generation Host Coordinator authority");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
const fn role_index(role: ParticipantRole) -> usize {
|
|
match role {
|
|
ParticipantRole::PreparedLower => 0,
|
|
ParticipantRole::StateAnchor => 1,
|
|
ParticipantRole::GitRouting => 2,
|
|
ParticipantRole::Network => 3,
|
|
ParticipantRole::Helper => 4,
|
|
ParticipantRole::Sandbox => 5,
|
|
ParticipantRole::Publication => 6,
|
|
}
|
|
}
|
|
|
|
include!("recovery/reconciliation.rs");
|
|
include!("recovery/tests.rs");
|