Implement exact bootstrap recovery and artifact GC
This commit is contained in:
parent
0f4845d4f8
commit
d25942596f
1 changed files with 1631 additions and 48 deletions
|
|
@ -223,6 +223,18 @@ pub struct RetainedProcess {
|
|||
identity: PidfsProcessIdentityV1,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum PidfsRecoveryMatch {
|
||||
BootMismatch,
|
||||
Matched(RetainedProcess),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum RecoveryActionResult<T> {
|
||||
BootReconciled(T),
|
||||
ProcessAction(T),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct CapturedProcess {
|
||||
process: RetainedProcess,
|
||||
|
|
@ -412,6 +424,27 @@ impl ChildBroker {
|
|||
/// Returns an error when causal launch, exact waiting, or the command exit status fails.
|
||||
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
|
||||
pub fn run_transient(&self, target: &TargetCommand) -> io::Result<ProcessExit> {
|
||||
self.run_transient_with_stdout(target, None)
|
||||
}
|
||||
|
||||
/// Runs a non-durable direct child with one private inherited stdout descriptor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when causal launch, exact waiting, or the command exit status fails.
|
||||
pub fn run_transient_with_private_stdout(
|
||||
&self,
|
||||
target: &TargetCommand,
|
||||
stdout: OwnedFd,
|
||||
) -> io::Result<ProcessExit> {
|
||||
self.run_transient_with_stdout(target, Some(stdout))
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
|
||||
fn run_transient_with_stdout(
|
||||
&self,
|
||||
target: &TargetCommand,
|
||||
stdout: Option<OwnedFd>,
|
||||
) -> io::Result<ProcessExit> {
|
||||
let _guard = self
|
||||
.exclusion
|
||||
.lock()
|
||||
|
|
@ -426,7 +459,7 @@ impl ChildBroker {
|
|||
target.append_to(&mut command);
|
||||
command
|
||||
.stdin(wait.into_stdio())
|
||||
.stdout(Stdio::null())
|
||||
.stdout(stdout.map_or_else(Stdio::null, Stdio::from))
|
||||
.stderr(Stdio::null());
|
||||
let mut child = command.spawn()?;
|
||||
let process = match acquire_spawned_child(child.id()) {
|
||||
|
|
@ -911,41 +944,87 @@ fn fail_after_pidfd_closed<T>(process: RetainedProcess, error: io::Error) -> io:
|
|||
/// Refuses before PID lookup on boot mismatch and before action on every later mismatch.
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
pub fn recover_process(identity: PidfsProcessIdentityV1) -> io::Result<RetainedProcess> {
|
||||
recover_process_with(identity, read_boot_id, |host_pid| {
|
||||
let raw_pid = i32::try_from(host_pid)
|
||||
.map_err(|_| io::Error::other("stored PID locator is out of range"))?;
|
||||
let pid = Pid::from_raw(raw_pid)
|
||||
.ok_or_else(|| io::Error::other("stored PID locator is invalid"))?;
|
||||
require_host_pid_namespace(host_pid)?;
|
||||
rustix::process::pidfd_open(pid, PidfdFlags::empty()).map_err(Into::into)
|
||||
})
|
||||
match match_recovery_process(identity)? {
|
||||
PidfsRecoveryMatch::BootMismatch => Err(io::Error::other(
|
||||
"stored process authority belongs to another boot",
|
||||
)),
|
||||
PidfsRecoveryMatch::Matched(process) => Ok(process),
|
||||
}
|
||||
}
|
||||
|
||||
fn recover_process_with<Boot, Open>(
|
||||
/// Reopens one stored process identity or reports boot mismatch before PID lookup.
|
||||
///
|
||||
/// # Errors
|
||||
/// Refuses host-PID ambiguity, PID lookup, pidfs validation, identity, and negative-evidence
|
||||
/// failures without returning an actionable descriptor.
|
||||
#[allow(clippy::disallowed_methods)]
|
||||
pub fn match_recovery_process(identity: PidfsProcessIdentityV1) -> io::Result<PidfsRecoveryMatch> {
|
||||
match_recovery_process_with(
|
||||
identity,
|
||||
read_boot_id,
|
||||
|host_pid| {
|
||||
let raw_pid = i32::try_from(host_pid)
|
||||
.map_err(|_| io::Error::other("stored PID locator is out of range"))?;
|
||||
let pid = Pid::from_raw(raw_pid)
|
||||
.ok_or_else(|| io::Error::other("stored PID locator is invalid"))?;
|
||||
require_host_pid_namespace(host_pid)?;
|
||||
rustix::process::pidfd_open(pid, PidfdFlags::empty()).map_err(Into::into)
|
||||
},
|
||||
|pidfd, boot_id| authority_from_pidfd(pidfd, boot_id),
|
||||
read_start_time,
|
||||
)
|
||||
}
|
||||
|
||||
/// Reconciles reboot-safe physical evidence or acts through the exactly reopened pidfd.
|
||||
///
|
||||
/// # Errors
|
||||
/// Propagates reconciliation, pidfs matching, or same-descriptor action failure. The process
|
||||
/// action is never called on boot mismatch or an unequal process identity.
|
||||
pub fn recover_process_action<T>(
|
||||
identity: PidfsProcessIdentityV1,
|
||||
reconcile_boot: impl FnOnce() -> io::Result<T>,
|
||||
action: impl FnOnce(&RetainedProcess) -> io::Result<T>,
|
||||
) -> io::Result<RecoveryActionResult<T>> {
|
||||
match match_recovery_process(identity)? {
|
||||
PidfsRecoveryMatch::BootMismatch => {
|
||||
reconcile_boot().map(RecoveryActionResult::BootReconciled)
|
||||
}
|
||||
PidfsRecoveryMatch::Matched(process) => {
|
||||
action(&process).map(RecoveryActionResult::ProcessAction)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn match_recovery_process_with<Boot, Open, Inspect, Start>(
|
||||
identity: PidfsProcessIdentityV1,
|
||||
boot: Boot,
|
||||
open: Open,
|
||||
) -> io::Result<RetainedProcess>
|
||||
inspect: Inspect,
|
||||
start: Start,
|
||||
) -> io::Result<PidfsRecoveryMatch>
|
||||
where
|
||||
Boot: FnOnce() -> io::Result<BootId>,
|
||||
Open: FnOnce(u32) -> io::Result<OwnedFd>,
|
||||
Inspect: FnOnce(BorrowedFd<'_>, BootId) -> io::Result<PidfsProcessAuthorityV1>,
|
||||
Start: FnOnce(u32) -> io::Result<u64>,
|
||||
{
|
||||
let current_boot = boot()?;
|
||||
if current_boot != identity.authority().boot_id {
|
||||
return Err(io::Error::other(
|
||||
"stored process authority belongs to another boot",
|
||||
));
|
||||
return Ok(PidfsRecoveryMatch::BootMismatch);
|
||||
}
|
||||
let host_pid = identity.host_pid().get();
|
||||
let pidfd = open(host_pid)?;
|
||||
let observed = authority_from_pidfd(pidfd.as_fd(), current_boot)?;
|
||||
let observed = inspect(pidfd.as_fd(), current_boot)?;
|
||||
if observed != identity.authority() {
|
||||
return Err(io::Error::other("pidfs process authority mismatch"));
|
||||
}
|
||||
if read_start_time(host_pid)? != identity.start_time_ticks().get() {
|
||||
if start(host_pid)? != identity.start_time_ticks().get() {
|
||||
return Err(io::Error::other("process start time mismatch"));
|
||||
}
|
||||
Ok(RetainedProcess { pidfd, identity })
|
||||
Ok(PidfsRecoveryMatch::Matched(RetainedProcess {
|
||||
pidfd,
|
||||
identity,
|
||||
}))
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
|
|
@ -1491,18 +1570,138 @@ mod tests {
|
|||
ProcessStartTimeTicks::new(1),
|
||||
);
|
||||
let opened = Cell::new(false);
|
||||
let result = recover_process_with(
|
||||
let result = match_recovery_process_with(
|
||||
identity,
|
||||
|| Ok(BootId::new([2; 16])),
|
||||
|_| {
|
||||
opened.set(true);
|
||||
unreachable!()
|
||||
},
|
||||
|_, _| unreachable!(),
|
||||
|_| unreachable!(),
|
||||
);
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(result, Ok(PidfsRecoveryMatch::BootMismatch)));
|
||||
assert!(!opened.get());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_refuses_each_post_open_mismatch_without_returning_authority() {
|
||||
let authority = PidfsProcessAuthorityV1 {
|
||||
boot_id: BootId::new([1; 16]),
|
||||
pidfs_st_dev: PidfsDeviceId::new(7),
|
||||
pidfs_st_ino: PidfsInodeId::new(9),
|
||||
};
|
||||
let identity = PidfsProcessIdentityV1::new(
|
||||
authority,
|
||||
HostPidLocator::new(41),
|
||||
ProcessStartTimeTicks::new(43),
|
||||
);
|
||||
for (observed, start) in [
|
||||
(
|
||||
PidfsProcessAuthorityV1 {
|
||||
pidfs_st_dev: PidfsDeviceId::new(8),
|
||||
..authority
|
||||
},
|
||||
43,
|
||||
),
|
||||
(
|
||||
PidfsProcessAuthorityV1 {
|
||||
pidfs_st_ino: PidfsInodeId::new(10),
|
||||
..authority
|
||||
},
|
||||
43,
|
||||
),
|
||||
(authority, 44),
|
||||
] {
|
||||
let events = Rc::new(std::cell::RefCell::new(Vec::new()));
|
||||
let open_events = Rc::clone(&events);
|
||||
let inspect_events = Rc::clone(&events);
|
||||
let start_events = Rc::clone(&events);
|
||||
let result = match_recovery_process_with(
|
||||
identity,
|
||||
|| Ok(authority.boot_id),
|
||||
move |pid| {
|
||||
open_events.borrow_mut().push(("open", u64::from(pid)));
|
||||
Ok(File::open("/dev/null").unwrap().into())
|
||||
},
|
||||
move |fd, _| {
|
||||
inspect_events
|
||||
.borrow_mut()
|
||||
.push(("inspect", u64::try_from(fd.as_raw_fd()).unwrap()));
|
||||
Ok(observed)
|
||||
},
|
||||
move |pid| {
|
||||
start_events.borrow_mut().push(("start", u64::from(pid)));
|
||||
Ok(start)
|
||||
},
|
||||
);
|
||||
assert!(result.is_err());
|
||||
assert_eq!(events.borrow()[0], ("open", 41));
|
||||
assert_eq!(events.borrow()[1].0, "inspect");
|
||||
if observed == authority {
|
||||
assert_eq!(events.borrow()[2], ("start", 41));
|
||||
} else {
|
||||
assert_eq!(events.borrow().len(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
for failure in ["open", "inspect", "start"] {
|
||||
let result = match_recovery_process_with(
|
||||
identity,
|
||||
|| Ok(authority.boot_id),
|
||||
|_| {
|
||||
if failure == "open" {
|
||||
Err(io::Error::from_raw_os_error(3))
|
||||
} else {
|
||||
Ok(File::open("/dev/null").unwrap().into())
|
||||
}
|
||||
},
|
||||
|_, _| {
|
||||
if failure == "inspect" {
|
||||
Err(io::Error::other("injected descriptor inspection error"))
|
||||
} else {
|
||||
Ok(authority)
|
||||
}
|
||||
},
|
||||
|_| {
|
||||
if failure == "start" {
|
||||
Err(io::Error::other("injected start observation error"))
|
||||
} else {
|
||||
Ok(43)
|
||||
}
|
||||
},
|
||||
);
|
||||
assert!(result.is_err(), "accepted injected {failure} failure");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_action_reconciles_boot_without_process_action() {
|
||||
let mut old_boot = *read_boot_id().unwrap().as_bytes();
|
||||
old_boot[0] ^= u8::MAX;
|
||||
let identity = PidfsProcessIdentityV1::new(
|
||||
PidfsProcessAuthorityV1 {
|
||||
boot_id: BootId::new(old_boot),
|
||||
pidfs_st_dev: PidfsDeviceId::new(1),
|
||||
pidfs_st_ino: PidfsInodeId::new(1),
|
||||
},
|
||||
HostPidLocator::new(u32::MAX),
|
||||
ProcessStartTimeTicks::new(u64::MAX),
|
||||
);
|
||||
let action_called = Cell::new(false);
|
||||
let result = recover_process_action(
|
||||
identity,
|
||||
|| Ok("reconciled"),
|
||||
|_| {
|
||||
action_called.set(true);
|
||||
Ok("acted")
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(result, RecoveryActionResult::BootReconciled("reconciled"));
|
||||
assert!(!action_called.get());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pidfs_magic_precedes_same_descriptor_stat_and_errors_refuse() {
|
||||
let file = File::open("/dev/null").unwrap();
|
||||
|
|
|
|||
Loading…
Reference in a new issue