ms2k/lazaret
SHA256
Watch
0
Code Releases 3 Activity

Close hostile recovery and concurrency gaps

Browse source
This commit is contained in:
ms2k 2026-07-18 05:47:15 +09:00
commit 3f9b1b3543
16 changed files with 306 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

7
crates/lazaret-client/src/lib.rs
View file

@ -212,8 +212,8 @@ pub fn run<B: ClientBackend>(
let endpoint = match backend.observe().map_err(ClientError::Backend)? {
DomainStatus::Ready(endpoint) => endpoint,
DomainStatus::Starting => backend.wait_until_ready()?,
DomainStatus::Stale => return Err(ClientError::StaleCoordinator),
DomainStatus::Absent => match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
DomainStatus::Stale | DomainStatus::Absent => {
match backend.acquire_bootstrap().map_err(ClientError::Backend)? {
BootstrapAcquisition::Leader => match backend.bootstrap() {
Ok(endpoint) => endpoint,
Err(ClientError::BootstrapOwnerDisconnected) => {
@ -222,7 +222,8 @@ pub fn run<B: ClientBackend>(
Err(error) => return Err(error),
},
BootstrapAcquisition::Waiter => backend.wait_until_ready()?,
},
}
}
};
backend.execute(endpoint, &command.argv, stdout, stderr)
}

22
crates/lazaret-client/src/production.rs
View file

@ -84,6 +84,9 @@ impl ProductionClientBackend {
Ok((authority, socket_path, publication))
if publication_matches(configuration, &publication) =>
{
if !control_socket_is_live(&socket_path) {
return Ok(Discovery::Stale);
}
Ok(Discovery::Ready(Box::new(ControlEndpoint::confined(
socket_path,
publication,
@ -177,8 +180,7 @@ impl ProductionClientBackend {
loop {
match self.discover().map_err(ClientError::Backend)? {
Discovery::Ready(endpoint) => return Ok(*endpoint),
Discovery::Stale => return Err(ClientError::StaleCoordinator),
Discovery::Absent => {}
Discovery::Stale | Discovery::Absent => {}
}
if self
.coordinator
@ -231,11 +233,7 @@ impl ClientBackend for ProductionClientBackend {
Discovery::Stale => return Ok(DomainStatus::Stale),
Discovery::Absent => {}
}
if self.domain_lock_is_available()? {
Ok(DomainStatus::Absent)
} else {
Ok(DomainStatus::Starting)
}
}
fn acquire_bootstrap(&mut self) -> Result<BootstrapAcquisition, Self::Error> {
@ -273,8 +271,7 @@ impl ClientBackend for ProductionClientBackend {
loop {
match self.discover().map_err(ClientError::Backend)? {
Discovery::Ready(endpoint) => return Ok(*endpoint),
Discovery::Stale => return Err(ClientError::StaleCoordinator),
Discovery::Absent => {}
Discovery::Stale | Discovery::Absent => {}
}
match std::os::unix::net::UnixStream::connect(&socket) {
Ok(mut stream) => {
@ -407,6 +404,10 @@ pub(crate) fn open_private_domain_lock(root: &impl AsFd) -> Result<File, Product
Ok(file)
}
fn control_socket_is_live(socket_path: &std::path::Path) -> bool {
std::os::unix::net::UnixStream::connect(socket_path).is_ok()
}
fn publication_matches(
configuration: &ResolvedDomainConfiguration,
publication: &lazaret_core::records::ProtocolPublicationIdentity,
@ -587,7 +588,10 @@ mod tests {
let (server, _) = listener.accept().unwrap();
assert_eq!(observed, publication);
drop((authority, client, server));
assert!(control_socket_is_live(&runtime_path.join("c.sock")));
let (probe, _) = listener.accept().unwrap();
drop((authority, client, server, probe, listener));
assert!(!control_socket_is_live(&runtime_path.join("c.sock")));
}
#[test]

21
crates/lazaret-client/tests/cli_contract.rs
View file

@ -177,12 +177,27 @@ fn nonzero_guest_exit_is_the_cli_status() {
}
#[test]
fn stale_and_lost_coordinators_are_actionable_failures() {
let mut stale = FakeBackend {
fn stale_domain_enters_lock_arbitration_while_live_stale_and_lost_faces_refuse() {
let mut unowned = FakeBackend {
observations: [DomainStatus::Stale].into(),
acquisition: Some(BootstrapAcquisition::Leader),
bootstrap: Some(Ok(Endpoint::Published)),
..FakeBackend::default()
};
assert!(invoke(&mut stale).unwrap_err().contains("stale"));
assert_eq!(invoke(&mut unowned).unwrap().0, 0);
assert_eq!(
unowned.actions,
["validate", "observe", "acquire", "bootstrap", "execute"]
);
let mut live = FakeBackend {
observations: [DomainStatus::Stale].into(),
acquisition: Some(BootstrapAcquisition::Waiter),
waited: Some(Err(ClientError::StaleCoordinator)),
..FakeBackend::default()
};
assert!(invoke(&mut live).unwrap_err().contains("stale"));
assert_eq!(live.actions, ["validate", "observe", "acquire", "wait"]);
let mut lost = FakeBackend {
observations: [DomainStatus::Ready(Endpoint::Existing)].into(),

5
crates/lazaret-core/src/configuration.rs
View file

@ -1597,7 +1597,10 @@ fn ensure_owned_directory(path: &Path, uid: u32) -> Result<PathBuf, Configuratio
let mut current = open_absolute_directory(&existing_canonical)?;
let mut canonical = existing_canonical;
for component in missing.into_iter().rev() {
rustix::fs::mkdirat(&current, &component, Mode::RWXU).map_err(io_error)?;
match rustix::fs::mkdirat(&current, &component, Mode::RWXU) {
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
Err(error) => return Err(io_error(error)),
}
let next = openat2(
&current,
&component,

26
crates/lazaret-core/tests/configuration.rs
View file

@ -3,6 +3,7 @@ use std::fs;
use std::os::unix::fs::{MetadataExt, PermissionsExt, symlink};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Barrier};
use lazaret_core::codec::{CodecError, decode, encode};
use lazaret_core::configuration::{
@ -645,6 +646,31 @@ fn volume_symlinks_and_lazaret_owned_mode_drift_are_rejected() {
);
}
#[test]
fn concurrent_loaders_share_exact_owned_directory_creation() {
let tree = TestTree::new();
let config = write_configuration(&tree, &valid_configuration(&tree));
let barrier = Arc::new(Barrier::new(8));
std::thread::scope(|scope| {
let handles = (0..8)
.map(|_| {
let barrier = Arc::clone(&barrier);
let config = config.clone();
let tree = &tree;
scope.spawn(move || {
let mut environment = BTreeMap::new();
environment.insert("HOST_TOKEN".into(), "value".into());
barrier.wait();
load_trusted_configuration(request(tree, config, environment))
})
})
.collect::<Vec<_>>();
for handle in handles {
handle.join().unwrap().unwrap();
}
});
}
#[test]
fn configuration_source_permissions_links_and_size_are_bounded() {
for mutation in 0..3 {

38
crates/lazaret-host-coordinator/src/client_bootstrap.rs
View file

@ -183,7 +183,7 @@ fn serve_published(
loop {
while let Ok((connection_id, result)) = completions.try_recv() {
active.remove(&connection_id);
connection_completion(result, false)?;
connection_completion(result)?;
}
require_escalation_running(&escalation_completions, escalation_enabled)?;
let face = {
@ -196,7 +196,7 @@ fn serve_published(
}
drop(lifecycle);
while !active.is_empty() {
wait_for_connection_completion(&completions, &mut active, true)?;
wait_for_connection_completion(&completions, &mut active)?;
}
if let Some(escalation) = &escalation_handle {
escalation.stop();
@ -222,7 +222,7 @@ fn serve_published(
face
};
if active.len() == MAX_CONCURRENT_CONNECTIONS {
wait_for_connection_completion(&completions, &mut active, false)?;
wait_for_connection_completion(&completions, &mut active)?;
continue;
}
match face.accept_connection() {
@ -299,24 +299,19 @@ fn require_escalation_running(
fn wait_for_connection_completion(
completions: &std::sync::mpsc::Receiver<(u64, Result<(), ControlError>)>,
active: &mut BTreeMap<u64, std::os::unix::net::UnixStream>,
cancellation_requested: bool,
) -> Result<(), ProductionColdStartError> {
let (connection_id, result) = completions
.recv()
.map_err(ProductionColdStartError::service)?;
active.remove(&connection_id);
connection_completion(result, cancellation_requested)
connection_completion(result)
}
fn connection_completion(
result: Result<(), ControlError>,
cancellation_requested: bool,
) -> Result<(), ProductionColdStartError> {
fn connection_completion(result: Result<(), ControlError>) -> Result<(), ProductionColdStartError> {
match result {
Ok(()) => Ok(()),
Err(ControlError::Io(error))
if cancellation_requested
&& matches!(
if matches!(
error.kind(),
std::io::ErrorKind::UnexpectedEof
| std::io::ErrorKind::ConnectionReset
@ -583,30 +578,23 @@ mod tests {
}
#[test]
fn withdrawal_suppresses_only_cancelled_socket_io() {
fn client_disconnects_and_withdrawal_are_connection_local() {
assert!(
connection_completion(
Err(ControlError::Io(std::io::Error::new(
connection_completion(Err(ControlError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"withdrawn socket",
))),
true,
)
))))
.is_ok()
);
assert!(
connection_completion(
Err(ControlError::Io(std::io::Error::new(
connection_completion(Err(ControlError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"ordinary socket failure",
))),
false,
)
.is_err()
))))
.is_ok()
);
assert!(
connection_completion(Err(ControlError::Lifecycle("teardown failed".into())), true,)
.is_err()
connection_completion(Err(ControlError::Lifecycle("teardown failed".into()))).is_err()
);
}
}

10
crates/lazaret-host-coordinator/src/control.rs
View file

@ -2,10 +2,10 @@ use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::fmt;
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Seek, SeekFrom, Write};
use std::io::{self, Read, Write};
use std::os::fd::{AsFd, AsRawFd};
use std::os::unix::ffi::OsStringExt;
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard};
@ -1743,10 +1743,8 @@ impl PublishedControlFace {
"publication objects changed identity, ownership, or mode",
));
}
let mut exact_identity = self.identity_file.try_clone()?;
exact_identity.seek(SeekFrom::Start(0))?;
let mut bytes = Vec::new();
exact_identity.read_to_end(&mut bytes)?;
let mut bytes = vec![0; self.publication_bytes.len()];
self.identity_file.read_exact_at(&mut bytes, 0)?;
if bytes != self.publication_bytes {
return Err(ControlError::InvalidPublication(
"publication identity bytes changed",

49
crates/lazaret-host-coordinator/src/recovery.rs
View file

@ -630,7 +630,9 @@ where
{
recover_state_reset(configuration, store).map_err(|error| error.to_string())?;
failpoint(ProductionRecoveryBoundary::Snapshot, GenerationId::new(0))?;
let snapshot = store.snapshot().map_err(|error| error.to_string())?;
let snapshot = store
.snapshot()
.map_err(|error| format!("Domain Store snapshot failed: {error}"))?;
if snapshot.domain.domain_id != configuration.domain_id() {
return Err("recovery Domain differs from resolved configuration".into());
}
@ -651,7 +653,13 @@ where
&mut failpoint,
&mut open,
&mut delete,
)?;
)
.map_err(|error| {
format!(
"generation {} recovery failed: {error}",
generation.generation_id.get()
)
})?;
}
failpoint(
ProductionRecoveryBoundary::PrepareOperations,
@ -725,26 +733,34 @@ where
if !session_updates.is_empty() {
store
.apply(&session_updates)
.map_err(|error| error.to_string())?;
.map_err(|error| format!("Session failure update failed: {error}"))?;
}
let participants = store
.snapshot()
.map_err(|error| error.to_string())?
.map_err(|error| format!("Participant snapshot failed: {error}"))?
.participants;
let ownership = runtime_recovery_ownership(&participants, generation)?;
let mut owners = seal_runtime_owners(&ownership, open).map_err(|error| error.to_string())?;
let ownership = runtime_recovery_ownership(&participants, generation)
.map_err(|error| format!("runtime ownership inventory failed: {error}"))?;
let mut owners = seal_runtime_owners(&ownership, open)
.map_err(|error| format!("runtime owner acquisition failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Publication, generation_id)?;
recover_publication(configuration, store, generation_id)?;
recover_publication(configuration, store, generation_id)
.map_err(|error| format!("Publication recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::Sandbox, generation_id)?;
reconcile_runtime_before_state_anchor(&mut owners).map_err(|error| error.to_string())?;
require_sandbox_absent(configuration, store, generation_id, delete)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)?;
reconcile_runtime_before_state_anchor(&mut owners)
.map_err(|error| format!("runtime-owner reconciliation failed: {error}"))?;
require_sandbox_absent(configuration, store, generation_id, delete)
.map_err(|error| format!("Sandbox absence failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Sandbox)
.map_err(|error| format!("Sandbox cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::HelperAndNetwork, generation_id)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Network)?;
clean_role_after_absence(store, generation_id, ParticipantRole::Helper)
.map_err(|error| format!("Helper cleanup failed: {error}"))?;
clean_role_after_absence(store, generation_id, ParticipantRole::Network)
.map_err(|error| format!("Network cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::GitRouting, generation_id)?;
require_mount_absent(
@ -753,13 +769,16 @@ where
"Git-routing",
)
.map_err(|error| error.to_string())?;
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)?;
clean_role_after_absence(store, generation_id, ParticipantRole::GitRouting)
.map_err(|error| format!("Git-routing cleanup failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::StateAnchor, generation_id)?;
recover_state_anchor(configuration, store, catalog, generation, &mut owners)?;
recover_state_anchor(configuration, store, catalog, generation, &mut owners)
.map_err(|error| format!("State Anchor recovery failed: {error}"))?;
failpoint(ProductionRecoveryBoundary::PreparedLower, generation_id)?;
recover_partial_prepared_lower(configuration, store, catalog, generation_id)?;
recover_partial_prepared_lower(configuration, store, catalog, generation_id)
.map_err(|error| format!("prepared-lower recovery failed: {error}"))?;
let snapshot = store.snapshot().map_err(|error| error.to_string())?;
if snapshot.participants.iter().any(|participant| {

31
crates/lazaret-host-coordinator/src/runtime.rs
View file

@ -946,6 +946,17 @@ pub struct CommittedSandboxRuntime {
}
impl CommittedSandboxRuntime {
fn receive_runtime_event(&mut self) -> Result<SupervisorEvent, SandboxCoordinatorError> {
loop {
let event = self.control.receive()?;
if event == SupervisorEvent::HelperExited {
self.network_evidence = None;
} else {
return Ok(event);
}
}
}
#[must_use]
pub const fn network_evidence(&self) -> Option<&NetworkReadinessEvidence> {
self.network_evidence.as_ref()
@ -984,11 +995,11 @@ impl CommittedSandboxRuntime {
/// # Errors
/// Returns an error if the RS cannot apply the exact retained-runsc pidfd action.
pub fn signal_attached_runsc(
&self,
&mut self,
signal: AttachedRunscSignal,
) -> Result<(), SandboxCoordinatorError> {
self.control.signal_attached_runsc(signal)?;
if self.control.receive()? == SupervisorEvent::SignalApplied(signal) {
if self.receive_runtime_event()? == SupervisorEvent::SignalApplied(signal) {
Ok(())
} else {
Err(SandboxCoordinatorError::InvalidState(
@ -1007,7 +1018,7 @@ impl CommittedSandboxRuntime {
) -> Result<(), SandboxCoordinatorError> {
coordinator.begin_release(self.generation_id)?;
self.control.stop_sandbox()?;
if self.control.receive()? != SupervisorEvent::SandboxStopped {
if self.receive_runtime_event()? != SupervisorEvent::SandboxStopped {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not stop the sandbox",
));
@ -1027,7 +1038,7 @@ impl CommittedSandboxRuntime {
.begin_release(self.generation_id)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error)))?;
self.control.stop_git_routing()?;
match self.control.receive()? {
match self.receive_runtime_event()? {
SupervisorEvent::GitRoutingStopped { mount } if mount == expected => coordinator
.complete_clean(self.generation_id, mount)
.map_err(|error| SandboxCoordinatorError::Io(io::Error::other(error))),
@ -1046,7 +1057,7 @@ impl CommittedSandboxRuntime {
/// Refuses an unexpected RS response or channel failure.
pub fn stop_sidecar(&mut self) -> Result<(), SandboxCoordinatorError> {
self.control.stop_sidecar()?;
if self.control.receive()? == SupervisorEvent::SidecarStopped {
if self.receive_runtime_event()? == SupervisorEvent::SidecarStopped {
self.network_evidence = None;
Ok(())
} else {
@ -1065,7 +1076,7 @@ impl CommittedSandboxRuntime {
expected_mount: MountIdentity,
) -> Result<(), SandboxCoordinatorError> {
self.control.unmount_state_anchor()?;
match self.control.receive()? {
match self.receive_runtime_event()? {
SupervisorEvent::StateAnchorUnmounted { mount } if mount == expected_mount => Ok(()),
_ => Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not confirm the exact State Anchor unmount",
@ -1085,7 +1096,7 @@ impl CommittedSandboxRuntime {
) -> Result<StateAnchorAbsenceObservation, SandboxCoordinatorError> {
self.control
.observe_state_anchor_absence(user_namespace, mount_namespace, mount)?;
match self.control.receive()? {
match self.receive_runtime_event()? {
SupervisorEvent::StateAnchorAbsenceObserved {
user_namespace: observed_user_namespace,
mount_namespace: observed_mount_namespace,
@ -1118,7 +1129,7 @@ impl CommittedSandboxRuntime {
/// Refuses an unexpected response or an RS that does not exit.
pub fn graceful_exit(mut self) -> Result<(), SandboxCoordinatorError> {
self.control.exit()?;
if self.control.receive()? != SupervisorEvent::Exited {
if self.receive_runtime_event()? != SupervisorEvent::Exited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not acknowledge graceful exit",
));
@ -1132,7 +1143,7 @@ impl CommittedSandboxRuntime {
/// # Errors
/// Refuses any other event or an RS that does not then exit.
pub fn wait_for_attached_runsc_exit(mut self) -> Result<(), SandboxCoordinatorError> {
if self.control.receive()? != SupervisorEvent::AttachedRunscExited {
if self.receive_runtime_event()? != SupervisorEvent::AttachedRunscExited {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not report attached-runsc loss",
));
@ -1160,7 +1171,7 @@ impl CommittedSandboxRuntime {
) -> Result<(), SandboxCoordinatorError> {
coordinator.begin_release(self.generation_id)?;
self.control.stop()?;
if self.control.receive()? != SupervisorEvent::Stopped {
if self.receive_runtime_event()? != SupervisorEvent::Stopped {
return Err(SandboxCoordinatorError::InvalidState(
"Runtime Supervisor did not complete sandbox stop",
));

16
crates/lazaret-host-coordinator/tests/control.rs
View file

@ -516,6 +516,22 @@ fn duplicate_session_cli_and_anchor_identities_are_refused_while_registered() {
first_server.join().unwrap().unwrap();
}
#[test]
fn concurrent_connections_validate_publication_without_shared_file_offsets() {
let directory = TestDirectory::new("concurrent-publication");
let identity = generation_identity(7);
let (face, _store) = publish(&directory, &identity);
let clients = (40..56)
.map(|value| {
let face = Arc::clone(&face);
thread::spawn(move || enroll(&face, ClientId::new(bytes(value))))
})
.collect::<Vec<_>>();
for client in clients {
client.join().unwrap();
}
}
#[test]
fn anonymous_connections_are_enrollment_only_bounded_and_exact_alpn_is_required() {
let directory = TestDirectory::new("anonymous");

4
crates/lazaret-host-coordinator/tests/runtime-composition.rs
View file

@ -753,7 +753,7 @@ fn cross_bundle_handoff_refuses_target_work() -> Result<(), MatrixError> {
fn normal_stop_and_stop_resume() -> Result<(), MatrixError> {
let fixture = Fixture::new()?;
let runtime = fixture.launch_committed()?;
let mut runtime = fixture.launch_committed()?;
wait_for_activity(&fixture.activity)?;
runtime
.signal_attached_runsc(AttachedRunscSignal::Stop)
@ -789,7 +789,7 @@ fn normal_stop_and_stop_resume() -> Result<(), MatrixError> {
fn attached_runsc_loss_stops_guest() -> Result<(), MatrixError> {
let fixture = Fixture::new()?;
let runtime = fixture.launch_committed()?;
let mut runtime = fixture.launch_committed()?;
wait_for_activity(&fixture.activity)?;
runtime
.signal_attached_runsc(AttachedRunscSignal::Kill)

58
crates/lazaret-linux/src/process.rs
View file

@ -1070,12 +1070,12 @@ where
return Ok(PidfsRecoveryMatch::BootMismatch);
}
let host_pid = identity.host_pid().get();
let pidfd = open(host_pid)?;
let pidfd = normalize_process_absence(open(host_pid))?;
let observed = inspect(pidfd.as_fd(), current_boot)?;
if observed != identity.authority() {
return Err(io::Error::other("pidfs process authority mismatch"));
}
if start(host_pid)? != identity.start_time_ticks().get() {
if normalize_process_absence(start(host_pid))? != identity.start_time_ticks().get() {
return Err(io::Error::other("process start time mismatch"));
}
Ok(PidfsRecoveryMatch::Matched(RetainedProcess {
@ -1084,6 +1084,16 @@ where
}))
}
fn normalize_process_absence<T>(result: io::Result<T>) -> io::Result<T> {
result.map_err(|error| {
if error.raw_os_error() == Some(rustix::io::Errno::NOENT.raw_os_error()) {
io::Error::from_raw_os_error(rustix::io::Errno::SRCH.raw_os_error())
} else {
error
}
})
}
/// # Errors
/// Refuses if arming fails or the exact parent changes during the recheck.
pub fn arm_parent_death_and_recheck(
@ -1745,6 +1755,50 @@ mod tests {
}
}
#[test]
fn recovery_normalizes_vanished_proc_evidence_to_esrch() {
let authority = PidfsProcessAuthorityV1 {
boot_id: BootId::new([1; 16]),
pidfs_st_dev: PidfsDeviceId::new(7),
pidfs_st_ino: PidfsInodeId::new(9),
};
let identity = PidfsProcessIdentityV1::new(
authority,
HostPidLocator::new(41),
ProcessStartTimeTicks::new(43),
);
for missing in ["open", "start"] {
let result = match_recovery_process_with(
identity,
|| Ok(authority.boot_id),
|_| {
if missing == "open" {
Err(io::Error::from_raw_os_error(
rustix::io::Errno::NOENT.raw_os_error(),
))
} else {
Ok(File::open("/dev/null").unwrap().into())
}
},
|_, _| Ok(authority),
|_| {
if missing == "start" {
Err(io::Error::from_raw_os_error(
rustix::io::Errno::NOENT.raw_os_error(),
))
} else {
Ok(43)
}
},
)
.unwrap_err();
assert_eq!(
result.raw_os_error(),
Some(rustix::io::Errno::SRCH.raw_os_error())
);
}
}
#[test]
fn recovery_action_reconciles_boot_without_process_action() {
let mut old_boot = *read_boot_id().unwrap().as_bytes();

39
crates/lazaret-linux/tests/capability-drop-helper.rs
View file

@ -8,42 +8,55 @@ use rustix::thread::{
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
fn main() -> ExitCode {
if std::env::var_os("LAZARET_CAPABILITY_DROP_ISOLATED").is_none() {
let before = lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN)
.expect("inspect unprivileged capability state");
if lazaret_linux::capability::irreversibly_drop(CapabilitySet::NET_ADMIN).is_err()
|| lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN).ok() != Some(before)
{
return ExitCode::FAILURE;
}
return match std::process::Command::new("unshare")
let isolated = match std::process::Command::new("unshare")
.args(["--user", "--map-root-user"])
.arg(std::env::current_exe().expect("test executable path"))
.env("LAZARET_CAPABILITY_DROP_ISOLATED", "1")
.output()
{
Ok(output) if output.status.success() => ExitCode::SUCCESS,
Ok(output) if output.status.success() => true,
Ok(output)
if String::from_utf8_lossy(&output.stderr).contains("Operation not permitted") =>
{
eprintln!("SKIP isolated capability-positive branch: user namespaces unavailable");
ExitCode::SUCCESS
true
}
Ok(output) => {
eprintln!(
"isolated capability-positive branch failed: {}",
String::from_utf8_lossy(&output.stderr)
);
ExitCode::FAILURE
false
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
eprintln!("SKIP isolated capability-positive branch: unshare is unavailable");
ExitCode::SUCCESS
true
}
Err(error) => {
eprintln!("failed to run isolated capability-positive branch: {error}");
ExitCode::FAILURE
false
}
};
if !isolated {
return ExitCode::FAILURE;
}
let before = lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN)
.expect("inspect unprivileged capability state");
return match lazaret_linux::capability::irreversibly_drop(CapabilitySet::NET_ADMIN) {
Ok(())
if lazaret_linux::capability::require_regain_refused(CapabilitySet::NET_ADMIN)
.is_ok() =>
{
ExitCode::SUCCESS
}
Err(_)
if lazaret_linux::capability::inspect(CapabilitySet::NET_ADMIN).ok()
== Some(before) =>
{
ExitCode::SUCCESS
}
Ok(()) | Err(_) => ExitCode::FAILURE,
};
}
let capability = CapabilitySet::NET_ADMIN;

1
deny.toml
View file

@ -6,6 +6,7 @@ multiple-versions = "deny"
wildcards = "deny"
skip = [
{ name = "getrandom", version = "0.2.17", reason = "oci-client authentication uses jsonwebtoken's current getrandom line" },
{ name = "nix", version = "0.30.1", reason = "fuser 0.17.0 uses its current nix line while Lazaret directly uses 0.31.3" },
{ name = "oci-spec", version = "0.9.0", reason = "oci-client 0.17.0 uses its current OCI types while the runtime supervisor directly uses 0.10.0" },
{ name = "untrusted", version = "0.7.1", reason = "oci-client rustls authentication uses aws-lc-rs's compatibility line" },
]

22
tests/tooling/test_verify.py
View file

@ -119,6 +119,20 @@ class VerificationTests(unittest.TestCase):
self.assertFalse(VERIFY.license_permitted("GPL-3.0-only", allowed))
self.assertFalse(VERIFY.license_permitted("MIT AND GPL-3.0-only", allowed))
def test_only_the_exact_materialized_fuser_source_is_approved(self) -> None:
package = {
"name": "fuser",
"version": "0.17.0",
"manifest_path": str(VERIFY.MATERIALIZED_FUSER_MANIFEST),
}
self.assertTrue(VERIFY.is_materialized_fuser(package))
self.assertFalse(VERIFY.is_materialized_fuser({**package, "version": "0.18.0"}))
self.assertFalse(
VERIFY.is_materialized_fuser(
{**package, "manifest_path": str(ROOT / "fuser/Cargo.toml")}
)
)
def test_workspace_source_discovery_is_not_confined_to_crates(self) -> None:
with tempfile.TemporaryDirectory(dir=ROOT) as directory:
package_root = Path(directory)
@ -192,8 +206,12 @@ class VerificationTests(unittest.TestCase):
VERIFY.audit_process_syscalls({})
def test_process_syscall_audit_rejects_failed_binary(self) -> None:
with self.assertRaisesRegex(VERIFY.VerificationError, "exercised path exited 1"):
VERIFY.validate_trace_exit("example", 1)
for binary, returncode in VERIFY.PROCESS_TRACE_EXIT_CODES.items():
VERIFY.validate_trace_exit(binary, returncode)
with self.assertRaisesRegex(VERIFY.VerificationError, "expected"):
VERIFY.validate_trace_exit(binary, returncode + 1)
with self.assertRaisesRegex(VERIFY.VerificationError, "expected None"):
VERIFY.validate_trace_exit("example", 0)
def test_workspace_binary_discovery_covers_every_current_target(self) -> None:
self.assertEqual(

30
tools/lazaret_verify.py
View file

@ -14,6 +14,9 @@ import tempfile
import tomllib
ROOT = Path(__file__).resolve().parent.parent
MATERIALIZED_FUSER_MANIFEST = (
ROOT / ".lazaret/vendor/fuser-0.17.0/Cargo.toml"
).resolve()
MODEL_CONFIGURATIONS = (
(ROOT / "docs/protocol/Lazaret.tla", ROOT / "docs/protocol/Lazaret.cfg"),
(ROOT / "docs/protocol/PreparedLower.tla", ROOT / "docs/protocol/PreparedLower.cfg"),
@ -227,6 +230,16 @@ FORBIDDEN_PROCESS_SYSCALLS = tuple(
r"^rt_sigaction\(SIGCHLD,",
)
)
PROCESS_TRACE_EXIT_CODES = {
"lazaret": 2,
"lazaret-escalation-client": 2,
"lazaret-git-sentry": 1,
"lazaret-host-coordinator": 0,
"lazaret-ignored-workspace": 1,
"lazaret-launch-wrapper": 0,
"lazaret-runtime-supervisor": 0,
"lazaret-workspace-benchmark": 1,
}
LINT_BYPASS = re.compile(
r"#\s*!?\s*\[[^]]*(?:allow|expect)\s*\([^]]*"
r"(?:warnings|clippy\s*::\s*(?:all|style|restriction)|disallowed_methods|disallowed_types)"
@ -397,7 +410,8 @@ def audit_dependencies(metadata: dict) -> None:
if package["id"] not in workspace_members:
source = package.get("source") or ""
require(
source.startswith("registry+https://github.com/rust-lang/crates.io-index"),
source.startswith("registry+https://github.com/rust-lang/crates.io-index")
or is_materialized_fuser(package),
f"unapproved dependency source: {package['name']}",
)
license_expression = package.get("license") or ""
@ -429,6 +443,14 @@ def audit_dependencies(metadata: dict) -> None:
print("PASS dependency source, wildcard, and license audit")
def is_materialized_fuser(package: dict) -> bool:
return (
package.get("name") == "fuser"
and package.get("version") == "0.17.0"
and Path(package.get("manifest_path", "")).resolve() == MATERIALIZED_FUSER_MANIFEST
)
def license_permitted(expression: str, allowed: set[str]) -> bool:
expression = re.sub(r"\s+WITH\s+[A-Za-z0-9.-]+", "", expression)
expression = re.sub(r"\s*/\s*", " OR ", expression)
@ -573,7 +595,11 @@ def audit_process_syscalls(metadata: dict) -> None:
def validate_trace_exit(binary: str, returncode: int) -> None:
require(returncode == 0, f"{binary}: exercised path exited {returncode}")
expected = PROCESS_TRACE_EXIT_CODES.get(binary)
require(
returncode == expected,
f"{binary}: exercised path exited {returncode}, expected {expected}",
)
def refresh_advisories() -> None: