Implement supervised rootless network readiness
This commit is contained in:
parent
407b75b1ff
commit
ba6a075839
1 changed files with 2917 additions and 54 deletions
|
|
@ -1,7 +1,9 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::net::{Ipv4Addr, SocketAddrV4, TcpListener, TcpStream, ToSocketAddrs};
|
||||
use std::os::linux::net::SocketAddrExt;
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::os::unix::net::UnixListener;
|
||||
use std::os::unix::net::{SocketAddr, UnixListener, UnixStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
use std::thread;
|
||||
|
|
@ -13,11 +15,13 @@ use lazaret_core::configuration::{
|
|||
use lazaret_core::identity::{
|
||||
BundleDigest, ContainerId, CoordinatorId, GenerationId, HostPath, NetworkSidecarId, SandboxId,
|
||||
};
|
||||
use lazaret_core::network::{DnsQueryExpectation, ReachabilityExpectation, derive_ipv4_policy};
|
||||
use lazaret_core::records::{
|
||||
Domain, DurableDecision, Generation, HostProcessOwnershipSlot, ParticipantRole,
|
||||
Domain, DurableDecision, Generation, HostProcessOwnershipSlot, IpAddress, ParticipantRole,
|
||||
ParticipantState, ProcessOwnershipCommitAck,
|
||||
};
|
||||
use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
|
||||
use lazaret_host_coordinator::network::NetworkCoordinator;
|
||||
use lazaret_host_coordinator::runtime::{
|
||||
AwaitingAttachedRunsc, CommittedSandboxRuntime, SandboxCoordinator,
|
||||
};
|
||||
|
|
@ -26,18 +30,22 @@ use lazaret_linux::platform::{PlatformObservation, check_platform};
|
|||
use lazaret_linux::process::{
|
||||
ProcessOwnershipContext, RetainedProcess, TargetCommand, child_broker, current_process_identity,
|
||||
};
|
||||
use lazaret_runtime_supervisor::network::{DnsProbeTarget, NetworkProbeRequest, TcpProbeTarget};
|
||||
use lazaret_runtime_supervisor::sandbox::{
|
||||
NETWORK_NAMESPACE_PATH, RootProcess, SandboxBuildRequest, SandboxBundle, build_sandbox_bundle,
|
||||
production_sandbox_layout,
|
||||
};
|
||||
use lazaret_runtime_supervisor::supervision::{
|
||||
AttachedRunscSignal, RuntimeSupervisorLaunch, dispatch_private_runtime_role,
|
||||
AttachedRunscSignal, RuntimeNetworkLaunch, RuntimeSupervisorLaunch,
|
||||
dispatch_private_runtime_role,
|
||||
};
|
||||
use rustix::process::Signal;
|
||||
use tempfile::TempDir;
|
||||
|
||||
const SKIP: u8 = 77;
|
||||
const HOST_COORDINATOR_DRIVER: &str = "--runtime-composition-host-coordinator";
|
||||
const SANDBOX_NETWORK_PROBE: &str = "--sandbox-network-probe";
|
||||
const NETWORK_MATRIX: &str = "--network-matrix";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
|
||||
|
|
@ -50,6 +58,25 @@ fn main() -> ExitCode {
|
|||
if arguments.first().and_then(|argument| argument.to_str()) == Some(HOST_COORDINATOR_DRIVER) {
|
||||
return exit(run_host_coordinator_driver(&arguments[1..]));
|
||||
}
|
||||
if arguments.first().and_then(|argument| argument.to_str()) == Some(SANDBOX_NETWORK_PROBE) {
|
||||
return exit(run_sandbox_network_probe(&arguments[1..]));
|
||||
}
|
||||
if arguments.first().and_then(|argument| argument.to_str()) == Some(NETWORK_MATRIX) {
|
||||
return match run_network_matrix() {
|
||||
Ok(()) => {
|
||||
println!("PASS production network composition matrix");
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
Err(MatrixError::Skip(reason)) => {
|
||||
println!("SKIP network composition: {reason}");
|
||||
ExitCode::from(SKIP)
|
||||
}
|
||||
Err(MatrixError::Failure(error)) => {
|
||||
eprintln!("network composition failed: {error}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
};
|
||||
}
|
||||
match run_matrix() {
|
||||
Ok(()) => {
|
||||
println!("PASS foreground runtime composition and causal-loss matrix");
|
||||
|
|
@ -88,6 +115,26 @@ impl From<std::io::Error> for MatrixError {
|
|||
}
|
||||
|
||||
fn run_matrix() -> Result<(), MatrixError> {
|
||||
require_matrix_prerequisites()?;
|
||||
lost_acknowledgment_refuses_target_work()?;
|
||||
normal_stop_and_stop_resume()?;
|
||||
attached_runsc_loss_stops_guest()?;
|
||||
runtime_supervisor_loss_stops_guest()?;
|
||||
host_coordinator_loss_stops_guest()?;
|
||||
run_network_cases()
|
||||
}
|
||||
|
||||
fn run_network_matrix() -> Result<(), MatrixError> {
|
||||
require_matrix_prerequisites()?;
|
||||
run_network_cases()
|
||||
}
|
||||
|
||||
fn run_network_cases() -> Result<(), MatrixError> {
|
||||
failed_probe_blocks_sandbox_start()?;
|
||||
networked_sandbox_view_isolated()
|
||||
}
|
||||
|
||||
fn require_matrix_prerequisites() -> Result<(), MatrixError> {
|
||||
let release = fs::read_to_string("/proc/sys/kernel/osrelease")?;
|
||||
check_platform(&PlatformObservation {
|
||||
operating_system: std::env::consts::OS,
|
||||
|
|
@ -95,12 +142,7 @@ fn run_matrix() -> Result<(), MatrixError> {
|
|||
kernel_release: release.trim(),
|
||||
})
|
||||
.map_err(|error| MatrixError::Skip(error.to_string()))?;
|
||||
require_runtime_prerequisites()?;
|
||||
lost_acknowledgment_refuses_target_work()?;
|
||||
normal_stop_and_stop_resume()?;
|
||||
attached_runsc_loss_stops_guest()?;
|
||||
runtime_supervisor_loss_stops_guest()?;
|
||||
host_coordinator_loss_stops_guest()
|
||||
require_runtime_prerequisites()
|
||||
}
|
||||
|
||||
fn require_runtime_prerequisites() -> Result<(), MatrixError> {
|
||||
|
|
@ -115,6 +157,8 @@ fn require_runtime_prerequisites() -> Result<(), MatrixError> {
|
|||
PathBuf::from("/bin/sh"),
|
||||
PathBuf::from("/bin/sleep"),
|
||||
PathBuf::from("/lib64/libc.so.6"),
|
||||
PathBuf::from("/lib64/libgcc_s.so.1"),
|
||||
PathBuf::from("/lib64/libm.so.6"),
|
||||
PathBuf::from("/lib64/libtinfo.so.6"),
|
||||
PathBuf::from("/lib64/ld-linux-x86-64.so.2"),
|
||||
] {
|
||||
|
|
@ -409,8 +453,8 @@ impl Fixture {
|
|||
})
|
||||
.map_err(failure)?;
|
||||
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
|
||||
stage_rootfs_and_control(&runtime_domain)?;
|
||||
let container_id = ContainerId::new([6; 32]);
|
||||
stage_rootfs_and_control(&runtime_domain, container_id)?;
|
||||
let root_process = RootProcess::new(
|
||||
"/bin/sh",
|
||||
[
|
||||
|
|
@ -502,7 +546,299 @@ impl Fixture {
|
|||
}
|
||||
}
|
||||
|
||||
fn stage_rootfs_and_control(runtime_domain: &Path) -> Result<(), MatrixError> {
|
||||
struct NetworkFixture {
|
||||
_tree: TempDir,
|
||||
_host_loopback: TcpListener,
|
||||
_host_abstract: UnixListener,
|
||||
configuration: ResolvedDomainConfiguration,
|
||||
store: HostCoordinatorDomainStore,
|
||||
bundle: SandboxBundle,
|
||||
policy: lazaret_core::network::CanonicalDerivedNetworkPolicy,
|
||||
probe_result: PathBuf,
|
||||
}
|
||||
|
||||
impl NetworkFixture {
|
||||
fn new() -> Result<Self, MatrixError> {
|
||||
let tree = tempfile::tempdir()?;
|
||||
let root = tree.path();
|
||||
let configuration_directory = root.join("configuration");
|
||||
let workspace = root.join("workspace");
|
||||
let runtime = root.join("runtime");
|
||||
fs::create_dir(&configuration_directory)?;
|
||||
fs::create_dir(&workspace)?;
|
||||
fs::create_dir(&runtime)?;
|
||||
let configuration_path = configuration_directory.join("lazaret.toml");
|
||||
fs::write(
|
||||
&configuration_path,
|
||||
format!(
|
||||
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[profiles.default]\nimage = \"example.invalid/network-fixture:v1\"\npersistent_dirs = []\n",
|
||||
root.join("data").display()
|
||||
),
|
||||
)?;
|
||||
let configuration = load_trusted_configuration(ConfigurationLoadRequest {
|
||||
explicit_configuration_path: Some(configuration_path),
|
||||
selected_profile: "default".into(),
|
||||
workspace: workspace.clone(),
|
||||
home: root.join("home"),
|
||||
xdg_config_home: None,
|
||||
xdg_data_home: None,
|
||||
xdg_runtime_dir: runtime,
|
||||
environment: BTreeMap::new(),
|
||||
sandbox_layout: production_sandbox_layout().map_err(failure)?,
|
||||
})
|
||||
.map_err(failure)?;
|
||||
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
|
||||
let container_id = ContainerId::new([16; 32]);
|
||||
stage_rootfs_and_control(&runtime_domain, container_id)?;
|
||||
let rootfs = runtime_domain.join("state-anchor/merged");
|
||||
fs::copy(
|
||||
std::env::current_exe()?,
|
||||
rootfs.join("bin/lazaret-network-probe"),
|
||||
)?;
|
||||
fs::create_dir_all(rootfs.join("etc"))?;
|
||||
fs::write(rootfs.join("etc/resolv.conf"), b"nameserver 10.0.2.3\n")?;
|
||||
let host_loopback = TcpListener::bind((Ipv4Addr::LOCALHOST, 0))?;
|
||||
let host_loopback_port = host_loopback.local_addr()?.port();
|
||||
let abstract_name = format!("lazaret-network-v3-{}", std::process::id()).into_bytes();
|
||||
let host_abstract =
|
||||
UnixListener::bind_addr(&SocketAddr::from_abstract_name(&abstract_name)?)?;
|
||||
let root_process = RootProcess::new(
|
||||
"/bin/lazaret-network-probe",
|
||||
vec![
|
||||
SANDBOX_NETWORK_PROBE.into(),
|
||||
host_loopback_port.to_string(),
|
||||
String::from_utf8(abstract_name.clone()).map_err(failure)?,
|
||||
],
|
||||
)
|
||||
.map_err(failure)?;
|
||||
let namespace =
|
||||
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
|
||||
let bundle = build_sandbox_bundle(SandboxBuildRequest {
|
||||
configuration: &configuration,
|
||||
container_id,
|
||||
expected_network_namespace: namespace,
|
||||
root_process,
|
||||
terminal: None,
|
||||
})
|
||||
.map_err(failure)?;
|
||||
let bundle_path = runtime_domain
|
||||
.join("bundles")
|
||||
.join(hex(container_id.as_bytes()));
|
||||
fs::create_dir_all(&bundle_path)?;
|
||||
fs::write(bundle_path.join("config.json"), bundle.specification_json())?;
|
||||
fs::create_dir_all(runtime_domain.join("runsc"))?;
|
||||
let domain = Domain {
|
||||
domain_id: configuration.domain_id(),
|
||||
profile_name: configuration.selected_profile().to_owned(),
|
||||
canonical_workspace: configuration.workspace().canonical_label().clone(),
|
||||
};
|
||||
let persistent_domain = host_path(configuration.persistent_domain_root().canonical_label());
|
||||
let lock = try_exclusive_flock(lock_file(&persistent_domain.join("domain.lock")))?;
|
||||
let store =
|
||||
HostCoordinatorDomainStore::open(lock, &persistent_domain.join("domain.redb"), &domain)
|
||||
.map_err(failure)?;
|
||||
store
|
||||
.apply(&[DomainStoreMutation::PutGeneration(Generation {
|
||||
domain_id: configuration.domain_id(),
|
||||
generation_id: GenerationId::new(7),
|
||||
configuration_fingerprint: configuration.configuration_fingerprint(),
|
||||
coordinator_id: CoordinatorId::new([13; 32]),
|
||||
network_sidecar_id: NetworkSidecarId::new([14; 32]),
|
||||
sandbox_id: SandboxId::new([15; 32]),
|
||||
durable_decision: DurableDecision::None,
|
||||
})])
|
||||
.map_err(failure)?;
|
||||
SandboxCoordinator::new(&store)
|
||||
.begin_intent(
|
||||
GenerationId::new(7),
|
||||
container_id,
|
||||
&runtime_domain.join("runsc"),
|
||||
BundleDigest::new([17; 32]),
|
||||
current_process_identity()?,
|
||||
)
|
||||
.map_err(failure)?;
|
||||
let policy =
|
||||
derive_ipv4_policy(&configuration, [IpAddress::V4([10, 0, 2, 3])]).map_err(failure)?;
|
||||
Ok(Self {
|
||||
_tree: tree,
|
||||
_host_loopback: host_loopback,
|
||||
_host_abstract: host_abstract,
|
||||
configuration,
|
||||
store,
|
||||
bundle,
|
||||
policy,
|
||||
probe_result: workspace.join("network-probe"),
|
||||
})
|
||||
}
|
||||
|
||||
fn launch(&self) -> Result<CommittedSandboxRuntime, MatrixError> {
|
||||
self.launch_with_permitted([1, 1, 1, 1], 443)
|
||||
}
|
||||
|
||||
fn launch_with_permitted(
|
||||
&self,
|
||||
permitted_address: [u8; 4],
|
||||
permitted_port: u16,
|
||||
) -> Result<CommittedSandboxRuntime, MatrixError> {
|
||||
let launch = RuntimeSupervisorLaunch::from_networked_runsc_command(
|
||||
ProcessOwnershipContext {
|
||||
domain_id: self.configuration.domain_id(),
|
||||
generation_id: GenerationId::new(7),
|
||||
coordinator_id: CoordinatorId::new([13; 32]),
|
||||
slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
|
||||
},
|
||||
self.bundle.command(),
|
||||
RuntimeNetworkLaunch {
|
||||
policy: &self.policy,
|
||||
configuration_fingerprint: self.configuration.configuration_fingerprint(),
|
||||
network_sidecar_id: NetworkSidecarId::new([14; 32]),
|
||||
},
|
||||
)?;
|
||||
let sandbox = SandboxCoordinator::new(&self.store);
|
||||
let network = NetworkCoordinator::new(&self.store);
|
||||
let mut pending = AwaitingAttachedRunsc::launch_networked(&sandbox, &network, &launch)
|
||||
.map_err(failure)?;
|
||||
pending.receive_helper_staged(&network).map_err(failure)?;
|
||||
pending
|
||||
.receive_network_readiness(
|
||||
&network,
|
||||
&self.policy,
|
||||
|namespace| NetworkProbeRequest {
|
||||
domain_id: self.configuration.domain_id(),
|
||||
generation_id: GenerationId::new(7),
|
||||
enforcement_namespace: namespace,
|
||||
denied: vec![TcpProbeTarget {
|
||||
address: IpAddress::V4([10, 1, 2, 3]),
|
||||
port: 443,
|
||||
}],
|
||||
permitted: vec![TcpProbeTarget {
|
||||
address: IpAddress::V4(permitted_address),
|
||||
port: permitted_port,
|
||||
}],
|
||||
dns: vec![DnsProbeTarget {
|
||||
resolver: IpAddress::V4([10, 0, 2, 3]),
|
||||
query_name: "example.com".into(),
|
||||
}],
|
||||
},
|
||||
&ReachabilityExpectation {
|
||||
denied_destinations: vec![IpAddress::V4([10, 1, 2, 3])],
|
||||
permitted_destinations: vec![IpAddress::V4(permitted_address)],
|
||||
dns_queries: vec![DnsQueryExpectation {
|
||||
resolver: IpAddress::V4([10, 0, 2, 3]),
|
||||
query_name: "example.com".into(),
|
||||
}],
|
||||
},
|
||||
)
|
||||
.map_err(failure)?;
|
||||
let staged = pending.receive_staged(&sandbox).map_err(failure)?;
|
||||
let (runtime, _) = pending
|
||||
.acknowledge(staged)
|
||||
.map_err(failure)?
|
||||
.commit_networked(&sandbox)
|
||||
.map_err(failure)?;
|
||||
Ok(runtime)
|
||||
}
|
||||
}
|
||||
|
||||
fn failed_probe_blocks_sandbox_start() -> Result<(), MatrixError> {
|
||||
let fixture = NetworkFixture::new()?;
|
||||
match fixture.launch_with_permitted([203, 0, 113, 1], 9) {
|
||||
Ok(runtime) => {
|
||||
runtime
|
||||
.stop(&SandboxCoordinator::new(&fixture.store))
|
||||
.map_err(failure)?;
|
||||
Err(failure("failed physical probe started the sandbox"))
|
||||
}
|
||||
Err(_) => {
|
||||
let sandbox = sandbox_participant(&fixture.store);
|
||||
if sandbox.state() != ParticipantState::Intent || fixture.probe_result.exists() {
|
||||
Err(failure(
|
||||
"failed physical probe advanced Sandbox state or guest work",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn networked_sandbox_view_isolated() -> Result<(), MatrixError> {
|
||||
let fixture = NetworkFixture::new()?;
|
||||
let runtime = fixture.launch()?;
|
||||
for _ in 0..6000 {
|
||||
if fixture.probe_result.is_file() {
|
||||
break;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(5));
|
||||
}
|
||||
let result = fs::read_to_string(&fixture.probe_result)?;
|
||||
for required in [
|
||||
"denied=true",
|
||||
"public=true",
|
||||
"dns=true",
|
||||
"host_loopback=false",
|
||||
"abstract_socket=false",
|
||||
] {
|
||||
if !result.lines().any(|line| line == required) {
|
||||
return Err(failure(format!(
|
||||
"guest network assertion missing: {required}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
runtime
|
||||
.stop(&SandboxCoordinator::new(&fixture.store))
|
||||
.map_err(failure)
|
||||
}
|
||||
|
||||
fn run_sandbox_network_probe(arguments: &[std::ffi::OsString]) -> std::io::Result<()> {
|
||||
let host_port = arguments
|
||||
.first()
|
||||
.and_then(|value| value.to_str())
|
||||
.and_then(|value| value.parse::<u16>().ok())
|
||||
.filter(|port| *port != 0)
|
||||
.ok_or_else(|| std::io::Error::other("sandbox probe host port is invalid"))?;
|
||||
let abstract_name = arguments
|
||||
.get(1)
|
||||
.and_then(|value| value.to_str())
|
||||
.ok_or_else(|| std::io::Error::other("sandbox probe abstract name is invalid"))?;
|
||||
let timeout = Duration::from_secs(3);
|
||||
let denied = TcpStream::connect_timeout(
|
||||
&SocketAddrV4::new(Ipv4Addr::new(10, 1, 2, 3), 443).into(),
|
||||
timeout,
|
||||
)
|
||||
.is_err();
|
||||
let public = TcpStream::connect_timeout(
|
||||
&SocketAddrV4::new(Ipv4Addr::new(1, 1, 1, 1), 443).into(),
|
||||
timeout,
|
||||
)
|
||||
.is_ok();
|
||||
let dns = ("example.com", 443)
|
||||
.to_socket_addrs()
|
||||
.is_ok_and(|mut addresses| addresses.next().is_some());
|
||||
let host_loopback = TcpStream::connect_timeout(
|
||||
&SocketAddrV4::new(Ipv4Addr::new(10, 0, 2, 2), host_port).into(),
|
||||
timeout,
|
||||
)
|
||||
.is_ok();
|
||||
let abstract_socket =
|
||||
UnixStream::connect_addr(&SocketAddr::from_abstract_name(abstract_name.as_bytes())?)
|
||||
.is_ok();
|
||||
fs::write(
|
||||
"/workspace/.network-probe",
|
||||
format!(
|
||||
"denied={denied}\npublic={public}\ndns={dns}\nhost_loopback={host_loopback}\nabstract_socket={abstract_socket}\n"
|
||||
),
|
||||
)?;
|
||||
fs::rename("/workspace/.network-probe", "/workspace/network-probe")?;
|
||||
loop {
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
fn stage_rootfs_and_control(
|
||||
runtime_domain: &Path,
|
||||
container_id: ContainerId,
|
||||
) -> Result<(), MatrixError> {
|
||||
let rootfs = runtime_domain.join("state-anchor/merged");
|
||||
for directory in ["bin", "lib64"] {
|
||||
fs::create_dir_all(rootfs.join(directory))?;
|
||||
|
|
@ -511,12 +847,16 @@ fn stage_rootfs_and_control(runtime_domain: &Path) -> Result<(), MatrixError> {
|
|||
("/bin/sh", "bin/sh"),
|
||||
("/bin/sleep", "bin/sleep"),
|
||||
("/lib64/libc.so.6", "lib64/libc.so.6"),
|
||||
("/lib64/libgcc_s.so.1", "lib64/libgcc_s.so.1"),
|
||||
("/lib64/libm.so.6", "lib64/libm.so.6"),
|
||||
("/lib64/libtinfo.so.6", "lib64/libtinfo.so.6"),
|
||||
("/lib64/ld-linux-x86-64.so.2", "lib64/ld-linux-x86-64.so.2"),
|
||||
] {
|
||||
fs::copy(source, rootfs.join(destination))?;
|
||||
}
|
||||
let control = runtime_domain.join("sandbox-control").join(hex(&[6; 32]));
|
||||
let control = runtime_domain
|
||||
.join("sandbox-control")
|
||||
.join(hex(container_id.as_bytes()));
|
||||
for directory in ["credentials", "mailbox/requests", "mailbox/results", "bin"] {
|
||||
fs::create_dir_all(control.join(directory))?;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue