Confine runtime staging to retained authority

crates/lazaret-host-coordinator/src/production.rs

@@ -1,6 +1,6 @@
 use std::ffi::OsString;
 use std::fmt;
-use std::fs::{self, File, OpenOptions};
+use std::fs::File;
 use std::io::Write;
 use std::os::fd::{AsFd, OwnedFd};
 use std::os::unix::ffi::OsStringExt;
@@ -8,7 +8,7 @@ use std::os::unix::fs::{FileExt, PermissionsExt};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
-use lazaret_core::configuration::ResolvedDomainConfiguration;
+use lazaret_core::configuration::{ConfinedPathPlan, ResolvedDomainConfiguration};
 use lazaret_core::identity::{BundleDigest, ContainerId};
 use lazaret_core::network::{ReachabilityExpectation, derive_ipv4_policy};
 use lazaret_core::records::{
@@ -24,7 +24,10 @@ use lazaret_runtime_supervisor::supervision::{RuntimeNetworkLaunch, RuntimeSuper
 use lazaret_state_anchor::{StateAnchorLaunchAuthority, StateAnchorLayout};
 use lazaret_storage_catalog::Catalog;
 use rustix::event::{PollFd, PollFlags, Timespec, poll};
-use rustix::fs::{FileType, fstat};
+use rustix::fs::{
+    AtFlags, FileType, Mode, OFlags, RawDir, ResolveFlags, fchmod, fstat, mkdirat, openat2, statat,
+    unlinkat,
+};
 use rustix::io::{FdFlags, fcntl_setfd};
 use sha2::{Digest, Sha256};
 
@@ -132,9 +135,10 @@ impl<'a> ProductionColdStart<'a> {
         let state_anchor_layout = StateAnchorLayout::create(inputs.configuration)
             .map_err(|error| failure(format!("State Anchor layout: {error}")))?;
         let staging = RuntimeStaging::create(
-            &runtime_domain,
+            inputs.configuration.runtime_domain_root(),
             inputs.container_id,
             inputs.escalation_client,
+            inputs.invoking_uid,
         )
         .map_err(|error| failure(format!("runtime staging: {error}")))?;
         let expected_network_namespace = namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))
@@ -265,6 +269,7 @@ fn require_inputs(inputs: &ProductionColdStartInputs<'_>) -> Result<(), Producti
         || inputs.network_probe.domain_id != inputs.generation.domain_id
         || inputs.network_probe.generation_id != inputs.generation.generation_id
         || inputs.resolver_addresses.is_empty()
+        || inputs.invoking_uid != rustix::process::getuid().as_raw()
     {
         return Err(ProductionColdStartError(
             "production cold-start inputs disagree".into(),
@@ -274,56 +279,79 @@ fn require_inputs(inputs: &ProductionColdStartInputs<'_>) -> Result<(), Producti
 }
 
 struct RuntimeStaging {
-    bundle: PathBuf,
+    bundles: OwnedFd,
-    control: PathBuf,
+    bundle: OwnedFd,
+    controls: OwnedFd,
+    control: OwnedFd,
+    name: OsString,
+    bundle_identity: FileIdentity,
+    control_identity: FileIdentity,
 }
 
 impl RuntimeStaging {
     fn create(
-        runtime_domain: &Path,
+        runtime_domain: &ConfinedPathPlan,
         container_id: ContainerId,
         escalation_client: &File,
+        invoking_uid: u32,
     ) -> Result<Self, ProductionColdStartError> {
-        let name = hex(container_id.as_bytes());
+        let root = runtime_domain.open_authority().map_err(failure)?;
-        let bundles = runtime_domain.join("bundles");
+        let bundles = open_or_create_private_directory(&root, "bundles", invoking_uid, false)?;
-        let controls = runtime_domain.join("sandbox-control");
+        let controls =
-        let runsc = runtime_domain.join("runsc");
+            open_or_create_private_directory(&root, "sandbox-control", invoking_uid, false)?;
-        for directory in [&bundles, &controls, &runsc] {
+        drop(open_or_create_private_directory(
-            ensure_directory(directory)
+            &root,
-                .map_err(|error| failure(format!("creating {}: {error}", directory.display())))?;
+            "runsc",
-        }
+            invoking_uid,
-        let bundle = bundles.join(&name);
+            false,
-        let control = controls.join(name);
+        )?);
-        fs::create_dir(&bundle)
+        let name = OsString::from(hex(container_id.as_bytes()));
-            .map_err(|error| failure(format!("creating bundle leaf: {error}")))?;
+        let bundle = open_or_create_private_directory(&bundles, &name, invoking_uid, true)?;
-        set_private_directory(&bundle)?;
+        let bundle_identity = FileIdentity::capture(&bundle)?;
-        if let Err(error) = fs::create_dir(&control) {
+        let control = match open_or_create_private_directory(&controls, &name, invoking_uid, true) {
-            let _ = fs::remove_dir(&bundle);
+            Ok(control) => control,
-            return Err(failure(error));
+            Err(error) => {
-        }
+                remove_staging_directory(&bundles, &name, &bundle, bundle_identity);
-        if let Err(error) = set_private_directory(&control) {
-            let _ = fs::remove_dir(&control);
-            let _ = fs::remove_dir(&bundle);
                 return Err(error);
             }
-        let staging = Self { bundle, control };
+        };
+        let control_identity = FileIdentity::capture(&control)?;
+        let staging = Self {
+            bundles,
+            bundle,
+            controls,
+            control,
+            name,
+            bundle_identity,
+            control_identity,
+        };
         staging.populate_control(escalation_client)?;
         Ok(staging)
     }
 
     fn populate_control(&self, escalation_client: &File) -> Result<(), ProductionColdStartError> {
-        for relative in [
+        let invoking_uid = fstat(&self.control).map_err(failure)?.st_uid;
+        drop(open_or_create_private_directory(
+            &self.control,
             "credentials",
-            "mailbox",
+            invoking_uid,
-            "mailbox/requests",
+            true,
-            "mailbox/results",
+        )?);
-            "bin",
+        let mailbox =
-        ] {
+            open_or_create_private_directory(&self.control, "mailbox", invoking_uid, true)?;
-            let directory = self.control.join(relative);
+        drop(open_or_create_private_directory(
-            fs::create_dir(&directory)
+            &mailbox,
-                .map_err(|error| failure(format!("creating control {relative}: {error}")))?;
+            "requests",
-            set_private_directory(&directory)?;
+            invoking_uid,
-        }
+            true,
+        )?);
+        drop(open_or_create_private_directory(
+            &mailbox,
+            "results",
+            invoking_uid,
+            true,
+        )?);
+        let bin = open_or_create_private_directory(&self.control, "bin", invoking_uid, true)?;
         let source = escalation_client
             .metadata()
             .map_err(|error| failure(format!("inspecting escalation client: {error}")))?;
@@ -332,37 +360,44 @@ impl RuntimeStaging {
                 "escalation client is not an executable regular file".into(),
             ));
         }
-        copy_descriptor(
+        copy_descriptor(escalation_client, &bin, "lazaret-escalate", invoking_uid)?;
-            escalation_client,
-            &self.control.join("bin/lazaret-escalate"),
-        )?;
         Ok(())
     }
 
     fn write_bundle(&self, bytes: &[u8]) -> Result<(), ProductionColdStartError> {
-        let path = self.bundle.join("config.json");
+        let descriptor = openat2(
-        let mut file = OpenOptions::new()
+            &self.bundle,
-            .write(true)
+            "config.json",
-            .create_new(true)
+            OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
-            .open(path)
+            Mode::RUSR | Mode::WUSR,
+            confined_resolve_flags(),
+        )
         .map_err(failure)?;
+        let mut file = File::from(descriptor);
         file.write_all(bytes).map_err(failure)?;
-        file.set_permissions(fs::Permissions::from_mode(0o600))
+        fchmod(file.as_fd(), Mode::RUSR | Mode::WUSR).map_err(failure)?;
-            .map_err(failure)?;
         file.sync_all().map_err(failure)?;
+        require_private_file(&file, fstat(&self.bundle).map_err(failure)?.st_uid, 0o600)?;
         Ok(())
     }
 }
 
-fn copy_descriptor(source: &File, destination: &Path) -> Result<(), ProductionColdStartError> {
+fn copy_descriptor(
-    let mut destination = OpenOptions::new()
+    source: &File,
-        .write(true)
+    parent: &OwnedFd,
-        .create_new(true)
+    name: &str,
-        .open(destination)
+    invoking_uid: u32,
-        .map_err(failure)?;
+) -> Result<(), ProductionColdStartError> {
-    destination
+    let descriptor = openat2(
-        .set_permissions(fs::Permissions::from_mode(0o700))
+        parent,
+        name,
+        OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC | OFlags::NOFOLLOW,
+        Mode::RWXU,
+        confined_resolve_flags(),
+    )
     .map_err(failure)?;
+    let mut destination = File::from(descriptor);
+    fchmod(destination.as_fd(), Mode::RWXU).map_err(failure)?;
     let mut offset = 0_u64;
     let mut buffer = [0_u8; 16 * 1024];
     loop {
@@ -375,35 +410,171 @@ fn copy_descriptor(source: &File, destination: &Path) -> Result<(), ProductionCo
             .checked_add(read as u64)
             .ok_or_else(|| ProductionColdStartError("escalation client is too large".into()))?;
     }
-    destination.sync_all().map_err(failure)
+    destination.sync_all().map_err(failure)?;
+    require_private_file(&destination, invoking_uid, 0o700)
 }
 
-fn set_private_directory(path: &Path) -> Result<(), ProductionColdStartError> {
+#[derive(Clone, Copy)]
-    fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(failure)
+struct FileIdentity {
+    device: u64,
+    inode: u64,
+}
+
+impl FileIdentity {
+    fn capture(descriptor: impl AsFd) -> Result<Self, ProductionColdStartError> {
+        let metadata = fstat(descriptor).map_err(failure)?;
+        Ok(Self {
+            device: metadata.st_dev,
+            inode: metadata.st_ino,
+        })
+    }
+
+    fn matches(self, metadata: &rustix::fs::Stat) -> bool {
+        metadata.st_dev == self.device && metadata.st_ino == self.inode
+    }
 }
 
 impl Drop for RuntimeStaging {
     fn drop(&mut self) {
-        let _ = fs::remove_dir_all(&self.control);
+        remove_staging_directory(
-        let _ = fs::remove_dir_all(&self.bundle);
+            &self.controls,
+            &self.name,
+            &self.control,
+            self.control_identity,
+        );
+        remove_staging_directory(
+            &self.bundles,
+            &self.name,
+            &self.bundle,
+            self.bundle_identity,
+        );
     }
 }
 
-fn ensure_directory(path: &Path) -> Result<(), ProductionColdStartError> {
+fn open_or_create_private_directory(
-    match fs::create_dir(path) {
+    parent: impl AsFd,
-        Ok(()) => Ok(()),
+    name: impl AsRef<std::ffi::OsStr>,
-        Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
+    invoking_uid: u32,
-            let metadata = fs::symlink_metadata(path).map_err(failure)?;
+    require_new: bool,
-            if metadata.file_type().is_dir() {
+) -> Result<OwnedFd, ProductionColdStartError> {
+    let name = name.as_ref();
+    let created = match mkdirat(parent.as_fd(), name, Mode::RWXU) {
+        Ok(()) => true,
+        Err(rustix::io::Errno::EXIST) if !require_new => false,
+        Err(error) => return Err(failure(error)),
+    };
+    let directory = openat2(
+        parent.as_fd(),
+        name,
+        OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
+        Mode::empty(),
+        confined_resolve_flags(),
+    )
+    .map_err(failure)?;
+    if created {
+        fchmod(&directory, Mode::RWXU).map_err(failure)?;
+    }
+    require_private_directory(&directory, invoking_uid)?;
+    Ok(directory)
+}
+
+fn require_private_directory(
+    descriptor: impl AsFd,
+    invoking_uid: u32,
+) -> Result<(), ProductionColdStartError> {
+    let metadata = fstat(descriptor).map_err(failure)?;
+    if !FileType::from_raw_mode(metadata.st_mode).is_dir()
+        || metadata.st_uid != invoking_uid
+        || metadata.st_mode & 0o7777 != 0o700
+    {
+        return Err(ProductionColdStartError(
+            "runtime staging directory is not invoking-user-owned mode 0700".into(),
+        ));
+    }
     Ok(())
+}
+
+fn require_private_file(
+    descriptor: impl AsFd,
+    invoking_uid: u32,
+    mode: u32,
+) -> Result<(), ProductionColdStartError> {
+    let metadata = fstat(descriptor).map_err(failure)?;
+    if !FileType::from_raw_mode(metadata.st_mode).is_file()
+        || metadata.st_uid != invoking_uid
+        || metadata.st_mode & 0o7777 != mode
+        || metadata.st_nlink != 1
+    {
+        return Err(ProductionColdStartError(
+            "runtime staging file has unsafe type, ownership, mode, or link count".into(),
+        ));
+    }
+    Ok(())
+}
+
+fn remove_staging_directory(
+    parent: impl AsFd,
+    name: &std::ffi::OsStr,
+    directory: impl AsFd,
+    expected: FileIdentity,
+) {
+    let retained = fstat(directory.as_fd());
+    if !retained
+        .as_ref()
+        .is_ok_and(|metadata| expected.matches(metadata))
+    {
+        return;
+    }
+    if remove_children(directory.as_fd()).is_err() {
+        return;
+    }
+    let Ok(current) = statat(parent.as_fd(), name, AtFlags::SYMLINK_NOFOLLOW) else {
+        return;
+    };
+    if expected.matches(&current) && FileType::from_raw_mode(current.st_mode).is_dir() {
+        let _ = unlinkat(parent.as_fd(), name, AtFlags::REMOVEDIR);
+    }
+}
+
+fn remove_children(directory: impl AsFd) -> std::io::Result<()> {
+    let duplicate = rustix::io::dup(directory.as_fd())?;
+    let mut buffer = [std::mem::MaybeUninit::uninit(); 8192];
+    let names = {
+        let mut names = Vec::new();
+        let mut entries = RawDir::new(duplicate, &mut buffer);
+        while let Some(entry) = entries.next() {
+            let entry = entry?;
+            let bytes = entry.file_name().to_bytes();
+            if bytes != b"." && bytes != b".." {
+                names.push(OsString::from_vec(bytes.to_vec()));
+            }
+        }
+        names
+    };
+    for name in names {
+        let metadata = statat(directory.as_fd(), &name, AtFlags::SYMLINK_NOFOLLOW)?;
+        if FileType::from_raw_mode(metadata.st_mode).is_dir() {
+            let child = openat2(
+                directory.as_fd(),
+                &name,
+                OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
+                Mode::empty(),
+                confined_resolve_flags(),
+            )?;
+            remove_children(&child)?;
+            unlinkat(directory.as_fd(), &name, AtFlags::REMOVEDIR)?;
         } else {
-                Err(ProductionColdStartError(
+            unlinkat(directory.as_fd(), &name, AtFlags::empty())?;
-                    "production staging path is not a directory".into(),
-                ))
         }
     }
-        Err(error) => Err(failure(error)),
+    Ok(())
-    }
+}
+
+fn confined_resolve_flags() -> ResolveFlags {
+    ResolveFlags::BENEATH
+        | ResolveFlags::NO_MAGICLINKS
+        | ResolveFlags::NO_SYMLINKS
+        | ResolveFlags::NO_XDEV
 }
 
 fn host_path(bytes: &[u8]) -> PathBuf {

crates/lazaret-host-coordinator/tests/production.rs

@@ -23,6 +23,81 @@ use rustix::process::getuid;
 
 #[test]
 fn validated_inputs_stage_one_complete_plan_and_drop_exact_runtime_artifacts() {
+    let fixture = ProductionFixture::new();
+    let staged = ProductionColdStart::stage(fixture.inputs(&fixture.escalation_client)).unwrap();
+    let bundle = fixture.bundle();
+    let control = fixture.control();
+    assert_private_artifacts(&bundle, &control);
+    drop(staged);
+    assert!(!bundle.exists());
+    assert!(!control.exists());
+    assert!(
+        ProductionColdStart::stage(fixture.inputs(&fixture.invalid_escalation_client)).is_err()
+    );
+    assert!(!bundle.exists());
+    assert!(!control.exists());
+}
+
+#[test]
+fn cleanup_leaves_a_replacement_directory_untouched() {
+    let fixture = ProductionFixture::new();
+    let staged = ProductionColdStart::stage(fixture.inputs(&fixture.escalation_client)).unwrap();
+    let bundle = fixture.bundle();
+    let moved = bundle.with_extension("moved");
+    fs::rename(&bundle, &moved).unwrap();
+    fs::create_dir(&bundle).unwrap();
+    fs::write(bundle.join("replacement"), b"trusted replacement").unwrap();
+    drop(staged);
+    assert_eq!(
+        fs::read(bundle.join("replacement")).unwrap(),
+        b"trusted replacement"
+    );
+    assert!(moved.is_dir());
+}
+
+#[test]
+fn fixed_runtime_parents_refuse_symlinks_and_wrong_modes() {
+    let fixture = ProductionFixture::new();
+    let bundles = fixture.runtime_domain.join("bundles");
+    fs::create_dir(&bundles).unwrap();
+    fs::set_permissions(&bundles, fs::Permissions::from_mode(0o755)).unwrap();
+    assert!(ProductionColdStart::stage(fixture.inputs(&fixture.escalation_client)).is_err());
+
+    fs::remove_dir(&bundles).unwrap();
+    let controls = fixture.runtime_domain.join("sandbox-control");
+    let redirect = fixture.tree.path().join("redirect");
+    fs::create_dir(&redirect).unwrap();
+    std::os::unix::fs::symlink(&redirect, &controls).unwrap();
+    assert!(ProductionColdStart::stage(fixture.inputs(&fixture.escalation_client)).is_err());
+    assert!(redirect.read_dir().unwrap().next().is_none());
+}
+
+#[test]
+fn replaced_runtime_domain_authority_is_not_used_for_staging() {
+    let fixture = ProductionFixture::new();
+    let moved = fixture.runtime_domain.with_extension("moved");
+    fs::rename(&fixture.runtime_domain, &moved).unwrap();
+    fs::create_dir(&fixture.runtime_domain).unwrap();
+    fs::set_permissions(&fixture.runtime_domain, fs::Permissions::from_mode(0o700)).unwrap();
+    assert!(ProductionColdStart::stage(fixture.inputs(&fixture.escalation_client)).is_err());
+    assert!(fixture.runtime_domain.read_dir().unwrap().next().is_none());
+}
+
+struct ProductionFixture {
+    tree: tempfile::TempDir,
+    configuration: lazaret_core::configuration::ResolvedDomainConfiguration,
+    runtime_domain: PathBuf,
+    catalog: Catalog,
+    store: Arc<HostCoordinatorDomainStore>,
+    artifact: PreparedArtifact,
+    generation: Generation,
+    container_id: ContainerId,
+    escalation_client: File,
+    invalid_escalation_client: File,
+}
+
+impl ProductionFixture {
+    fn new() -> Self {
         let tree = tempfile::tempdir().unwrap();
         let configuration = configuration(tree.path());
         let persistent_domain = host_path(
@@ -84,19 +159,32 @@ fn validated_inputs_stage_one_complete_plan_and_drop_exact_runtime_artifacts() {
             sandbox_id: SandboxId::new([4; 32]),
             durable_decision: DurableDecision::None,
         };
-    let container_id = ContainerId::new([5; 32]);
+        Self {
-    let escalation_client = File::open("/bin/true").unwrap();
+            tree,
-    let invalid_escalation_client = File::open("/dev/null").unwrap();
+            configuration,
-    let inputs = |escalation_client| ProductionColdStartInputs {
+            runtime_domain,
-        store: Arc::clone(&store),
+            catalog,
-        catalog: &catalog,
+            store,
-        configuration: &configuration,
+            artifact,
-        generation: generation.clone(),
+            generation,
-        prepared_artifact: &artifact,
+            container_id: ContainerId::new([5; 32]),
-        root_process: RootProcess::new("/bin/sh", ["-c".into(), "sleep infinity".into()]).unwrap(),
+            escalation_client: File::open("/bin/true").unwrap(),
+            invalid_escalation_client: File::open("/dev/null").unwrap(),
+        }
+    }
+
+    fn inputs<'a>(&'a self, escalation_client: &'a File) -> ProductionColdStartInputs<'a> {
+        ProductionColdStartInputs {
+            store: Arc::clone(&self.store),
+            catalog: &self.catalog,
+            configuration: &self.configuration,
+            generation: self.generation.clone(),
+            prepared_artifact: &self.artifact,
+            root_process: RootProcess::new("/bin/sh", ["-c".into(), "sleep infinity".into()])
+                .unwrap(),
             resolver_addresses: vec![IpAddress::V4([10, 0, 2, 3])],
             network_probe: NetworkProbeRequest {
-            domain_id: configuration.domain_id(),
+                domain_id: self.configuration.domain_id(),
                 generation_id: GenerationId::new(1),
                 enforcement_namespace: namespace_identity(Path::new("/proc/self/ns/net")).unwrap(),
                 denied: Vec::new(),
@@ -108,21 +196,23 @@ fn validated_inputs_stage_one_complete_plan_and_drop_exact_runtime_artifacts() {
                 permitted_destinations: Vec::new(),
                 dns_queries: Vec::new(),
             },
-        container_id,
+            container_id: self.container_id,
             invoking_uid: getuid().as_raw(),
             escalation_client,
-    };
+        }
-    let staged = ProductionColdStart::stage(inputs(&escalation_client)).unwrap();
+    }
-    let name = hex(container_id.as_bytes());
+
-    let bundle = runtime_domain.join("bundles").join(&name);
+    fn bundle(&self) -> PathBuf {
-    let control = runtime_domain.join("sandbox-control").join(&name);
+        self.runtime_domain
-    assert_private_artifacts(&bundle, &control);
+            .join("bundles")
-    drop(staged);
+            .join(hex(self.container_id.as_bytes()))
-    assert!(!bundle.exists());
+    }
-    assert!(!control.exists());
+
-    assert!(ProductionColdStart::stage(inputs(&invalid_escalation_client)).is_err());
+    fn control(&self) -> PathBuf {
-    assert!(!bundle.exists());
+        self.runtime_domain
-    assert!(!control.exists());
+            .join("sandbox-control")
+            .join(hex(self.container_id.as_bytes()))
+    }
 }
 
 fn assert_private_artifacts(bundle: &Path, control: &Path) {