Bind cold start to production authorities

crates/lazaret-host-coordinator/src/lib.rs

@@ -6,5 +6,6 @@ pub mod domain_store;
 pub mod escalation;
 pub mod lifecycle;
 pub mod network;
+pub mod production;
 pub mod runtime;
 pub mod state_anchor;

crates/lazaret-host-coordinator/src/lifecycle.rs

@@ -282,17 +282,53 @@ where
         generation: Generation,
         session: Session,
     ) -> Result<(), LifecycleError<A::Error>> {
+        self.bootstrap_new_domain_while_owner_live(generation, session, || true)
+            .map(|_| ())
+    }
+
+    /// Drives cold start while checking the bootstrap owner's exact lifetime authority.
+    ///
+    /// Returns `false` after applying the modeled bootstrap-owner disconnect transition.
+    ///
+    /// # Errors
+    /// Returns the first invalid transition, durable update, or production adapter failure.
+    pub fn bootstrap_new_domain_while_owner_live(
+        &mut self,
+        generation: Generation,
+        session: Session,
+        mut owner_live: impl FnMut() -> bool,
+    ) -> Result<bool, LifecycleError<A::Error>> {
+        let session_id = session.session_id;
         self.start_new_domain(generation, session)?;
+        macro_rules! checkpoint {
+            () => {
+                if !owner_live() {
+                    self.disconnect_session(session_id)?;
+                    return Ok(false);
+                }
+            };
+        }
+        checkpoint!();
         self.create_trust_anchor()?;
+        checkpoint!();
         self.start_runtime_supervisor()?;
+        checkpoint!();
         self.lease_prepared_lower()?;
+        checkpoint!();
         self.mount_state_anchor()?;
+        checkpoint!();
         self.start_sidecar()?;
+        checkpoint!();
         self.start_connectivity_helper()?;
+        checkpoint!();
         self.mark_network_ready()?;
+        checkpoint!();
         self.arm_runtime_causal_edge()?;
+        checkpoint!();
         self.start_sandbox()?;
-        self.publish_domain()
+        checkpoint!();
+        self.publish_domain()?;
+        Ok(true)
     }
 
     /// # Errors

crates/lazaret-host-coordinator/src/production.rs

@@ -0,0 +1,445 @@
+use std::ffi::OsString;
+use std::fmt;
+use std::fs::{self, File, OpenOptions};
+use std::io::Write;
+use std::os::fd::{AsFd, OwnedFd};
+use std::os::unix::ffi::OsStringExt;
+use std::os::unix::fs::{FileExt, PermissionsExt};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex};
+
+use lazaret_core::configuration::ResolvedDomainConfiguration;
+use lazaret_core::identity::{BundleDigest, ContainerId};
+use lazaret_core::network::{ReachabilityExpectation, derive_ipv4_policy};
+use lazaret_core::records::{
+    Generation, GenerationIdentity, HostProcessOwnershipSlot, IpAddress, PreparedArtifact, Session,
+};
+use lazaret_linux::filesystem::namespace_identity;
+use lazaret_linux::process::ProcessOwnershipContext;
+use lazaret_runtime_supervisor::network::NetworkProbeRequest;
+use lazaret_runtime_supervisor::sandbox::{
+    RootProcess, SandboxBuildRequest, SandboxBundle, build_sandbox_bundle,
+};
+use lazaret_runtime_supervisor::supervision::{RuntimeNetworkLaunch, RuntimeSupervisorLaunch};
+use lazaret_state_anchor::{StateAnchorLaunchAuthority, StateAnchorLayout};
+use lazaret_storage_catalog::Catalog;
+use rustix::event::{PollFd, PollFlags, Timespec, poll};
+use rustix::fs::{FileType, fstat};
+use rustix::io::{FdFlags, fcntl_setfd};
+use sha2::{Digest, Sha256};
+
+use crate::bootstrap::{
+    ProductionBootstrapPlan, ProductionHostCoordinatorLifecycle, production_lifecycle,
+};
+use crate::domain_store::HostCoordinatorDomainStore;
+
+const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
+
+#[derive(Debug)]
+pub struct ProductionColdStartError(String);
+
+impl fmt::Display for ProductionColdStartError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str(&self.0)
+    }
+}
+
+impl std::error::Error for ProductionColdStartError {}
+
+fn failure(error: impl fmt::Display) -> ProductionColdStartError {
+    ProductionColdStartError(error.to_string())
+}
+
+pub struct ProductionColdStartInputs<'a> {
+    pub store: Arc<HostCoordinatorDomainStore>,
+    pub catalog: &'a Catalog,
+    pub configuration: &'a ResolvedDomainConfiguration,
+    pub generation: Generation,
+    pub prepared_artifact: &'a PreparedArtifact,
+    pub root_process: RootProcess,
+    pub resolver_addresses: Vec<IpAddress>,
+    pub network_probe: NetworkProbeRequest,
+    pub reachability: ReachabilityExpectation,
+    pub container_id: ContainerId,
+    pub invoking_uid: u32,
+    pub escalation_client: &'a File,
+}
+
+pub struct BootstrapOwnerGuard {
+    descriptor: OwnedFd,
+}
+
+impl BootstrapOwnerGuard {
+    /// Adopts an inherited bootstrap-owner lifetime pipe without PID sampling.
+    ///
+    /// # Errors
+    /// Refuses a descriptor that is not a pipe or cannot be confined to this process.
+    pub fn from_inherited(descriptor: OwnedFd) -> Result<Self, ProductionColdStartError> {
+        let metadata = fstat(&descriptor).map_err(failure)?;
+        if !FileType::from_raw_mode(metadata.st_mode).is_fifo() {
+            return Err(ProductionColdStartError(
+                "bootstrap-owner lifetime descriptor is not a pipe".into(),
+            ));
+        }
+        fcntl_setfd(descriptor.as_fd(), FdFlags::CLOEXEC).map_err(failure)?;
+        Ok(Self { descriptor })
+    }
+
+    fn is_live(&self) -> std::io::Result<bool> {
+        let mut descriptors = [PollFd::new(&self.descriptor, PollFlags::IN)];
+        poll(&mut descriptors, Some(&Timespec::default()))?;
+        Ok(!descriptors[0]
+            .revents()
+            .intersects(PollFlags::IN | PollFlags::HUP | PollFlags::ERR))
+    }
+}
+
+pub struct ProductionColdStart<'a> {
+    store: Arc<HostCoordinatorDomainStore>,
+    catalog: &'a Catalog,
+    configuration: &'a ResolvedDomainConfiguration,
+    generation: Generation,
+    generation_identity: GenerationIdentity,
+    prepared_artifact: &'a PreparedArtifact,
+    state_anchor_layout: StateAnchorLayout<'a>,
+    runtime_launch: Option<RuntimeSupervisorLaunch>,
+    sandbox_bundle: SandboxBundle,
+    network_policy: lazaret_core::network::CanonicalDerivedNetworkPolicy,
+    network_probe: NetworkProbeRequest,
+    reachability: ReachabilityExpectation,
+    container_id: ContainerId,
+    runtime_root: PathBuf,
+    bundle_digest: BundleDigest,
+    invoking_uid: u32,
+    _staging: RuntimeStaging,
+}
+
+impl<'a> ProductionColdStart<'a> {
+    /// Stages one production generation from validated component authorities.
+    ///
+    /// # Errors
+    /// Refuses mismatched generation/configuration/probe inputs, changed host authority, unsafe
+    /// staging paths, or component construction failure.
+    pub fn stage(inputs: ProductionColdStartInputs<'a>) -> Result<Self, ProductionColdStartError> {
+        require_inputs(&inputs)?;
+        let runtime_domain = host_path(
+            inputs
+                .configuration
+                .runtime_domain_root()
+                .canonical_label()
+                .as_bytes(),
+        );
+        let state_anchor_layout = StateAnchorLayout::create(inputs.configuration)
+            .map_err(|error| failure(format!("State Anchor layout: {error}")))?;
+        let staging = RuntimeStaging::create(
+            &runtime_domain,
+            inputs.container_id,
+            inputs.escalation_client,
+        )
+        .map_err(|error| failure(format!("runtime staging: {error}")))?;
+        let expected_network_namespace = namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))
+            .map_err(|error| failure(format!("network namespace: {error}")))?;
+        let sandbox_bundle = build_sandbox_bundle(SandboxBuildRequest {
+            configuration: inputs.configuration,
+            container_id: inputs.container_id,
+            expected_network_namespace,
+            root_process: inputs.root_process,
+            terminal: None,
+        })
+        .map_err(|error| failure(format!("sandbox bundle: {error}")))?;
+        staging
+            .write_bundle(sandbox_bundle.specification_json())
+            .map_err(|error| failure(format!("bundle staging: {error}")))?;
+        let network_policy = derive_ipv4_policy(inputs.configuration, inputs.resolver_addresses)
+            .map_err(|error| failure(format!("network policy: {error}")))?;
+        let ownership = ProcessOwnershipContext {
+            domain_id: inputs.generation.domain_id,
+            generation_id: inputs.generation.generation_id,
+            coordinator_id: inputs.generation.coordinator_id,
+            slot: HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
+        };
+        let state_anchor =
+            StateAnchorLaunchAuthority::capture(&state_anchor_layout, inputs.prepared_artifact)
+                .map_err(|error| failure(format!("State Anchor authority: {error}")))?;
+        let runtime_launch = RuntimeSupervisorLaunch::from_networked_runsc_command(
+            ownership,
+            sandbox_bundle.command(),
+            RuntimeNetworkLaunch {
+                policy: &network_policy,
+                configuration_fingerprint: inputs.generation.configuration_fingerprint,
+                network_sidecar_id: inputs.generation.network_sidecar_id,
+            },
+        )
+        .map_err(|error| failure(format!("Runtime Supervisor launch: {error}")))?
+        .with_state_anchor(state_anchor);
+        let bundle_digest =
+            BundleDigest::new(Sha256::digest(sandbox_bundle.specification_json()).into());
+        let generation_identity = GenerationIdentity {
+            agent_profile: inputs.configuration.selected_profile().to_owned(),
+            domain_id: inputs.generation.domain_id,
+            generation_id: inputs.generation.generation_id,
+            configuration_fingerprint: inputs.generation.configuration_fingerprint,
+            coordinator_id: inputs.generation.coordinator_id,
+            sandbox_id: inputs.generation.sandbox_id,
+        };
+        Ok(Self {
+            store: inputs.store,
+            catalog: inputs.catalog,
+            configuration: inputs.configuration,
+            generation: inputs.generation,
+            generation_identity,
+            prepared_artifact: inputs.prepared_artifact,
+            state_anchor_layout,
+            runtime_launch: Some(runtime_launch),
+            sandbox_bundle,
+            network_policy,
+            network_probe: inputs.network_probe,
+            reachability: inputs.reachability,
+            container_id: inputs.container_id,
+            runtime_root: runtime_domain.join("runsc"),
+            bundle_digest,
+            invoking_uid: inputs.invoking_uid,
+            _staging: staging,
+        })
+    }
+
+    /// Runs cold start and exposes the exact retained lifecycle to one production service loop.
+    ///
+    /// # Errors
+    /// Returns staging, lifecycle, or service-loop failure. The service is single-use.
+    pub fn run<Service>(
+        mut self,
+        bootstrap_owner: Session,
+        owner_guard: &BootstrapOwnerGuard,
+        service: Service,
+    ) -> Result<(), ProductionColdStartError>
+    where
+        Service: FnOnce(
+            &Mutex<ProductionHostCoordinatorLifecycle<'_, '_>>,
+        ) -> Result<(), ProductionColdStartError>,
+    {
+        let store = Arc::clone(&self.store);
+        let generation = self.generation.clone();
+        let plan = self.take_plan()?;
+        let mut lifecycle = production_lifecycle(&store, plan).map_err(failure)?;
+        let published = lifecycle
+            .bootstrap_new_domain_while_owner_live(generation, bootstrap_owner, || {
+                owner_guard.is_live().unwrap_or(false)
+            })
+            .map_err(failure)?;
+        if !published {
+            return Err(ProductionColdStartError(
+                "bootstrap owner disconnected before publication".into(),
+            ));
+        }
+        service(&Mutex::new(lifecycle))
+    }
+
+    fn take_plan(&mut self) -> Result<ProductionBootstrapPlan<'_>, ProductionColdStartError> {
+        Ok(ProductionBootstrapPlan {
+            store: Arc::clone(&self.store),
+            catalog: self.catalog,
+            configuration: self.configuration,
+            generation_identity: self.generation_identity.clone(),
+            prepared_artifact: self.prepared_artifact,
+            state_anchor_layout: &self.state_anchor_layout,
+            runtime_launch: self.runtime_launch.take().ok_or_else(|| {
+                ProductionColdStartError("production cold start was already consumed".into())
+            })?,
+            sandbox_bundle: &self.sandbox_bundle,
+            network_policy: &self.network_policy,
+            network_probe: self.network_probe.clone(),
+            reachability: &self.reachability,
+            container_id: self.container_id,
+            runtime_root: self.runtime_root.clone(),
+            bundle_digest: self.bundle_digest,
+            invoking_uid: self.invoking_uid,
+        })
+    }
+}
+
+fn require_inputs(inputs: &ProductionColdStartInputs<'_>) -> Result<(), ProductionColdStartError> {
+    if inputs.generation.domain_id != inputs.configuration.domain_id()
+        || inputs.generation.configuration_fingerprint
+            != inputs.configuration.configuration_fingerprint()
+        || inputs.network_probe.domain_id != inputs.generation.domain_id
+        || inputs.network_probe.generation_id != inputs.generation.generation_id
+        || inputs.resolver_addresses.is_empty()
+    {
+        return Err(ProductionColdStartError(
+            "production cold-start inputs disagree".into(),
+        ));
+    }
+    Ok(())
+}
+
+struct RuntimeStaging {
+    bundle: PathBuf,
+    control: PathBuf,
+}
+
+impl RuntimeStaging {
+    fn create(
+        runtime_domain: &Path,
+        container_id: ContainerId,
+        escalation_client: &File,
+    ) -> Result<Self, ProductionColdStartError> {
+        let name = hex(container_id.as_bytes());
+        let bundles = runtime_domain.join("bundles");
+        let controls = runtime_domain.join("sandbox-control");
+        let runsc = runtime_domain.join("runsc");
+        for directory in [&bundles, &controls, &runsc] {
+            ensure_directory(directory)
+                .map_err(|error| failure(format!("creating {}: {error}", directory.display())))?;
+        }
+        let bundle = bundles.join(&name);
+        let control = controls.join(name);
+        fs::create_dir(&bundle)
+            .map_err(|error| failure(format!("creating bundle leaf: {error}")))?;
+        set_private_directory(&bundle)?;
+        if let Err(error) = fs::create_dir(&control) {
+            let _ = fs::remove_dir(&bundle);
+            return Err(failure(error));
+        }
+        if let Err(error) = set_private_directory(&control) {
+            let _ = fs::remove_dir(&control);
+            let _ = fs::remove_dir(&bundle);
+            return Err(error);
+        }
+        let staging = Self { bundle, control };
+        staging.populate_control(escalation_client)?;
+        Ok(staging)
+    }
+
+    fn populate_control(&self, escalation_client: &File) -> Result<(), ProductionColdStartError> {
+        for relative in [
+            "credentials",
+            "mailbox",
+            "mailbox/requests",
+            "mailbox/results",
+            "bin",
+        ] {
+            let directory = self.control.join(relative);
+            fs::create_dir(&directory)
+                .map_err(|error| failure(format!("creating control {relative}: {error}")))?;
+            set_private_directory(&directory)?;
+        }
+        let source = escalation_client
+            .metadata()
+            .map_err(|error| failure(format!("inspecting escalation client: {error}")))?;
+        if !source.file_type().is_file() || source.permissions().mode() & 0o111 == 0 {
+            return Err(ProductionColdStartError(
+                "escalation client is not an executable regular file".into(),
+            ));
+        }
+        copy_descriptor(
+            escalation_client,
+            &self.control.join("bin/lazaret-escalate"),
+        )?;
+        Ok(())
+    }
+
+    fn write_bundle(&self, bytes: &[u8]) -> Result<(), ProductionColdStartError> {
+        let path = self.bundle.join("config.json");
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create_new(true)
+            .open(path)
+            .map_err(failure)?;
+        file.write_all(bytes).map_err(failure)?;
+        file.set_permissions(fs::Permissions::from_mode(0o600))
+            .map_err(failure)?;
+        file.sync_all().map_err(failure)?;
+        Ok(())
+    }
+}
+
+fn copy_descriptor(source: &File, destination: &Path) -> Result<(), ProductionColdStartError> {
+    let mut destination = OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(destination)
+        .map_err(failure)?;
+    destination
+        .set_permissions(fs::Permissions::from_mode(0o700))
+        .map_err(failure)?;
+    let mut offset = 0_u64;
+    let mut buffer = [0_u8; 16 * 1024];
+    loop {
+        let read = source.read_at(&mut buffer, offset).map_err(failure)?;
+        if read == 0 {
+            break;
+        }
+        destination.write_all(&buffer[..read]).map_err(failure)?;
+        offset = offset
+            .checked_add(read as u64)
+            .ok_or_else(|| ProductionColdStartError("escalation client is too large".into()))?;
+    }
+    destination.sync_all().map_err(failure)
+}
+
+fn set_private_directory(path: &Path) -> Result<(), ProductionColdStartError> {
+    fs::set_permissions(path, fs::Permissions::from_mode(0o700)).map_err(failure)
+}
+
+impl Drop for RuntimeStaging {
+    fn drop(&mut self) {
+        let _ = fs::remove_dir_all(&self.control);
+        let _ = fs::remove_dir_all(&self.bundle);
+    }
+}
+
+fn ensure_directory(path: &Path) -> Result<(), ProductionColdStartError> {
+    match fs::create_dir(path) {
+        Ok(()) => Ok(()),
+        Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
+            let metadata = fs::symlink_metadata(path).map_err(failure)?;
+            if metadata.file_type().is_dir() {
+                Ok(())
+            } else {
+                Err(ProductionColdStartError(
+                    "production staging path is not a directory".into(),
+                ))
+            }
+        }
+        Err(error) => Err(failure(error)),
+    }
+}
+
+fn host_path(bytes: &[u8]) -> PathBuf {
+    PathBuf::from(OsString::from_vec(bytes.to_vec()))
+}
+
+fn hex(bytes: &[u8]) -> String {
+    use std::fmt::Write as _;
+
+    bytes.iter().fold(String::new(), |mut output, byte| {
+        write!(output, "{byte:02x}").expect("writing to a String cannot fail");
+        output
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs::File;
+    use std::os::fd::OwnedFd;
+
+    use rustix::pipe::{PipeFlags, pipe_with};
+
+    use super::BootstrapOwnerGuard;
+
+    #[test]
+    fn bootstrap_owner_guard_tracks_the_exact_inherited_pipe() {
+        let (reader, writer) = pipe_with(PipeFlags::CLOEXEC).unwrap();
+        let guard = BootstrapOwnerGuard::from_inherited(reader).unwrap();
+        assert!(guard.is_live().unwrap());
+        drop(writer);
+        assert!(!guard.is_live().unwrap());
+    }
+
+    #[test]
+    fn bootstrap_owner_guard_refuses_non_pipe_authority() {
+        let descriptor: OwnedFd = File::open("/dev/null").unwrap().into();
+        assert!(BootstrapOwnerGuard::from_inherited(descriptor).is_err());
+    }
+}