Make Git router teardown process-bounded

crates/lazaret-host-coordinator/tests/runtime-composition.rs

@@ -25,6 +25,7 @@ use lazaret_core::records::{
     SessionTerminalMessage, TerminalResize, TerminalSignal, TerminalSignalKind,
 };
 use lazaret_host_coordinator::domain_store::{DomainStoreMutation, HostCoordinatorDomainStore};
+use lazaret_host_coordinator::git_routing::GitRoutingCoordinator;
 use lazaret_host_coordinator::network::NetworkCoordinator;
 use lazaret_host_coordinator::runtime::{
     AwaitingAttachedRunsc, CommittedSandboxRuntime, SandboxCoordinator,
@@ -42,10 +43,11 @@ use lazaret_linux::process::{
 use lazaret_linux::user_namespace::UserNamespaceMap;
 use lazaret_runtime_supervisor::sandbox::{
     NETWORK_NAMESPACE_PATH, RootProcess, RunscCommand, SandboxBuildRequest, SandboxBundle,
-    build_sandbox_bundle, default_sandbox_layout,
+    build_sandbox_bundle, build_sandbox_bundle_with_workspace, default_sandbox_layout,
 };
 use lazaret_runtime_supervisor::supervision::{
-    AttachedRunscSignal, RuntimeNetworkLaunch, RuntimeSupervisorLaunch,
+    AttachedRunscSignal, RuntimeGitRoutingAuthority, RuntimeGitRoutingLaunch, RuntimeNetworkLaunch,
+    RuntimeSupervisorLaunch,
 };
 use rustix::process::Signal;
 use tempfile::TempDir;
@@ -250,6 +252,7 @@ fn run_matrix() -> Result<(), MatrixError> {
     require_matrix_prerequisites()?;
     lost_acknowledgment_refuses_target_work()?;
     replaced_handoff_authority_refuses_target_work()?;
+    git_routing_clean_stop()?;
     normal_stop_and_stop_resume()?;
     attached_runsc_loss_stops_guest()?;
     runtime_supervisor_loss_stops_guest()?;
@@ -622,11 +625,11 @@ mod network;
 
 use fixtures::{Fixture, NetworkFixture};
 use handoff::{
-    attached_runsc_loss_stops_guest, failure, host_coordinator_loss_stops_guest,
+    attached_runsc_loss_stops_guest, failure, git_routing_clean_stop,
-    lost_acknowledgment_refuses_target_work, matrix_io, normal_stop_and_stop_resume,
+    host_coordinator_loss_stops_guest, lost_acknowledgment_refuses_target_work, matrix_io,
-    replaced_handoff_authority_refuses_target_work, require_matrix_prerequisites,
+    normal_stop_and_stop_resume, replaced_handoff_authority_refuses_target_work,
-    run_handoff_matrix, run_host_coordinator_driver, run_network_cases,
+    require_matrix_prerequisites, run_handoff_matrix, run_host_coordinator_driver,
-    runtime_supervisor_loss_stops_guest,
+    run_network_cases, runtime_supervisor_loss_stops_guest,
 };
 use network::{
     hex, host_path, lock_file, networked_sandbox_view_isolated, run_sandbox_network_probe,

crates/lazaret-host-coordinator/tests/runtime-composition/fixtures.rs

@@ -1,15 +1,17 @@
 use super::{
     Arc, AwaitingAttachedRunsc, BENCHMARK_GUEST, BTreeMap, BundleDigest, CommittedSandboxRuntime,
     ConfigurationLoadRequest, ContainerId, CoordinatorId, Domain, DomainStoreMutation,
-    DurableDecision, Generation, GenerationId, HostCoordinatorDomainStore,
+    DurableDecision, File, Generation, GenerationId, GitRoutingCoordinator,
-    HostProcessOwnershipSlot, Ipv4Addr, MatrixError, NETWORK_NAMESPACE_PATH, NetworkCoordinator,
+    HostCoordinatorDomainStore, HostPath, HostProcessOwnershipSlot, Ipv4Addr, MatrixError,
-    NetworkSidecarId, Path, PathBuf, ProcessOwnershipContext, ResolvedDomainConfiguration,
+    NETWORK_NAMESPACE_PATH, NetworkCoordinator, NetworkSidecarId, Path, PathBuf,
-    RootProcess, RunscCommand, RuntimeNetworkLaunch, RuntimeSupervisorLaunch, SandboxBuildRequest,
+    ProcessOwnershipContext, ResolvedDomainConfiguration, RootProcess, RunscCommand,
-    SandboxBundle, SandboxCoordinator, SandboxId, SocketAddr, SocketAddrExt, TcpListener, TempDir,
+    RuntimeGitRoutingAuthority, RuntimeGitRoutingLaunch, RuntimeNetworkLaunch,
-    UnixListener, UserNamespaceMap, build_sandbox_bundle, current_mount_identity,
+    RuntimeSupervisorLaunch, SandboxBuildRequest, SandboxBundle, SandboxCoordinator, SandboxId,
-    current_process_identity, default_sandbox_layout, derive_ipv4_policy, failure, fs, hex,
+    SocketAddr, SocketAddrExt, TargetCommand, TcpListener, TempDir, UnixListener, UserNamespaceMap,
-    host_path, load_trusted_configuration, lock_file, stage_benchmark_rootfs_and_control,
+    build_sandbox_bundle, build_sandbox_bundle_with_workspace, child_broker,
-    stage_rootfs_and_control, try_exclusive_flock,
+    current_mount_identity, current_process_identity, default_sandbox_layout, derive_ipv4_policy,
+    failure, fs, hex, host_path, load_trusted_configuration, lock_file,
+    stage_benchmark_rootfs_and_control, stage_rootfs_and_control, try_exclusive_flock,
 };
 
 pub(super) struct Fixture {
@@ -19,17 +21,24 @@ pub(super) struct Fixture {
     pub(super) bundle: SandboxBundle,
     pub(super) activity: PathBuf,
     pub(super) benchmark_results: Option<PathBuf>,
+    git_routing: Option<GitRoutingFixture>,
 }
 
 impl Fixture {
     pub(super) fn new() -> Result<Self, MatrixError> {
         let tree = tempfile::tempdir()?;
         let root = tree.path().to_path_buf();
-        Self::build(&root, Some(tree))
+        Self::build(&root, Some(tree), false)
+    }
+
+    pub(super) fn new_git_routed() -> Result<Self, MatrixError> {
+        let tree = tempfile::tempdir()?;
+        let root = tree.path().to_path_buf();
+        Self::build(&root, Some(tree), true)
     }
 
     pub(super) fn in_directory(root: &Path) -> Result<Self, MatrixError> {
-        Self::build(root, None)
+        Self::build(root, None, false)
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -136,16 +145,24 @@ impl Fixture {
             bundle,
             workspace.join("activity"),
             Some(results),
+            None,
         )
     }
 
-    fn build(root: &Path, tree: Option<TempDir>) -> Result<Self, MatrixError> {
+    fn build(
+        root: &Path,
+        tree: Option<TempDir>,
+        route_git_workspace: bool,
+    ) -> Result<Self, MatrixError> {
         let configuration_directory = root.join("configuration");
         let workspace = root.join("workspace");
         let runtime = root.join("runtime");
         fs::create_dir(&configuration_directory)?;
         fs::create_dir(&workspace)?;
         fs::create_dir(&runtime)?;
+        let git_routing = route_git_workspace
+            .then(|| GitRoutingFixture::create(root, &workspace))
+            .transpose()?;
         let configuration_path = configuration_directory.join("lazaret.toml");
         fs::write(
             &configuration_path,
@@ -180,14 +197,20 @@ impl Fixture {
         .map_err(failure)?;
         let namespace =
             lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
-        let bundle = build_sandbox_bundle(SandboxBuildRequest {
+        let request = SandboxBuildRequest {
             configuration: &configuration,
             container_id,
             expected_network_namespace: namespace,
             root_process,
             terminal: None,
             prepared_root: runtime_domain.join("prepared-root"),
-        })
+        };
+        let bundle = match &git_routing {
+            Some(git_routing) => {
+                build_sandbox_bundle_with_workspace(request, git_routing.mountpoint.clone())
+            }
+            None => build_sandbox_bundle(request),
+        }
         .map_err(failure)?;
         Self::finish_build(
             tree,
@@ -195,6 +218,7 @@ impl Fixture {
             bundle,
             workspace.join("activity"),
             None,
+            git_routing,
         )
     }
 
@@ -204,6 +228,7 @@ impl Fixture {
         bundle: SandboxBundle,
         activity: PathBuf,
         benchmark_results: Option<PathBuf>,
+        git_routing: Option<GitRoutingFixture>,
     ) -> Result<Self, MatrixError> {
         let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
         let container_id = ContainerId::new([6; 32]);
@@ -252,6 +277,7 @@ impl Fixture {
             bundle,
             activity,
             benchmark_results,
+            git_routing,
         })
     }
 
@@ -263,7 +289,7 @@ impl Fixture {
         &self,
         command: &RunscCommand,
     ) -> Result<AwaitingAttachedRunsc, MatrixError> {
-        let launch = RuntimeSupervisorLaunch::from_runsc_command(
+        let mut launch = RuntimeSupervisorLaunch::from_runsc_command(
             ProcessOwnershipContext {
                 domain_id: self.configuration.domain_id(),
                 generation_id: GenerationId::new(7),
@@ -271,10 +297,40 @@ impl Fixture {
                 slot: lazaret_core::records::HostProcessOwnershipSlot::SandboxRuntimeSupervisor,
             },
             command,
-        )?
+        )?;
-        .with_user_namespace_map(UserNamespaceMap::resolve_current().map_err(failure)?)?;
+        if let Some(git_routing) = &self.git_routing {
-        AwaitingAttachedRunsc::launch(&SandboxCoordinator::new(&self.store), &launch)
+            launch = launch.with_git_routing(git_routing.launch()?)?;
-            .map_err(failure)
+        }
+        launch = launch
+            .with_user_namespace_map(UserNamespaceMap::resolve_current().map_err(failure)?)?;
+        let mut pending =
+            AwaitingAttachedRunsc::launch(&SandboxCoordinator::new(&self.store), &launch)
+                .map_err(failure)?;
+        if let Some(git_routing) = &self.git_routing {
+            let coordinator = GitRoutingCoordinator::new(&self.store);
+            coordinator
+                .begin_intent(
+                    GenerationId::new(7),
+                    HostPath::new(
+                        git_routing
+                            .mountpoint
+                            .as_os_str()
+                            .as_encoded_bytes()
+                            .to_vec(),
+                    ),
+                    HostPath::new(
+                        git_routing
+                            .workspace
+                            .as_os_str()
+                            .as_encoded_bytes()
+                            .to_vec(),
+                    ),
+                    HostPath::new(git_routing.isolated.as_os_str().as_encoded_bytes().to_vec()),
+                )
+                .map_err(failure)?;
+            pending.start_git_routing(&coordinator).map_err(failure)?;
+        }
+        Ok(pending)
     }
 
     pub(super) fn alternate_bundle(&self) -> Result<SandboxBundle, MatrixError> {
@@ -316,6 +372,68 @@ impl Fixture {
     }
 }
 
+struct GitRoutingFixture {
+    workspace: PathBuf,
+    isolated: PathBuf,
+    mountpoint: PathBuf,
+    git_common: PathBuf,
+    transient_hooks: PathBuf,
+    transient_config: PathBuf,
+}
+
+impl GitRoutingFixture {
+    fn create(root: &Path, workspace: &Path) -> Result<Self, MatrixError> {
+        child_broker()
+            .run_transient(
+                &TargetCommand::new("git")
+                    .argument("init")
+                    .argument("--quiet")
+                    .argument("--initial-branch=main")
+                    .argument(workspace),
+            )
+            .map_err(failure)?;
+        let routing = root.join("git-routing");
+        let isolated = routing.join("isolated");
+        let mountpoint = routing.join("workspace");
+        let transient_hooks = routing.join("hooks");
+        let transient_config = routing.join("config");
+        for directory in [&isolated, &mountpoint, &transient_hooks] {
+            fs::create_dir_all(directory)?;
+        }
+        fs::write(&transient_config, b"")?;
+        Ok(Self {
+            workspace: workspace.to_path_buf(),
+            isolated,
+            mountpoint,
+            git_common: workspace.join(".git"),
+            transient_hooks,
+            transient_config,
+        })
+    }
+
+    fn launch(&self) -> Result<RuntimeGitRoutingLaunch, MatrixError> {
+        RuntimeGitRoutingLaunch::new(RuntimeGitRoutingAuthority {
+            workspace: File::open(&self.workspace)?,
+            workspace_label: self.workspace.clone(),
+            isolated: File::open(&self.isolated)?,
+            mountpoint: File::open(&self.mountpoint)?,
+            mountpoint_label: self.mountpoint.clone(),
+            git_common: File::open(&self.git_common)?,
+            git_common_label: self.git_common.clone(),
+            transient_hooks: File::open(&self.transient_hooks)?,
+            transient_hooks_label: self.transient_hooks.clone(),
+            transient_config: File::options()
+                .read(true)
+                .write(true)
+                .open(&self.transient_config)?,
+            transient_config_label: self.transient_config.clone(),
+            global_excludes: None,
+            isolate_ignored: true,
+        })
+        .map_err(failure)
+    }
+}
+
 pub(super) struct NetworkFixture {
     _tree: TempDir,
     _host_loopback: TcpListener,

crates/lazaret-host-coordinator/tests/runtime-composition/handoff.rs

@@ -1,10 +1,11 @@
 use super::{
-    AttachedRunscSignal, CoordinatorId, Duration, Fixture, GenerationId, HOST_COORDINATOR_DRIVER,
+    AttachedRunscSignal, CoordinatorId, Duration, Fixture, GenerationId, GitRoutingCoordinator,
-    HostProcessOwnershipSlot, MatrixError, ParticipantState, Path, PathBuf, PidfsProcessIdentityV1,
+    HOST_COORDINATOR_DRIVER, HostProcessOwnershipSlot, MatrixError, ParticipantRole,
-    PlatformObservation, ProcessOwnershipCommitAck, ProcessOwnershipContext, RetainedProcess,
+    ParticipantState, Path, PathBuf, PidfsProcessIdentityV1, PlatformObservation,
-    SandboxCoordinator, Signal, TargetCommand, UnixListener, check_platform, child_broker,
+    ProcessOwnershipCommitAck, ProcessOwnershipContext, RetainedProcess, SandboxCoordinator,
-    current_process_identity, fs, host_path, networked_sandbox_view_isolated, read_start_time,
+    Signal, TargetCommand, UnixListener, check_platform, child_broker, current_process_identity,
-    sandbox_participant, thread, unreachable_permitted_destination_does_not_block_start,
+    fs, host_path, networked_sandbox_view_isolated, read_start_time, sandbox_participant, thread,
+    unreachable_permitted_destination_does_not_block_start,
 };
 
 pub(super) fn run_handoff_matrix() -> Result<(), MatrixError> {
@@ -175,6 +176,32 @@ pub(super) fn normal_stop_and_stop_resume() -> Result<(), MatrixError> {
     assert_activity_stable(&fixture.activity)
 }
 
+pub(super) fn git_routing_clean_stop() -> Result<(), MatrixError> {
+    let fixture = Fixture::new_git_routed()?;
+    let mut runtime = fixture.launch_committed()?;
+    wait_for_activity(&fixture.activity)?;
+    runtime
+        .stop_sandbox(&SandboxCoordinator::new(&fixture.store))
+        .map_err(failure)?;
+    runtime
+        .stop_git_routing(&GitRoutingCoordinator::new(&fixture.store))
+        .map_err(failure)?;
+    runtime.graceful_exit().map_err(failure)?;
+    let snapshot = fixture.store.snapshot().map_err(failure)?;
+    for role in [ParticipantRole::Sandbox, ParticipantRole::GitRouting] {
+        assert_eq!(
+            snapshot
+                .participants
+                .iter()
+                .find(|participant| participant.role() == role)
+                .ok_or_else(|| failure(format!("{role:?} participant is absent")))?
+                .state(),
+            ParticipantState::Clean
+        );
+    }
+    assert_activity_stable(&fixture.activity)
+}
+
 pub(super) fn attached_runsc_loss_stops_guest() -> Result<(), MatrixError> {
     let fixture = Fixture::new()?;
     let mut runtime = fixture.launch_committed()?;

crates/lazaret-ignored-workspace/src/filesystem.rs

@@ -202,11 +202,12 @@ impl IgnoredWorkspace {
             &mount_config(configuration.threads),
         )?;
         if let Err(error) = ready() {
-            let _ = session.umount_and_join();
+            drop(session);
             return Err(error);
         }
         std::io::copy(&mut std::io::stdin().lock(), &mut std::io::sink())?;
-        session.umount_and_join()
+        drop(session);
+        Ok(())
     }
 
     fn root(&self, backing: Classification) -> &BackingRoot {

crates/lazaret-ignored-workspace/src/filesystem/tests.rs

@@ -7,10 +7,17 @@ use fuser::{Config, LockOwner, MountOption};
 use nix::fcntl::{FcntlArg, fcntl};
 
 use super::nodes::{OpenFile, fuse_lock, record_lock};
-use super::{Classification, IgnoredWorkspace};
+use super::{Classification, IgnoredWorkspace, mount_config};
 
 static SEQUENCE: AtomicU64 = AtomicU64::new(0);
 
+#[test]
+fn multithreaded_router_uses_independent_fuse_channels() {
+    let config = mount_config(8);
+    assert_eq!(config.n_threads, Some(8));
+    assert!(config.clone_fd);
+}
+
 #[test]
 fn posix_lock_owners_use_distinct_backing_lock_domains() {
     let fixture = Fixture::new();