Recognize terminated runsc gofers during teardown

crates/lazaret-runtime-supervisor/src/sandbox.rs

@@ -41,13 +41,13 @@ use validation::{
 };
 
 pub const NETWORK_NAMESPACE_PATH: &str = "/proc/self/ns/net";
-pub const RUNSC_BUILD_ID: &str = "34ed2f67fc55ef97a0b1755b4a94ed682f6aae57a41ce27eceb66ea366185ca7";
+pub const RUNSC_BUILD_ID: &str = "9e711e005f13402e728a0c3ada3fdc67a38918da31f720e5be1006edc4272a51";
 pub const RUNSC_BUILD_ID_BYTES: [u8; 32] = [
-    52, 237, 47, 103, 252, 85, 239, 151, 160, 177, 117, 91, 74, 148, 237, 104, 47, 106, 174, 87,
-    164, 28, 226, 126, 206, 182, 110, 163, 102, 24, 92, 167,
+    158, 113, 30, 0, 95, 19, 64, 46, 114, 138, 12, 58, 218, 63, 220, 103, 163, 137, 24, 218, 49,
+    247, 32, 229, 190, 16, 6, 237, 196, 39, 42, 81,
 ];
 pub const RUNSC_EXECUTABLE_NAME: &str =
-    "lazaret-runsc-34ed2f67fc55ef97a0b1755b4a94ed682f6aae57a41ce27eceb66ea366185ca7";
+    "lazaret-runsc-9e711e005f13402e728a0c3ada3fdc67a38918da31f720e5be1006edc4272a51";
 pub const DOMAIN_OPEN_FILE_LIMIT: u64 = 524_288;
 
 const OCI_VERSION: &str = "1.2.1";

patches/gvisor-9f8f6c917-gofer-zombie-state.patch

@@ -0,0 +1,82 @@
+diff --git a/runsc/container/container.go b/runsc/container/container.go
+index f0bfbf2..9fe6cf5 100644
+--- a/runsc/container/container.go
++++ b/runsc/container/container.go
+@@ -1321,6 +1321,14 @@ func (c *Container) waitForStopped() error {
+ 	defer cancel()
+ 	b := backoff.WithContext(backoff.NewConstantBackOff(100*time.Millisecond), ctx)
+ 	op := func() error {
++		stopped, err := processIsTerminal(goferPid)
++		if err != nil {
++			return err
++		}
++		if stopped {
++			c.GoferPid.Store(0)
++			return nil
++		}
+ 		if err := unix.Kill(goferPid, 0); err == nil {
+ 			return fmt.Errorf("gofer is still running")
+ 		}
+@@ -1330,6 +1338,25 @@ func (c *Container) waitForStopped() error {
+ 	return backoff.Retry(op, b)
+ }
+ 
++func processIsTerminal(pid int) (bool, error) {
++	status, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", pid))
++	if errors.Is(err, os.ErrNotExist) {
++		return true, nil
++	}
++	if err != nil {
++		return false, fmt.Errorf("reading process state: %v", err)
++	}
++	return processStateIsTerminal(status)
++}
++
++func processStateIsTerminal(status []byte) (bool, error) {
++	nameEnd := strings.LastIndexByte(string(status), ')')
++	if nameEnd < 0 || nameEnd+2 >= len(status) {
++		return false, fmt.Errorf("process state is malformed")
++	}
++	return status[nameEnd+2] == 'Z' || status[nameEnd+2] == 'X', nil
++}
++
+ // shouldCreateDeviceGofer indicates whether a device gofer connection should
+ // be created.
+ func shouldCreateDeviceGofer(spec *specs.Spec, conf *config.Config) bool {
+diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go
+index 8b282c4..299745f 100644
+--- a/runsc/container/container_test.go
++++ b/runsc/container/container_test.go
+@@ -2965,6 +2965,32 @@ func TestGoferExits(t *testing.T) {
+ 	}
+ }
+ 
++func TestProcessStateIsTerminal(t *testing.T) {
++	tests := []struct {
++		name     string
++		status   string
++		terminal bool
++		valid    bool
++	}{
++		{name: "running", status: "123 (gofer) S 1", valid: true},
++		{name: "zombie", status: "123 (gofer) Z 1", terminal: true, valid: true},
++		{name: "dead", status: "123 (gofer) X 1", terminal: true, valid: true},
++		{name: "closing parenthesis in name", status: "123 (gofer)) Z 1", terminal: true, valid: true},
++		{name: "malformed", status: "123 gofer Z 1"},
++	}
++	for _, test := range tests {
++		t.Run(test.name, func(t *testing.T) {
++			terminal, err := processStateIsTerminal([]byte(test.status))
++			if (err == nil) != test.valid {
++				t.Fatalf("processStateIsTerminal() error = %v, want valid %t", err, test.valid)
++			}
++			if terminal != test.terminal {
++				t.Errorf("processStateIsTerminal() = %t, want %t", terminal, test.terminal)
++			}
++		})
++	}
++}
++
+ func TestRootNotMount(t *testing.T) {
+ 	appSym, err := testutil.FindFile("test/cmd/test_app/test_app")
+ 	if err != nil {

tests/tooling/test_verify.py

@@ -407,7 +407,7 @@ class VerificationTests(unittest.TestCase):
         )
 
     def test_runsc_materializer_and_install_identity_are_exact(self) -> None:
-        build_id = "34ed2f67fc55ef97a0b1755b4a94ed682f6aae57a41ce27eceb66ea366185ca7"
+        build_id = "9e711e005f13402e728a0c3ada3fdc67a38918da31f720e5be1006edc4272a51"
         materializer = (ROOT / "tools/materialize-runsc").read_text()
         installer = (ROOT / "tools/install-to-host").read_text()
         runtime = (ROOT / "crates/lazaret-runtime-supervisor/src/sandbox.rs").read_text()
@@ -441,6 +441,9 @@ class VerificationTests(unittest.TestCase):
         self.assertTrue(
             (ROOT / "patches/gvisor-9f8f6c917-hostinet-recv-flags.patch").is_file()
         )
+        self.assertTrue(
+            (ROOT / "patches/gvisor-9f8f6c917-gofer-zombie-state.patch").is_file()
+        )
         self.assertTrue(
             (ROOT / "patches/gvisor-9f8f6c917-MODULE.bazel.lock").is_file()
         )

tools/install-to-host

@@ -59,7 +59,7 @@ if readelf -l "$sandbox_init" | grep -q 'Requesting program interpreter' ||
 fi
 install -m 755 "$sandbox_init" "$install_dir/lazaret-sandbox-init"
 
-runsc_build_id=34ed2f67fc55ef97a0b1755b4a94ed682f6aae57a41ce27eceb66ea366185ca7
+runsc_build_id=9e711e005f13402e728a0c3ada3fdc67a38918da31f720e5be1006edc4272a51
 runsc_name="lazaret-runsc-$runsc_build_id"
 runsc_source="$root/.lazaret/bin/$runsc_name"
 runsc_destination="$install_dir/$runsc_name"

tools/materialize-runsc

@@ -9,18 +9,20 @@ patch_files=(
     "$root/patches/gvisor-9f8f6c917-private-ownership.patch"
     "$root/patches/gvisor-9f8f6c917-per-description-file-handles.patch"
     "$root/patches/gvisor-9f8f6c917-hostinet-recv-flags.patch"
+    "$root/patches/gvisor-9f8f6c917-gofer-zombie-state.patch"
 )
 patch_checksums=(
     541420eece9ec7efbef156bdb726744ead6b4d991d41c5560f4fdfe83fb2357a
     79a8c98502d3bb7c254ffb0885dbd26afffac227d04a0091ff91383df0842561
     2403efb81df3185145d18ebdf8d511252fbc2fb414bae4746d7fe471ec4ea9d9
     d3f49a22fe60955d422f63557bc16907fdf65dff368dff296af5535a67b76e74
+    2b7c7444dc56ff47e63af951c24d1dd30092c278c4cba7a6dcd3c3f1062c3206
 )
-patch_digest=bd37be0b8303f3a1248abc1fc898bb2cc513bad1404743428f8e9be99cbafc84
+patch_digest=669b658a29c006d7fd0a472f959265ee79f1fbc78673d0cdc3b31a7721afab98
 bazel_version=8.3.1
 module_lock="$root/patches/gvisor-9f8f6c917-MODULE.bazel.lock"
 module_checksum=8402c7beb4baf2c666f4b78e400ea3f15514b56117598c2cb5411d6a35208d34
-build_id=34ed2f67fc55ef97a0b1755b4a94ed682f6aae57a41ce27eceb66ea366185ca7
+build_id=9e711e005f13402e728a0c3ada3fdc67a38918da31f720e5be1006edc4272a51
 archive="$root/.lazaret/cache/gvisor-$revision.tar.gz"
 source="$root/.lazaret/vendor/gvisor-$build_id"
 binary="$root/.lazaret/bin/lazaret-runsc-$build_id"
@@ -93,6 +95,8 @@ fi
 if [[ "$mode" == test ]]; then
     make -C "$source" test \
         TARGETS='//pkg/sentry/fsimpl/gofer:gofer_test //runsc/fsgofer:lisafs_test //test/syscalls:socket_ip_tcp_generic_loopback_test_native //test/syscalls:socket_ip_tcp_generic_loopback_test_runsc_systrap_hostnet'
+    make -C "$source" test OPTIONS='--test_filter=TestProcessStateIsTerminal' \
+        TARGETS=//runsc/container:container_test
 fi
 if [[ "$mode" != source ]]; then
     make -C "$source" build TARGETS=//runsc:runsc