Replace compose runtime with scoped kind deployment

.env.example

@@ -1,25 +0,0 @@
-NUTHATCH_LISTEN=127.0.0.1:8080
-NUTHATCH_PUBLIC_BASE_URL=http://127.0.0.1:8080
-NUTHATCH_JWT_SECRET=dev-only-change-me-to-a-long-random-string
-NUTHATCH_SESSION_HOURS=12
-NUTHATCH_COOKIE_NAME=nuthatch_session
-NUTHATCH_CSRF_COOKIE_NAME=nuthatch_csrf
-
-NUTHATCH_PG_CONN=host=127.0.0.1 port=5432 user=nuthatch password=nuthatch dbname=nuthatch
-NUTHATCH_REDIS_URL=redis://127.0.0.1:6379/0
-
-NUTHATCH_S3_ENDPOINT=http://127.0.0.1:9000
-NUTHATCH_S3_REGION=us-east-1
-NUTHATCH_S3_BUCKET=nuthatch
-NUTHATCH_S3_ACCESS_KEY=minioadmin
-NUTHATCH_S3_SECRET_KEY=minioadmin
-NUTHATCH_S3_PATH_STYLE=true
-
-NUTHATCH_PERSONAL_QUOTA_BYTES=1073741824
-NUTHATCH_SHARED_QUOTA_BYTES=10737418240
-NUTHATCH_MAX_OBJECT_BYTES=104857600
-NUTHATCH_THROTTLE_BYTES_PER_SEC=10485760
-NUTHATCH_THROTTLE_BURST_BYTES=20971520
-NUTHATCH_LIST_CACHE_TTL_SEC=30
-NUTHATCH_SEARCH_MAX_KEYS=500
-NUTHATCH_AUTH_MODE=dev

Dockerfile

@@ -31,23 +31,13 @@ RUN apt-get update \
        libpq5 \
        ca-certificates \
        curl \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  && groupadd --system --gid 10001 nuthatch \
+  && useradd --system --uid 10001 --gid 10001 --no-create-home nuthatch
 
 WORKDIR /app
 COPY sql ./sql
-
+USER 10001:10001
-ENV NUTHATCH_LISTEN=0.0.0.0:8080 \
-    NUTHATCH_PUBLIC_BASE_URL=http://127.0.0.1:8080 \
-    NUTHATCH_JWT_SECRET=dev-only-change-me-to-a-long-random-string \
-    NUTHATCH_PG_CONN="host=postgres port=5432 user=nuthatch password=nuthatch dbname=nuthatch" \
-    NUTHATCH_REDIS_URL=redis://redis:6379/0 \
-    NUTHATCH_S3_ENDPOINT=http://minio:9000 \
-    NUTHATCH_S3_REGION=us-east-1 \
-    NUTHATCH_S3_BUCKET=nuthatch \
-    NUTHATCH_S3_ACCESS_KEY=minioadmin \
-    NUTHATCH_S3_SECRET_KEY=minioadmin \
-    NUTHATCH_S3_PATH_STYLE=true \
-    NUTHATCH_AUTH_MODE=dev
 
 FROM runtime-base AS test-runtime
 

Makefile

@@ -1,6 +1,4 @@
-.PHONY: test check-dependencies check-fixture-lib check-kind-matrix check-storage-foundation check-frontend build-frontend mvp-up mvp-down mvp-build mvp-logs mvp-ps
+.PHONY: test check-dependencies check-fixture-lib check-kind-matrix check-storage-foundation check-app app-up app-down check-frontend build-frontend
-
-COMPOSE := nerdctl compose -f deploy/docker-compose.yml
 
 test:
 	cabal test --offline nuthatch-unit
@@ -17,24 +15,17 @@ check-kind-matrix:
 check-storage-foundation:
 	./tests/kind/wp04-run.sh
 
+check-app:
+	./tests/kind/wp05-run.sh
+
+app-up:
+	NUTHATCH_APP_INTERACTIVE=1 ./tests/kind/wp05-run.sh
+
+app-down:
+	./tests/kind/destroy.sh
+
 check-frontend:
 	npm --prefix web run check
 
 build-frontend:
 	npm --prefix web run build
-
-mvp-build:
-	$(COMPOSE) build nuthatch
-
-mvp-up:
-	$(COMPOSE) up --build -d
-	@echo "Nuthatch MVP: http://127.0.0.1:8080  (dev login with any single-token username)"
-
-mvp-down:
-	$(COMPOSE) down -v
-
-mvp-logs:
-	$(COMPOSE) logs -f nuthatch
-
-mvp-ps:
-	$(COMPOSE) ps

deploy/docker-compose.yml

@@ -1,48 +0,0 @@
-services:
-  postgres:
-    image: postgres:16-alpine
-    environment:
-      POSTGRES_USER: nuthatch
-      POSTGRES_PASSWORD: nuthatch
-      POSTGRES_DB: nuthatch
-
-  redis:
-    image: redis:7-alpine
-
-  minio:
-    image: minio/minio:RELEASE.2025-04-22T22-12-26Z
-    command: server /data --console-address ":9001"
-    environment:
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
-
-  minio-init:
-    image: minio/mc:RELEASE.2025-04-16T18-13-26Z
-    entrypoint:
-      - /bin/sh
-      - -c
-      - |
-        until mc alias set local http://minio:9000 minioadmin minioadmin; do sleep 1; done
-        mc mb --ignore-existing local/nuthatch
-        exit 0
-
-  nuthatch:
-    build:
-      context: ..
-      dockerfile: Dockerfile
-    environment:
-      NUTHATCH_LISTEN: 0.0.0.0:8080
-      NUTHATCH_PUBLIC_BASE_URL: http://127.0.0.1:8080
-      NUTHATCH_JWT_SECRET: dev-only-change-me-to-a-long-random-string
-      NUTHATCH_PG_CONN: host=postgres port=5432 user=nuthatch password=nuthatch dbname=nuthatch
-      NUTHATCH_REDIS_URL: redis://redis:6379/0
-      NUTHATCH_S3_ENDPOINT: http://minio:9000
-      NUTHATCH_S3_REGION: us-east-1
-      NUTHATCH_S3_BUCKET: nuthatch
-      NUTHATCH_S3_ACCESS_KEY: minioadmin
-      NUTHATCH_S3_SECRET_KEY: minioadmin
-      NUTHATCH_S3_PATH_STYLE: "true"
-      NUTHATCH_AUTH_MODE: dev
-    ports:
-      - "8080:8080"
-    restart: on-failure

deploy/kubernetes/nuthatch.yaml.in

@@ -0,0 +1,81 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nuthatch-@@LANE@@
+  namespace: nuthatch-testbed
+data:
+  NUTHATCH_LISTEN: 0.0.0.0:8080
+  NUTHATCH_PUBLIC_BASE_URL: http://nuthatch-@@LANE@@:8080
+  NUTHATCH_REDIS_URL: "@@REDIS_URL@@"
+  NUTHATCH_S3_ENDPOINT: "@@S3_ENDPOINT@@"
+  NUTHATCH_S3_REGION: "@@S3_REGION@@"
+  NUTHATCH_S3_BUCKET: "@@S3_BUCKET@@"
+  NUTHATCH_S3_PATH_STYLE: "true"
+  NUTHATCH_AUTH_MODE: dev
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: nuthatch-@@LANE@@
+  namespace: nuthatch-testbed
+type: Opaque
+stringData:
+  NUTHATCH_JWT_SECRET: synthetic-kind-jwt-secret-@@LANE@@
+  NUTHATCH_PG_CONN: "@@PG_CONN@@"
+  NUTHATCH_S3_ACCESS_KEY: "@@S3_ACCESS_KEY@@"
+  NUTHATCH_S3_SECRET_KEY: "@@S3_SECRET_KEY@@"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nuthatch-@@LANE@@
+  namespace: nuthatch-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {app: nuthatch-@@LANE@@}
+  template:
+    metadata:
+      labels: {app: nuthatch-@@LANE@@}
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: nuthatch
+          image: @@APP_IMAGE@@
+          imagePullPolicy: Never
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: {drop: ["ALL"]}
+          envFrom:
+            - configMapRef: {name: nuthatch-@@LANE@@}
+            - secretRef: {name: nuthatch-@@LANE@@}
+          ports:
+            - {name: http, containerPort: 8080}
+          livenessProbe:
+            httpGet: {path: /healthz, port: http}
+            periodSeconds: 2
+            failureThreshold: 15
+          readinessProbe:
+            httpGet: {path: /readyz, port: http}
+            periodSeconds: 2
+            failureThreshold: 30
+          resources:
+            requests: {cpu: 100m, memory: 96Mi}
+            limits: {cpu: "2", memory: 512Mi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nuthatch-@@LANE@@
+  namespace: nuthatch-testbed
+spec:
+  selector: {app: nuthatch-@@LANE@@}
+  ports:
+    - {name: http, port: 8080, targetPort: http}

docs/spec/testing.md

@@ -25,8 +25,11 @@ digest-pinned.
 dependency image through its product interface, and destroys the fixture.
 `make check-kind-matrix` runs a fixture-consumer Job sequentially across the
 Redis/DragonflyDB × Keycloak/ZITADEL × Garage/VersityGW matrix without
-deploying Nuthatch. `tests/kind/setup.sh` creates the same ready fixture for
+deploying Nuthatch. `make check-app` deploys Nuthatch and runs its smoke Job
-interactive test work; `tests/kind/destroy.sh` removes it.
+across the Redis/DragonflyDB × Garage/VersityGW matrix. `make app-up` retains
+the first app lane for interactive work, and `make app-down` removes it.
+`tests/kind/setup.sh` creates the dependency fixture without Nuthatch for
+interactive test work; `tests/kind/destroy.sh` removes either form.
 `NUTHATCH_TESTBED_RETAIN=1` retains a failed all-in-one or matrix run for
 inspection.
 

src/Nuthatch/Config.hs

@@ -50,21 +50,20 @@ loadConfig = do
   let (host, portText) = splitHostPort listen
   port <- requireRead "NUTHATCH_LISTEN port" portText
   publicBase <- T.pack . fromMaybe ("http://" <> listen) <$> lookupEnv "NUTHATCH_PUBLIC_BASE_URL"
-  jwtSecret <- T.pack . fromMaybe "dev-only-change-me-to-a-long-random-string" <$> lookupEnv "NUTHATCH_JWT_SECRET"
+  jwtSecret <- T.pack <$> requireEnv "NUTHATCH_JWT_SECRET"
   when (T.length jwtSecret < 16) $
     fail "NUTHATCH_JWT_SECRET must be at least 16 characters"
   sessionHours <- envInteger "NUTHATCH_SESSION_HOURS" 12
   cookieName <- T.pack . fromMaybe "nuthatch_session" <$> lookupEnv "NUTHATCH_COOKIE_NAME"
   csrfCookie <- T.pack . fromMaybe "nuthatch_csrf" <$> lookupEnv "NUTHATCH_CSRF_COOKIE_NAME"
-  pgConn <- fromMaybe "host=127.0.0.1 port=5432 user=nuthatch password=nuthatch dbname=nuthatch"
+  pgConn <- requireEnv "NUTHATCH_PG_CONN"
-    <$> lookupEnv "NUTHATCH_PG_CONN"
+  redisUrl <- requireEnv "NUTHATCH_REDIS_URL"
-  redisUrl <- fromMaybe "redis://127.0.0.1:6379/0" <$> lookupEnv "NUTHATCH_REDIS_URL"
   (redisHost, redisPort, redisDb) <- parseRedisUrl redisUrl
-  s3Endpoint <- T.pack . fromMaybe "http://127.0.0.1:9000" <$> lookupEnv "NUTHATCH_S3_ENDPOINT"
+  s3Endpoint <- T.pack <$> requireEnv "NUTHATCH_S3_ENDPOINT"
   s3Region <- T.pack . fromMaybe "us-east-1" <$> lookupEnv "NUTHATCH_S3_REGION"
-  s3Bucket <- T.pack . fromMaybe "nuthatch" <$> lookupEnv "NUTHATCH_S3_BUCKET"
+  s3Bucket <- T.pack <$> requireEnv "NUTHATCH_S3_BUCKET"
-  s3Access <- T.pack . fromMaybe "minioadmin" <$> lookupEnv "NUTHATCH_S3_ACCESS_KEY"
+  s3Access <- T.pack <$> requireEnv "NUTHATCH_S3_ACCESS_KEY"
-  s3Secret <- T.pack . fromMaybe "minioadmin" <$> lookupEnv "NUTHATCH_S3_SECRET_KEY"
+  s3Secret <- T.pack <$> requireEnv "NUTHATCH_S3_SECRET_KEY"
   pathStyle <- envBool "NUTHATCH_S3_PATH_STYLE" True
   personalQuota <- envInteger "NUTHATCH_PERSONAL_QUOTA_BYTES" (1024 * 1024 * 1024)
   sharedQuota <- envInteger "NUTHATCH_SHARED_QUOTA_BYTES" (10 * 1024 * 1024 * 1024)
@@ -113,6 +112,13 @@ splitHostPort s =
     (h, ':' : p) -> (h, p)
     _            -> ("127.0.0.1", "8080")
 
+requireEnv :: String -> IO String
+requireEnv name = do
+  value <- lookupEnv name
+  case value of
+    Just present | not (null present) -> pure present
+    _ -> fail $ name <> " is required"
+
 envInteger :: String -> Integer -> IO Integer
 envInteger name def = do
   m <- lookupEnv name

tests/kind/dependencies.yaml.in

@@ -257,7 +257,7 @@ data:
     db_engine = "sqlite"
     replication_factor = 1
     rpc_bind_addr = "[::]:3901"
-    rpc_public_addr = "garage:3901"
+    rpc_public_addr = "127.0.0.1:3901"
     rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000"
     [s3_api]
     s3_region = "garage"
@@ -353,10 +353,12 @@ spec:
         - name: versitygw
           image: @@VERSITYGW_IMAGE@@
           imagePullPolicy: Never
-          args: [--port, ":10000", --iam-dir, /data, posix, --versioning-dir, /versions, /data]
+          args: [--port, ":10000", --admin-port, "127.0.0.1:10001", --iam-dir, /data, posix, --versioning-dir, /versions, /data]
           env:
             - {name: ROOT_ACCESS_KEY, value: synthetic-s3-access}
             - {name: ROOT_SECRET_KEY, value: synthetic-s3-secret}
+            - {name: ADMIN_ACCESS_KEY_ID, value: synthetic-s3-access}
+            - {name: ADMIN_SECRET_ACCESS_KEY, value: synthetic-s3-secret}
           ports:
             - {name: s3, containerPort: 10000}
           resources:

tests/kind/lib-test.sh

@@ -53,18 +53,25 @@ if write_lane_profile 001 - - - >/dev/null 2>&1; then
 fi
 
 source "$ROOT/tests/kind/lib.sh"
-kube_calls=0
 kube() {
-  kube_calls=$((kube_calls + 1))
+  return 1
-  case "$kube_calls" in
-    1) return 1 ;;
-    2) return 0 ;;
-    3) return 1 ;;
-    *) return 0 ;;
-  esac
 }
-if reset_lane_target_store garage nuthatch-lane-001 >/dev/null 2>&1; then
+if ensure_target_scope_canary garage >/dev/null 2>&1; then
-  printf 'target-store emptying failure was ignored\n' >&2
+  printf 'scope-canary provisioning failure was ignored\n' >&2
+  exit 1
+fi
+
+source "$ROOT/tests/kind/lib.sh"
+kube() {
+  return 1
+}
+if provision_lane_target_principal \
+  garage \
+  001 \
+  nuthatch-lane-001 \
+  access_key \
+  secret_key >/dev/null 2>&1; then
+  printf 'target-principal provisioning failure was ignored\n' >&2
   exit 1
 fi
 

tests/kind/lib.sh

@@ -461,18 +461,71 @@ target_loopback_endpoint() {
   esac
 }
 
-reset_lane_target_store() {
+ensure_target_scope_canary() {
   local provider="$1"
-  local bucket="$2"
   local endpoint
   endpoint="$(target_loopback_endpoint "$provider")" || return
   if ! kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
-    aws --endpoint-url "$endpoint" s3api create-bucket --bucket "$bucket" >&2; then
+    aws --endpoint-url "$endpoint" s3api create-bucket --bucket nuthatch-fixture >&2; then
     kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
-      aws --endpoint-url "$endpoint" s3api head-bucket --bucket "$bucket" >/dev/null || return
+      aws --endpoint-url "$endpoint" s3api head-bucket --bucket nuthatch-fixture >/dev/null || return
   fi
   kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$provider" --container s3-readiness -- \
-    aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >/dev/null || return
+    sh -ec 'printf scope-canary >/tmp/scope-canary && aws --endpoint-url "$1" s3api put-object --bucket nuthatch-fixture --key scope-canary --body /tmp/scope-canary >/dev/null' \
+    sh "$endpoint"
+}
+
+provision_lane_target_principal() {
+  local provider="$1"
+  local lane="$2"
+  local bucket="$3"
+  local access_key_var="$4"
+  local secret_key_var="$5"
+  local endpoint
+  local access_key
+  local secret_key
+  endpoint="$(target_loopback_endpoint "$provider")" || return
+  case "$provider" in
+    garage)
+      printf -v access_key 'GKWP05%028d' "$((10#$lane))"
+      printf -v secret_key '%064d' "$((10#$lane))"
+      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
+        /garage bucket create "$bucket" >/dev/null || return
+      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
+        /garage bucket allow \
+        --read \
+        --write \
+        --owner \
+        "$bucket" \
+        --key GK00000000000000000000000000000000 >/dev/null || return
+      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
+        /garage key import --yes -n "nuthatch-$lane" "$access_key" "$secret_key" >/dev/null || return
+      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/garage --container garage -- \
+        /garage bucket allow --read --write "$bucket" --key "$access_key" >/dev/null || return
+      ;;
+    versitygw)
+      access_key="nuthatch-wp05-$lane"
+      printf -v secret_key 'synthetic-wp05-%049d' "$((10#$lane))"
+      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
+        /usr/local/bin/versitygw admin \
+        --endpoint-url http://127.0.0.1:10001 \
+        create-user \
+        --access "$access_key" \
+        --secret "$secret_key" \
+        --role user \
+        --user-id "$((10#$lane))" \
+        --group-id "$((10#$lane))" >/dev/null || return
+      kube --namespace "$FIXTURE_NAMESPACE" exec deployment/versitygw --container versitygw -- \
+        /usr/local/bin/versitygw admin \
+        --endpoint-url http://127.0.0.1:10001 \
+        create-bucket \
+        --bucket "$bucket" \
+        --owner "$access_key" >/dev/null || return
+      ;;
+    *) return 1 ;;
+  esac
+  printf -v "$access_key_var" '%s' "$access_key"
+  printf -v "$secret_key_var" '%s' "$secret_key"
 }
 
 write_lane_profile() {
@@ -505,19 +558,17 @@ write_lane_profile() {
   fi
   if fixture_has_family target; then
     target_bucket="nuthatch-lane-$lane"
-    reset_lane_target_store "$target" "$target_bucket" || return
     target_url="$(target_endpoint "$target")" || return
+    ensure_target_scope_canary "$target" || return
+    provision_lane_target_principal \
+      "$target" \
+      "$lane" \
+      "$target_bucket" \
+      target_access_key \
+      target_secret_key || return
     case "$target" in
-      garage)
+      garage) target_region=garage ;;
-        target_region=garage
+      versitygw) target_region=us-east-1 ;;
-        target_access_key=GK00000000000000000000000000000000
-        target_secret_key=0000000000000000000000000000000000000000000000000000000000000000
-        ;;
-      versitygw)
-        target_region=us-east-1
-        target_access_key=synthetic-s3-access
-        target_secret_key=synthetic-s3-secret
-        ;;
       *) return 1 ;;
     esac
   fi
@@ -669,6 +720,8 @@ destroy_fixture() {
     "$KUBECONFIG_PATH" \
     "$RUNTIME/dependencies.yaml" \
     "$RUNTIME/probe.yaml" \
-    "$RUNTIME/fixture.env" || status=1
+    "$RUNTIME/fixture.env" \
+    "$RUNTIME"/nuthatch-fixture-lane-*.env \
+    "$RUNTIME"/wp05-app-*.yaml || status=1
   return "$status"
 }

tests/kind/wp04-job.yaml.in

@@ -21,6 +21,7 @@ spec:
           envFrom:
             - configMapRef: {name: @@PROFILE_NAME@@}
           env:
+            - {name: NUTHATCH_JWT_SECRET, value: synthetic-wp04-jwt-secret}
             - {name: NUTHATCH_REDIS_URL, value: "redis://127.0.0.1:6379/0"}
           resources:
             requests: {cpu: 100m, memory: 32Mi}

tests/kind/wp05-job.yaml.in

@@ -0,0 +1,135 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: nuthatch-testbed
+spec:
+  activeDeadlineSeconds: 300
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      restartPolicy: Never
+      initContainers:
+        - name: credential-negative
+          image: @@AWS_CLI_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              export AWS_ACCESS_KEY_ID="$NUTHATCH_S3_ACCESS_KEY"
+              export AWS_SECRET_ACCESS_KEY="$NUTHATCH_S3_SECRET_KEY"
+              export AWS_DEFAULT_REGION="$NUTHATCH_S3_REGION"
+              printf denied-write >/tmp/denied-write
+
+              expect_denied() {
+                name="$1"
+                shift
+                if "$@" >/tmp/"$name".out 2>/tmp/"$name".err; then
+                  exit 1
+                fi
+                grep -E 'AccessDenied|Forbidden|403' /tmp/"$name".err
+              }
+
+              expect_denied head \
+                aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api head-bucket \
+                --bucket nuthatch-fixture
+              expect_denied list \
+                aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api list-objects-v2 \
+                --bucket nuthatch-fixture
+              expect_denied get \
+                aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api get-object \
+                --bucket nuthatch-fixture --key scope-canary /tmp/denied-read
+              expect_denied put \
+                aws --endpoint-url "$NUTHATCH_S3_ENDPOINT" s3api put-object \
+                --bucket nuthatch-fixture --key denied-write --body /tmp/denied-write
+          envFrom:
+            - configMapRef: {name: @@PROFILE_NAME@@}
+          resources:
+            requests: {cpu: 25m, memory: 64Mi}
+            limits: {cpu: "1", memory: 256Mi}
+      containers:
+        - name: smoke
+          image: @@CURL_IMAGE@@
+          imagePullPolicy: Never
+          command: [/bin/sh, -ec]
+          args:
+            - |
+              base="http://nuthatch-@@LANE@@:8080"
+              curl --fail --silent --show-error "$base/healthz" | grep -F '"ok":true'
+              curl --fail --silent --show-error "$base/readyz" | grep -F '"ready":true'
+              curl --fail --silent --show-error \
+                --cookie-jar /tmp/cookies \
+                --dump-header /tmp/login-headers \
+                --header 'Accept: application/json' \
+                --header 'Content-Type: application/json' \
+                --data '{"username":"wp05-smoke"}' \
+                "$base/login" >/tmp/login.json
+              csrf="$(awk 'BEGIN{IGNORECASE=1} $1=="X-CSRF-Token:"{gsub("\r","",$2); print $2}' /tmp/login-headers)"
+              test -n "$csrf"
+
+              exercise_root() {
+                root="$1"
+                suffix="$2"
+                first="first-$root"
+                second="second-$root"
+                printf '%s' "$first" >/tmp/body
+                size="$(wc -c </tmp/body | tr -d ' ')"
+                digest="$(sha256sum /tmp/body | cut -d' ' -f1)"
+                curl --fail --silent --show-error \
+                  --cookie /tmp/cookies \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $size" \
+                  --header "X-Content-SHA256: $digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/body \
+                  "$base/upload?root=$root&path=smoke-$suffix.txt&operation_id=00000000-0000-4000-8000-0000000000${suffix}" |
+                  grep -F '"result":"succeeded"'
+                curl --fail --silent --show-error \
+                  --cookie /tmp/cookies \
+                  --header 'Accept: application/json' \
+                  "$base/?root=$root" |
+                  grep -F "smoke-$suffix.txt"
+                curl --fail --silent --show-error \
+                  --cookie /tmp/cookies \
+                  "$base/download?root=$root&path=smoke-$suffix.txt" >/tmp/download
+                test "$(cat /tmp/download)" = "$first"
+
+                printf '%s' "$second" >/tmp/body
+                size="$(wc -c </tmp/body | tr -d ' ')"
+                digest="$(sha256sum /tmp/body | cut -d' ' -f1)"
+                curl --fail --silent --show-error \
+                  --cookie /tmp/cookies \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $size" \
+                  --header "X-Content-SHA256: $digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/body \
+                  "$base/upload?root=$root&path=smoke-$suffix.txt&operation_id=10000000-0000-4000-8000-0000000000${suffix}" |
+                  grep -F '"result":"succeeded"'
+                curl --fail --silent --show-error \
+                  --cookie /tmp/cookies \
+                  "$base/download?root=$root&path=smoke-$suffix.txt" >/tmp/download
+                test "$(cat /tmp/download)" = "$second"
+
+                curl --fail --silent --show-error \
+                  --cookie /tmp/cookies \
+                  --header 'Content-Type: application/json' \
+                  --data "{\"root\":\"$root\",\"path\":\"smoke-$suffix.txt\",\"operation_id\":\"20000000-0000-4000-8000-0000000000${suffix}\",\"csrf\":\"$csrf\"}" \
+                  "$base/delete" |
+                  grep -F '"result":"succeeded"'
+                if curl --fail --silent \
+                  --cookie /tmp/cookies \
+                  "$base/download?root=$root&path=smoke-$suffix.txt" >/dev/null; then
+                  exit 1
+                fi
+              }
+
+              exercise_root personal 01
+              exercise_root shared 02
+              printf 'WP-05 app smoke passed\n'
+          resources:
+            requests: {cpu: 50m, memory: 32Mi}
+            limits: {cpu: "1", memory: 128Mi}

tests/kind/wp05-run.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+APP_IMAGE=nuthatch-wp05:local
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+render_application() {
+  local lane="$1"
+  local profile_file="$2"
+  local pg_conn
+  local redis_url
+  local s3_endpoint
+  local s3_region
+  local s3_bucket
+  local s3_access_key
+  local s3_secret_key
+  pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
+  redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
+  s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
+  s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
+  s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
+  s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
+  s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
+  sed \
+    -e "s|@@LANE@@|$lane|g" \
+    -e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
+    -e "s|@@PG_CONN@@|$pg_conn|g" \
+    -e "s|@@REDIS_URL@@|$redis_url|g" \
+    -e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
+    -e "s|@@S3_REGION@@|$s3_region|g" \
+    -e "s|@@S3_BUCKET@@|$s3_bucket|g" \
+    -e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
+    -e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
+    "$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/wp05-app-$lane.yaml"
+}
+
+cleanup_application() {
+  local lane="$1"
+  kube --namespace "$FIXTURE_NAMESPACE" delete \
+    "deployment/nuthatch-$lane" \
+    "service/nuthatch-$lane" \
+    "configmap/nuthatch-$lane" \
+    "secret/nuthatch-$lane" \
+    --ignore-not-found >/dev/null
+  rm -f "$RUNTIME/wp05-app-$lane.yaml"
+}
+
+acquire_fixture_lock
+OWNS_FIXTURE=true
+setup_fixture durable coordination target
+source "$RUNTIME/fixture.env"
+"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
+  --file "$ROOT/Dockerfile" \
+  --tag "$APP_IMAGE" \
+  "$ROOT"
+load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
+
+while IFS=$'\t' read -r lane coordination oidc target; do
+  profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
+  render_application "$lane" "$RUNTIME/$profile.env"
+  kube apply --filename "$RUNTIME/wp05-app-$lane.yaml" >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status \
+    "deployment/nuthatch-$lane" --timeout=180s
+  if [[ "${NUTHATCH_APP_INTERACTIVE:-0}" = 1 ]]; then
+    SUCCEEDED=true
+    OWNS_FIXTURE=false
+    printf 'interactive kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'interactive context: %s\n' "$CONTEXT"
+    printf 'interactive namespace: %s\n' "$FIXTURE_NAMESPACE"
+    printf 'interactive service: nuthatch-%s:8080\n' "$lane"
+    exit 0
+  fi
+  job="wp05-app-$lane"
+  render_lane_job \
+    "$job" \
+    "$TESTBED/wp05-job.yaml.in" \
+    "$CURL_LOCAL_IMAGE" \
+    "$profile"
+  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
+  printf 'Running lane %s: coordination=%s target=%s\n' "$lane" "$coordination" "$target"
+  run_job_manifest "$job" "$RUNTIME/$job.yaml"
+  cleanup_application "$lane"
+  cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
+  cleanup_lane_resources "$job" "$profile"
+done < <(expand_fixture_matrix coordination target)
+
+SUCCEEDED=true
+printf 'Nuthatch WP-05 app gate passed\n'