Add complete system qualification evidence

2026-07-18 14:09:04 +09:00 · 2026-07-18 14:09:04 +09:00 · 6952767503
commit 6952767503
parent 775dd61963
8 changed files with 1256 additions and 84 deletions
--- a/tests/kind/deployment-job.yaml.in
+++ b/tests/kind/deployment-job.yaml.in
@ -35,6 +35,10 @@ spec:
              curl $retry --fail --silent http://remora-remora-api-metrics:9090/metrics | grep -q remora_background_runs_total
              curl $retry --fail --silent http://remora-remora-web/runtime-config.js | grep -q '"oidcClientId":"remora-web"'
              test "$(curl $retry --silent --output /dev/null --write-out '%{http_code}' http://remora-remora/v1/jobs)" = 401
+              if test -n "@@INGRESS_URL@@"; then
+                curl $retry --fail --silent --header 'Host: qualification-ingress' @@INGRESS_URL@@/ | grep -q '<title>Remora</title>'
+                test "$(curl $retry --silent --header 'Host: qualification-ingress' --output /dev/null --write-out '%{http_code}' @@INGRESS_URL@@/v1/jobs)" = 401
+              fi
              if curl --connect-timeout 2 --max-time 5 --insecure --silent --fail https://remora-remora-worker:8443/internal/v1/jobs/x/claim >/dev/null 2>&1; then
                exit 1
              fi
@ -50,6 +54,7 @@ spec:
                name: @@PROFILE_NAME@@
          env:
            - {name: REMORA_DEPLOYMENT_API_URL, value: "http://remora-remora/v1"}
+            - {name: REMORA_EXPECTED_JOB_ID, value: "@@EXPECTED_JOB_ID@@"}
          volumeMounts:
            - {name: profile, mountPath: /run/remora-fixture, readOnly: true}
          resources:
--- a/tests/kind/lib.sh
+++ b/tests/kind/lib.sh
@ -860,6 +860,27 @@ cleanup_lane_target_store() {
  fi
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >&2 || return
+  if [[ "$provider" = versitygw ]]; then
+    kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
+      /bin/sh -ec '
+        endpoint="$1"
+        bucket="$2"
+        while :; do
+          set -- $(aws --endpoint-url "$endpoint" s3api list-object-versions \
+            --bucket "$bucket" \
+            --query "[length(not_null(Versions, \`[]\`)), length(not_null(DeleteMarkers, \`[]\`))]" \
+            --output text)
+          [ "$1" -eq 0 ] && [ "$2" -eq 0 ] && break
+          objects="$(aws --endpoint-url "$endpoint" s3api list-object-versions \
+            --bucket "$bucket" \
+            --query "{Objects: [not_null(Versions, \`[]\`), not_null(DeleteMarkers, \`[]\`)][][].{Key:Key,VersionId:VersionId}, Quiet: \`true\`}" \
+            --output json)"
+          aws --endpoint-url "$endpoint" s3api delete-objects \
+            --bucket "$bucket" \
+            --delete "$objects" >/dev/null
+        done
+      ' sh "$endpoint" "$bucket" >&2 || return
+  fi
  kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
    aws --endpoint-url "$endpoint" s3api delete-bucket --bucket "$bucket" >&2 || return
  if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
@ -1108,6 +1129,9 @@ run_job_matrix() {
  local manifest
  select_fixture_families "${families[@]}" || return
  while IFS=$'\t' read -r lane coordination oidc target; do
+    if [[ -n "${REMORA_FIXTURE_LANE:-}" ]] && [[ "$lane" != "$REMORA_FIXTURE_LANE" ]]; then
+      continue
+    fi
    job="$job_base-$lane"
    profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
    manifest="$RUNTIME/$job.yaml"
@ -1163,7 +1187,8 @@ kube() {
 wait_for_job() {
  local namespace="$1"
  local job="$2"
-  local deadline=$((SECONDS + 300))
+  local timeout="${REMORA_JOB_TIMEOUT_SECONDS:-300}"
+  local deadline=$((SECONDS + timeout))
  local complete
  local failed
  while ((SECONDS < deadline)); do
@ -1180,7 +1205,7 @@ wait_for_job() {
    sleep 1
  done
  print_job_logs "$namespace" "$job" >&2 || true
-  printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
+  printf 'job/%s did not complete within %s seconds\n' "$job" "$timeout" >&2
  return 1
 }

--- a/tests/kind/qualification-ingress.yaml.in
+++ b/tests/kind/qualification-ingress.yaml.in
@ -0,0 +1,169 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+rules:
+  - apiGroups: [""]
+    resources: [configmaps, pods, secrets, endpoints]
+    verbs: [get, list, watch]
+  - apiGroups: [""]
+    resources: [services]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses, ingressclasses]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses/status]
+    verbs: [update]
+  - apiGroups: [coordination.k8s.io]
+    resources: [leases]
+    verbs: [get, create, update]
+  - apiGroups: [""]
+    resources: [events]
+    verbs: [create, patch]
+  - apiGroups: [discovery.k8s.io]
+    resources: [endpointslices]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: remora-qualification-ingress
+rules:
+  - apiGroups: [""]
+    resources: [configmaps, endpoints, nodes, pods, secrets, namespaces]
+    verbs: [list, watch]
+  - apiGroups: [""]
+    resources: [nodes]
+    verbs: [get]
+  - apiGroups: [""]
+    resources: [services]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses, ingressclasses]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses/status]
+    verbs: [update]
+  - apiGroups: [coordination.k8s.io]
+    resources: [leases]
+    verbs: [list, watch]
+  - apiGroups: [""]
+    resources: [events]
+    verbs: [create, patch]
+  - apiGroups: [discovery.k8s.io]
+    resources: [endpointslices]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: qualification-ingress
+subjects:
+  - kind: ServiceAccount
+    name: qualification-ingress
+    namespace: remora-testbed
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: remora-qualification-ingress
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: remora-qualification-ingress
+subjects:
+  - kind: ServiceAccount
+    name: qualification-ingress
+    namespace: remora-testbed
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: qualification-nginx
+spec:
+  controller: remora.dev/qualification-ingress
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+spec:
+  selector:
+    app.kubernetes.io/name: qualification-ingress
+  ports:
+    - {name: http, port: 80, targetPort: http}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: qualification-ingress
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: qualification-ingress
+    spec:
+      serviceAccountName: qualification-ingress
+      containers:
+        - name: controller
+          image: @@INGRESS_IMAGE@@
+          imagePullPolicy: Never
+          args:
+            - /nginx-ingress-controller
+            - --election-id=qualification-ingress
+            - --controller-class=remora.dev/qualification-ingress
+            - --ingress-class=qualification-nginx
+            - --configmap=$(POD_NAMESPACE)/qualification-ingress
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - {name: LD_PRELOAD, value: /usr/local/lib/libmimalloc.so}
+          ports:
+            - {name: http, containerPort: 80}
+          readinessProbe:
+            httpGet: {path: /healthz, port: 10254}
+            initialDelaySeconds: 5
+            periodSeconds: 2
+          resources:
+            requests: {cpu: 100m, memory: 90Mi}
+            limits: {cpu: "1", memory: 256Mi}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add: [NET_BIND_SERVICE]
+              drop: [ALL]
+            runAsGroup: 82
+            runAsNonRoot: true
+            runAsUser: 101
+            seccompProfile: {type: RuntimeDefault}
--- a/tests/kind/qualification-job.yaml.in
+++ b/tests/kind/qualification-job.yaml.in
@ -0,0 +1,38 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 3600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: remora-qualification
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef:
+                name: @@PROFILE_NAME@@
+          volumeMounts:
+            - {name: profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: "1", memory: 1Gi}
+            limits: {cpu: "4", memory: 12Gi}
+      volumes:
+        - name: profile
+          configMap:
+            name: @@PROFILE_NAME@@-file
--- a/tests/kind/run-deployment.sh
+++ b/tests/kind/run-deployment.sh
@ -17,7 +17,8 @@ cleanup() {
  rm -f \
    "$RUNTIME/deployment-values.yaml" \
    "$RUNTIME/deployment-rendered.yaml" \
-    "$RUNTIME/deployment-policy-rendered.yaml"
+    "$RUNTIME/deployment-policy-rendered.yaml" \
+    "$RUNTIME/qualification-ingress.yaml"
  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
    printf 'retained context: %s\n' "$CONTEXT"
@ -159,6 +160,7 @@ write_helm_values() {
  local target_prefix
  local target_region
  local vault_path
+  local public_origin=http://remora.test
  coordination_namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)"
  oidc_issuer="$(profile_value "$PROFILE" REMORA_OIDC_ISSUER)"
  oidc_client_id="$(profile_value "$PROFILE" REMORA_OIDC_CLIENT_ID)"
@ -168,6 +170,9 @@ write_helm_values() {
  target_prefix="$(profile_value "$PROFILE" REMORA_TARGET_PREFIX)"
  target_region="$(profile_value "$PROFILE" REMORA_TARGET_REGION)"
  vault_path="$(profile_value "$PROFILE" REMORA_VAULT_PATH)"
+  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+    public_origin=http://127.0.0.1:4174
+  fi
  cat > "$RUNTIME/deployment-values.yaml" <<EOF
 api:
  image: {repository: remora-api, tag: wp25, digest: "", pullPolicy: Never}
@ -175,7 +180,7 @@ worker:
  image: {repository: remora-worker, tag: wp25, digest: "", pullPolicy: Never}
 web:
  image: {repository: remora-web, tag: wp25, digest: "", pullPolicy: Never}
-  publicOrigin: http://remora.test
+  publicOrigin: $public_origin
  apiUrl: /v1/
  oidcClientId: remora-web
  cspConnectSrc: "'self' $oidc_issuer"
@ -204,7 +209,7 @@ vault:
  path: "$vault_path"
  kubernetesAuthMount: auth/remora-kubernetes
 runtime:
-  eventAllowedOrigins: http://remora.test
+  eventAllowedOrigins: $public_origin
  recoveryIntervalSeconds: 1
 networkPolicy:
  enabled: true
@ -226,6 +231,14 @@ observability:
  serviceMonitor: {enabled: false, interval: 30s}
  prometheusRule: {enabled: false}
 EOF
+  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+    cat >> "$RUNTIME/deployment-values.yaml" <<'EOF'
+ingress:
+  enabled: true
+  className: qualification-nginx
+  host: qualification-ingress
+EOF
+  fi
 }

 wait_for_queue_length() {
@ -243,6 +256,117 @@ wait_for_queue_length() {
  return 1
 }

+wait_for_available_replicas() {
+  local deployment="$1"
+  local expected="$2"
+  local deadline=$((SECONDS + 120))
+  local available
+  while ((SECONDS < deadline)); do
+    available="$(kube --namespace "$FIXTURE_NAMESPACE" get "deployment/$deployment" \
+      --output 'jsonpath={.status.availableReplicas}')"
+    if [[ "${available:-0}" = "$expected" ]]; then
+      return
+    fi
+    sleep 1
+  done
+  printf '%s did not reach %s available replicas\n' "$deployment" "$expected" >&2
+  return 1
+}
+
+run_deployment_test() {
+  local job="$1"
+  local expected_job_id="${2:-}"
+  local manifest="$RUNTIME/$job.yaml"
+  render_lane_job \
+    "$TESTBED/deployment-job.yaml.in" \
+    "$manifest" \
+    "$job" \
+    "$PROFILE" \
+    remora-kind-deployment:local
+  sed -i "s|@@EXPECTED_JOB_ID@@|$expected_job_id|g" "$manifest"
+  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+    sed -i 's|@@INGRESS_URL@@|http://qualification-ingress|g' "$manifest"
+  else
+    sed -i 's|@@INGRESS_URL@@||g' "$manifest"
+  fi
+  run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
+}
+
+run_system_browser_test() {
+  local job=system-browser
+  local manifest="$RUNTIME/$job.yaml"
+  render_lane_job \
+    "$TESTBED/system-browser-job.yaml.in" \
+    "$manifest" \
+    "$job" \
+    "$PROFILE" \
+    remora-kind-web-system:local
+  run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
+}
+
+install_qualification_ingress() {
+  local image
+  image="$(load_external_image ingress-nginx "$INGRESS_NGINX_IMAGE")"
+  sed "s|@@INGRESS_IMAGE@@|$image|g" \
+    "$TESTBED/qualification-ingress.yaml.in" \
+    > "$RUNTIME/qualification-ingress.yaml"
+  kube apply --filename "$RUNTIME/qualification-ingress.yaml"
+  wait_for_deployment qualification-ingress
+}
+
+operational_restore() {
+  local database="remora_lane_$LANE"
+  local backup="/tmp/remora-qualification.dump"
+  local expected_job_id
+  expected_job_id="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres psql \
+      --username remora \
+      --dbname "$database" \
+      --tuples-only \
+      --no-align \
+      --command "SELECT job_id FROM jobs WHERE label = 'deployment-coordination-rebuild'")"
+  [[ -n "$expected_job_id" ]]
+
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=0
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=300s
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres pg_dump \
+      --username remora \
+      --dbname "$database" \
+      --format custom \
+      --file "$backup"
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres psql \
+      --username remora \
+      --dbname postgres \
+      --set ON_ERROR_STOP=1 \
+      --command "DROP DATABASE $database" \
+      --command "CREATE DATABASE $database"
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres pg_restore \
+      --username remora \
+      --dbname "$database" \
+      --exit-on-error \
+      "$backup"
+
+  kube --namespace "$FIXTURE_NAMESPACE" delete pod \
+    --selector app=dragonfly \
+    --wait=true
+  wait_for_deployment dragonfly
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=2
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=600s
+  wait_for_queue_length 1
+  run_deployment_test deployment-restore "$expected_job_id"
+
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=0
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/versitygw --timeout=300s
+  wait_for_available_replicas remora-remora-api 0
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=1
+  wait_for_deployment versitygw
+  reset_lane_target_store versitygw "remora-lane-$LANE"
+  wait_for_available_replicas remora-remora-api 2
+}
+
 rollout_remora() {
  local component
  for component in api worker web; do
@ -306,13 +430,26 @@ validate_deployment_assets
  --target kind-deployment-runner \
  --tag remora-kind-deployment:local \
  "$ROOT"
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  "$REMORA_REAL_NERDCTL" --namespace default build \
+    --file "$ROOT/Dockerfile.test" \
+    --target kind-web-system-runner \
+    --tag remora-kind-web-system:local \
+    "$ROOT"
+fi

 OWNS_FIXTURE=true
 setup_fixture durable coordination oidc target secret
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  install_qualification_ingress
+fi
 load_local_image remora-api:wp25
 load_local_image remora-worker:wp25
 load_local_image remora-web:wp25
 load_local_image remora-kind-deployment:local
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  load_local_image remora-kind-web-system:local
+fi

 PROFILE="$(write_lane_profile "$LANE" dragonfly zitadel versitygw)"
 configure_vault_kubernetes_auth
@ -329,22 +466,19 @@ rollout_remora

 kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=0
 kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-worker --timeout=300s
-JOB=deployment-smoke
-MANIFEST="$RUNTIME/$JOB.yaml"
-render_lane_job \
-  "$TESTBED/deployment-job.yaml.in" \
-  "$MANIFEST" \
-  "$JOB" \
-  "$PROFILE" \
-  remora-kind-deployment:local
-run_job_manifest "$FIXTURE_NAMESPACE" "$JOB" "$MANIFEST"
+run_deployment_test deployment-smoke
 wait_for_queue_length 1

-kube --namespace "$FIXTURE_NAMESPACE" delete pod \
-  --selector app=dragonfly \
-  --wait=true
-wait_for_deployment dragonfly
-wait_for_queue_length 1
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  operational_restore
+  run_system_browser_test
+else
+  kube --namespace "$FIXTURE_NAMESPACE" delete pod \
+    --selector app=dragonfly \
+    --wait=true
+  wait_for_deployment dragonfly
+  wait_for_queue_length 1
+fi

 kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=2
 rollout_remora
--- a/tests/kind/run-qualification.sh
+++ b/tests/kind/run-qualification.sh
@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+export REMORA_JOB_TIMEOUT_SECONDS=900
+if [[ -n "${REMORA_FIXTURE_LANE:-}" ]] && [[ "${REMORA_QUALIFICATION_DIAGNOSTIC:-0}" != 1 ]]; then
+  printf 'REMORA_FIXTURE_LANE requires REMORA_QUALIFICATION_DIAGNOSTIC=1\n' >&2
+  exit 1
+fi
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-qualification-runner \
+  --tag remora-kind-qualification:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture all
+load_local_image remora-kind-qualification:local
+run_job_matrix \
+  qualification \
+  "$TESTBED/qualification-job.yaml.in" \
+  remora-kind-qualification:local \
+  all
+SUCCEEDED=true
+printf 'full provider-matrix qualification Jobs passed\n'
--- a/tests/kind/system-browser-job.yaml.in
+++ b/tests/kind/system-browser-job.yaml.in
@ -0,0 +1,36 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+  labels:
+    app.kubernetes.io/name: remora-system-browser-test
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: remora-system-browser-test
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef:
+                name: @@PROFILE_NAME@@
+          env:
+            - {name: REMORA_SYSTEM_BASE_URL, value: "http://127.0.0.1:4174"}
+          resources:
+            requests: {cpu: 500m, memory: 1Gi}
+            limits: {cpu: "4", memory: 6Gi}
--- a/tests/kind/versions.env
+++ b/tests/kind/versions.env
@ -16,3 +16,4 @@ VERSITYGW_IMAGE=versity/versitygw@sha256:4a2c613f6541410f87983c5134cd26a88dfc23f
 COREDNS_IMAGE=coredns/coredns@sha256:1ba6f47265602e2e50a9c4669e3a955e4298a0d30dc82f39293d4bf1a851e0ff
 CURL_IMAGE=curlimages/curl@sha256:1ab04d023ece37e6ec991bf3306ad04e0ef0084e94a5c6b6563cfcb9563169db
 AWS_CLI_IMAGE=amazon/aws-cli@sha256:10c3a6dff0fdcb3fbc86cd744709548ad041ebb6bbe14fbedc1595a5539d8d27
+INGRESS_NGINX_IMAGE=registry.k8s.io/ingress-nginx/controller@sha256:de8fd8f1def14ed36f59c9838bd3c920ecb01754072e3e9f10832d62ef876cb8