nuthatch/tests/kind/runtime-resilience-run.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"

USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-runtime-resilience:local

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
    printf 'retained context: %s\n' "$CONTEXT"
    exit "$status"
  fi
  if [[ "$OWNS_FIXTURE" = true ]]; then
    destroy_fixture || status=1
  fi
  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
    printf 'user kubeconfig changed unexpectedly\n' >&2
    status=1
  fi
  exit "$status"
}

trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

set_service_available() {
  local service="$1"
  local available="$2"
  local selector="$service"
  if [[ "$available" = false ]]; then
    selector=runtime-unavailable
  fi
  kube --namespace "$FIXTURE_NAMESPACE" patch service "$service" \
    --type merge \
    --patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
}

render_application() {
  local lane="$1"
  local profile_file="$2"
  local oidc_secret
  oidc_secret="$(<"$profile_file.oidc-secret")"
  sed \
    -e "s|@@LANE@@|$lane|g" \
    -e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
    -e "s|@@PG_CONN@@|$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")|g" \
    -e "s|@@REDIS_URL@@|$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")|g" \
    -e "s|@@OIDC_ISSUER@@|$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")|g" \
    -e "s|@@OIDC_CLIENT_ID@@|$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")|g" \
    -e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$(sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file")|g" \
    -e "s|@@OIDC_CLIENT_SECRET@@|$oidc_secret|g" \
    -e "s|@@S3_ENDPOINT@@|$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")|g" \
    -e "s|@@S3_REGION@@|$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")|g" \
    -e "s|@@S3_BUCKET@@|$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")|g" \
    -e "s|@@S3_ACCESS_KEY@@|$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")|g" \
    -e "s|@@S3_SECRET_KEY@@|$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")|g" \
    "$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/runtime-app-$lane.yaml"
  sed -i \
    -e 's/  replicas: 1/  replicas: 2/' \
    -e 's/NUTHATCH_LIST_CACHE_TTL_SEC: "30"/NUTHATCH_LIST_CACHE_TTL_SEC: "1"/' \
    -e 's/NUTHATCH_STARTUP_ATTEMPTS: "60"/NUTHATCH_STARTUP_ATTEMPTS: "4"/' \
    -e 's/NUTHATCH_STARTUP_DELAY_MS: "1000"/NUTHATCH_STARTUP_DELAY_MS: "250"/' \
    "$RUNTIME/runtime-app-$lane.yaml"
}

wait_for_startup_exhaustion() {
  local lane="$1"
  for _ in $(seq 1 120); do
    if kube --namespace "$FIXTURE_NAMESPACE" logs \
        --selector="app=nuthatch-$lane" --all-containers=true \
        --prefix=true 2>/dev/null |
        grep -F '"action":"dependency_startup"' |
        grep -F '"result":"exhausted"' >/dev/null; then
      return
    fi
    sleep 0.25
  done
  printf 'timed out waiting for bounded startup exhaustion\n' >&2
  return 1
}

run_runtime_probe() {
  local lane="$1"
  local profile="$2"
  local mode="$3"
  local job="runtime-probe-$mode-$lane"
  render_lane_job \
    "$job" \
    "$TESTBED/runtime-probe-job.yaml.in" \
    "$APP_IMAGE" \
    "$profile"
  sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
  run_job_manifest "$job" "$RUNTIME/$job.yaml"
}

set_profile_mode() {
  local profile="$1"
  local name="$2"
  local value="$3"
  local profile_file="$RUNTIME/$profile.env"
  sed -i "/^$name=/d" "$profile_file"
  if [[ -n "$value" ]]; then
    printf '%s=%s\n' "$name" "$value" >>"$profile_file"
  fi
  kube create configmap "$profile" \
    --namespace "$FIXTURE_NAMESPACE" \
    --from-env-file="$profile_file" \
    --dry-run=client \
    --output yaml |
    kube apply --filename - >/dev/null
}

run_product_probe() {
  local lane="$1"
  local profile="$2"
  local suffix="$3"
  local job="runtime-product-$suffix-$lane"
  render_lane_job \
    "$job" \
    "$TESTBED/product-matrix-job.yaml.in" \
    "$CURL_LOCAL_IMAGE" \
    "$profile"
  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
  run_job_manifest "$job" "$RUNTIME/$job.yaml"
}

run_database_failure_probe() {
  local lane="$1"
  local profile="$2"
  local database="$3"
  local job="runtime-product-database-unavailable-$lane"
  render_lane_job \
    "$job" \
    "$TESTBED/product-matrix-job.yaml.in" \
    "$CURL_LOCAL_IMAGE" \
    "$profile"
  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
  kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
  for _ in $(seq 1 240); do
    if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
        grep -F 'Database fault checkpoint' >/dev/null; then
      break
    fi
    sleep 0.25
  done
  kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" |
    grep -F 'Database fault checkpoint' >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
    --type merge \
    --patch '{"spec":{"publishNotReadyAddresses":true}}' >/dev/null
  set_service_available postgres false
  terminate_database_connections "$database"
  local status=0
  wait_for_job "$job" || status=$?
  set_service_available postgres true
  kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
    --type merge \
    --patch '{"spec":{"publishNotReadyAddresses":false}}' >/dev/null
  local restored=false
  for _ in $(seq 1 120); do
    if run_health_probe \
        "$lane" "$profile" 200 database-mutation-restored 2>/dev/null; then
      restored=true
      break
    fi
    sleep 0.25
  done
  test "$restored" = true
  printf 'PostgreSQL mutation recovery passed status=%s\n' "$status"
  return "$status"
}

run_health_probe() {
  local lane="$1"
  local profile="$2"
  local status="$3"
  local suffix="$4"
  local job="runtime-health-$suffix-$lane"
  render_lane_job \
    "$job" \
    "$TESTBED/runtime-health-job.yaml.in" \
    "$CURL_LOCAL_IMAGE" \
    "$profile"
  sed -i \
    -e "s|@@LANE@@|$lane|g" \
    -e "s|@@READY_STATUS@@|$status|g" \
    "$RUNTIME/$job.yaml"
  run_job_manifest "$job" "$RUNTIME/$job.yaml"
}

restart_count() {
  local lane="$1"
  kube --namespace "$FIXTURE_NAMESPACE" get pods \
    --selector="app=nuthatch-$lane" \
    --output json |
    jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0'
}

terminate_database_connections() {
  local database="$1"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username nuthatch \
      --dbname postgres \
      --set ON_ERROR_STOP=1 \
      --command \
        "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
      >/dev/null
}

acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
  --file "$ROOT/Dockerfile" \
  --target test-runtime \
  --tag "$APP_IMAGE" \
  "$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"

lane=01
coordination=redis
oidc=keycloak
target=garage
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
database="nuthatch_lane_$lane"
render_application "$lane" "$RUNTIME/$profile.env"
run_runtime_probe "$lane" "$profile" config

set_service_available postgres false
kube apply --filename "$RUNTIME/runtime-app-$lane.yaml" >/dev/null
wait_for_startup_exhaustion "$lane"
set_service_available postgres true

set_service_available redis false
set_service_available "$target" false
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
  "deployment/nuthatch-$lane" >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
  "deployment/nuthatch-$lane" --timeout=180s
run_health_probe "$lane" "$profile" 200 redis-down
run_runtime_probe "$lane" "$profile" unavailable

set_service_available redis true
set_service_available "$target" true
run_runtime_probe "$lane" "$profile" available
run_health_probe "$lane" "$profile" 200 restored
run_product_probe "$lane" "$profile" baseline
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE coherence
run_product_probe "$lane" "$profile" coherence
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""

set_service_available redis false
kube --namespace "$FIXTURE_NAMESPACE" rollout restart deployment/redis >/dev/null
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
  deployment/redis --timeout=120s
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE unavailable
run_product_probe "$lane" "$profile" redis-unavailable
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE ""
set_service_available redis true

set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE database-unavailable
run_database_failure_probe "$lane" "$profile" "$database"
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
printf 'Database mutation audit lane passed\n'

before_s3="$(restart_count "$lane")"
printf 'S3 runtime fault starting restarts=%s\n' "$before_s3"
set_service_available "$target" false
run_health_probe "$lane" "$profile" 200 s3-down
sleep 6
test "$(restart_count "$lane")" = "$before_s3"
set_service_available "$target" true

before_pg="$(restart_count "$lane")"
set_service_available postgres false
terminate_database_connections "$database"
run_health_probe "$lane" "$profile" 503 postgres-down
sleep 6
test "$(restart_count "$lane")" = "$before_pg"
set_service_available postgres true
restored=false
for _ in $(seq 1 120); do
  if run_health_probe "$lane" "$profile" 200 postgres-restored 2>/dev/null; then
    restored=true
    break
  fi
  sleep 0.25
done
test "$restored" = true

kube --namespace "$FIXTURE_NAMESPACE" logs \
  --selector="app=nuthatch-$lane" \
  --container nuthatch \
  --tail=-1 >"$RUNTIME/runtime-app.log"
grep '^{' "$RUNTIME/runtime-app.log" >"$RUNTIME/runtime-audit.jsonl"
test -s "$RUNTIME/runtime-audit.jsonl"
jq --slurp -e '
  length > 0
  and all(.[]; has("ts") and has("action") and has("result"))
' \
  "$RUNTIME/runtime-audit.jsonl" >/dev/null
for action in authentication login logout authorization put delete; do
  grep -F "\"action\":\"$action\"" "$RUNTIME/runtime-audit.jsonl" >/dev/null
done
jq --slurp -e '
  [.[] | select(.action == "put")] as $records
  | ($records | length) > 0
    and all($records[]; has("user_id") and has("root") and has("bytes"))
' "$RUNTIME/runtime-audit.jsonl" >/dev/null
jq --slurp -e '
  [.[] |
    select(
      (.action == "put" or .action == "delete")
      and (.result != "ok")
    )
  ] as $records
  | ($records | length) > 0
    and all($records[]; has("reason"))
' "$RUNTIME/runtime-audit.jsonl" >/dev/null
if grep -Ei 'Bearer |"authorization"[[:space:]]*:|client_secret|NUTHATCH_' \
    "$RUNTIME/runtime-app.log" >/dev/null; then
  printf 'audit output exposed prohibited data\n' >&2
  exit 1
fi
while IFS= read -r secret_value; do
  if [[ -n "$secret_value" ]] &&
      grep -F -- "$secret_value" "$RUNTIME/runtime-app.log" >/dev/null; then
    printf 'audit output exposed a configured secret\n' >&2
    exit 1
  fi
done < <(
  {
    printf '%s\n' synthetic-runtime-session-jwt-secret
    sed -n 's/^NUTHATCH_PG_CONN=//p' "$RUNTIME/$profile.env"
    sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$RUNTIME/$profile.env"
    sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$RUNTIME/$profile.env"
    sed -n '1p' "$RUNTIME/$profile.env.oidc-secret"
  }
)
for body_value in private-a runtime-coherence first-personal second-personal; do
  if grep -F -- "$body_value" "$RUNTIME/runtime-app.log" >/dev/null; then
    printf 'audit output exposed request body data\n' >&2
    exit 1
  fi
done

test "$(kube --namespace "$FIXTURE_NAMESPACE" get deployment \
  "nuthatch-$lane" --output jsonpath='{.status.readyReplicas}')" = 2

cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
cleanup_lane_resources "runtime-probe-available-$lane" "$profile"
SUCCEEDED=true
printf 'Nuthatch runtime resilience gate passed\n'