Split kind tests into component-owned probes

tests/kind/runtime-resilience-run.sh

@@ -118,17 +118,29 @@ set_profile_mode() {
     kube apply --filename - >/dev/null
 }
 
-run_product_probe() {
+run_runtime_http_probe() {
   local lane="$1"
   local profile="$2"
   local suffix="$3"
-  local job="runtime-product-$suffix-$lane"
+  shift 3
+  local probe_args
+  probe_args="$(
+    printf '"%s",' "$@"
+  )"
+  probe_args="${probe_args%,}"
+  local job="runtime-http-$suffix-$lane"
   render_lane_job \
     "$job" \
-    "$TESTBED/product-matrix-job.yaml.in" \
-    "$CURL_LOCAL_IMAGE" \
+    "$TESTBED/application-probe-job.yaml.in" \
+    "$APP_IMAGE" \
     "$profile"
-  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
+  sed -i \
+    -e "s|@@LANE@@|$lane|g" \
+    -e 's|@@COMPONENT@@|runtime|g' \
+    -e 's|@@PROBE@@|nuthatch-runtime-http-probe|g' \
+    -e "s|@@PROBE_ARGS@@|$probe_args|g" \
+    -e 's|@@DEADLINE_SECONDS@@|300|g' \
+    "$RUNTIME/$job.yaml"
   run_job_manifest "$job" "$RUNTIME/$job.yaml"
 }
 
@@ -136,13 +148,19 @@ run_database_failure_probe() {
   local lane="$1"
   local profile="$2"
   local database="$3"
-  local job="runtime-product-database-unavailable-$lane"
+  local job="runtime-http-database-unavailable-$lane"
   render_lane_job \
     "$job" \
-    "$TESTBED/product-matrix-job.yaml.in" \
-    "$CURL_LOCAL_IMAGE" \
+    "$TESTBED/application-probe-job.yaml.in" \
+    "$APP_IMAGE" \
     "$profile"
-  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
+  sed -i \
+    -e "s|@@LANE@@|$lane|g" \
+    -e 's|@@COMPONENT@@|runtime|g' \
+    -e 's|@@PROBE@@|nuthatch-runtime-http-probe|g' \
+    -e 's|@@PROBE_ARGS@@|"database-unavailable"|g' \
+    -e 's|@@DEADLINE_SECONDS@@|300|g' \
+    "$RUNTIME/$job.yaml"
   kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
   for _ in $(seq 1 240); do
     if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
@@ -186,16 +204,44 @@ run_health_probe() {
   local job="runtime-health-$suffix-$lane"
   render_lane_job \
     "$job" \
-    "$TESTBED/runtime-health-job.yaml.in" \
-    "$CURL_LOCAL_IMAGE" \
+    "$TESTBED/application-probe-job.yaml.in" \
+    "$APP_IMAGE" \
     "$profile"
   sed -i \
     -e "s|@@LANE@@|$lane|g" \
-    -e "s|@@READY_STATUS@@|$status|g" \
+    -e 's|@@COMPONENT@@|runtime|g' \
+    -e 's|@@PROBE@@|nuthatch-runtime-http-probe|g' \
+    -e "s|@@PROBE_ARGS@@|\"health\", \"$status\"|g" \
+    -e 's|@@DEADLINE_SECONDS@@|120|g' \
     "$RUNTIME/$job.yaml"
   run_job_manifest "$job" "$RUNTIME/$job.yaml"
 }
 
+run_runtime_audit_probe() {
+  local lane="$1"
+  local profile="$2"
+  local log_config_map="runtime-audit-log-$lane"
+  local job="runtime-audit-$lane"
+  kube --namespace "$FIXTURE_NAMESPACE" create configmap "$log_config_map" \
+    --from-file="app.log=$RUNTIME/runtime-app.log" \
+    --dry-run=client \
+    --output yaml |
+    kube apply --filename - >/dev/null
+  render_lane_job \
+    "$job" \
+    "$TESTBED/runtime-audit-job.yaml.in" \
+    "$APP_IMAGE" \
+    "$profile"
+  sed -i \
+    -e "s|@@LANE@@|$lane|g" \
+    -e "s|@@LOG_CONFIG_MAP@@|$log_config_map|g" \
+    "$RUNTIME/$job.yaml"
+  run_job_manifest "$job" "$RUNTIME/$job.yaml"
+  kube --namespace "$FIXTURE_NAMESPACE" delete \
+    "configmap/$log_config_map" \
+    --ignore-not-found >/dev/null
+}
+
 restart_count() {
   local lane="$1"
   kube --namespace "$FIXTURE_NAMESPACE" get pods \
@@ -254,9 +300,9 @@ set_service_available redis true
 set_service_available "$target" true
 run_runtime_probe "$lane" "$profile" available
 run_health_probe "$lane" "$profile" 200 restored
-run_product_probe "$lane" "$profile" baseline
+run_runtime_http_probe "$lane" "$profile" baseline baseline
 set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE coherence
-run_product_probe "$lane" "$profile" coherence
+run_runtime_http_probe "$lane" "$profile" coherence coherence
 set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
 
 set_service_available redis false
@@ -264,7 +310,7 @@ kube --namespace "$FIXTURE_NAMESPACE" rollout restart deployment/redis >/dev/nul
 kube --namespace "$FIXTURE_NAMESPACE" rollout status \
   deployment/redis --timeout=120s
 set_profile_mode "$profile" NUTHATCH_STREAMING_MODE unavailable
-run_product_probe "$lane" "$profile" redis-unavailable
+run_runtime_http_probe "$lane" "$profile" redis-unavailable redis-unavailable
 set_profile_mode "$profile" NUTHATCH_STREAMING_MODE ""
 set_service_available redis true
 
@@ -302,57 +348,7 @@ kube --namespace "$FIXTURE_NAMESPACE" logs \
   --selector="app=nuthatch-$lane" \
   --container nuthatch \
   --tail=-1 >"$RUNTIME/runtime-app.log"
-grep '^{' "$RUNTIME/runtime-app.log" >"$RUNTIME/runtime-audit.jsonl"
-test -s "$RUNTIME/runtime-audit.jsonl"
-jq --slurp -e '
-  length > 0
-  and all(.[]; has("ts") and has("action") and has("result"))
-' \
-  "$RUNTIME/runtime-audit.jsonl" >/dev/null
-for action in authentication login logout authorization put delete; do
-  grep -F "\"action\":\"$action\"" "$RUNTIME/runtime-audit.jsonl" >/dev/null
-done
-jq --slurp -e '
-  [.[] | select(.action == "put")] as $records
-  | ($records | length) > 0
-    and all($records[]; has("user_id") and has("root") and has("bytes"))
-' "$RUNTIME/runtime-audit.jsonl" >/dev/null
-jq --slurp -e '
-  [.[] |
-    select(
-      (.action == "put" or .action == "delete")
-      and (.result != "ok")
-    )
-  ] as $records
-  | ($records | length) > 0
-    and all($records[]; has("reason"))
-' "$RUNTIME/runtime-audit.jsonl" >/dev/null
-if grep -Ei 'Bearer |"authorization"[[:space:]]*:|client_secret|NUTHATCH_' \
-    "$RUNTIME/runtime-app.log" >/dev/null; then
-  printf 'audit output exposed prohibited data\n' >&2
-  exit 1
-fi
-while IFS= read -r secret_value; do
-  if [[ -n "$secret_value" ]] &&
-      grep -F -- "$secret_value" "$RUNTIME/runtime-app.log" >/dev/null; then
-    printf 'audit output exposed a configured secret\n' >&2
-    exit 1
-  fi
-done < <(
-  {
-    printf '%s\n' synthetic-runtime-session-jwt-secret
-    sed -n 's/^NUTHATCH_PG_CONN=//p' "$RUNTIME/$profile.env"
-    sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$RUNTIME/$profile.env"
-    sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$RUNTIME/$profile.env"
-    sed -n '1p' "$RUNTIME/$profile.env.oidc-secret"
-  }
-)
-for body_value in private-a runtime-coherence first-personal second-personal; do
-  if grep -F -- "$body_value" "$RUNTIME/runtime-app.log" >/dev/null; then
-    printf 'audit output exposed request body data\n' >&2
-    exit 1
-  fi
-done
+run_runtime_audit_probe "$lane" "$profile"
 
 test "$(kube --namespace "$FIXTURE_NAMESPACE" get deployment \
   "nuthatch-$lane" --output jsonpath='{.status.readyReplicas}')" = 2