Complete runtime resilience and audit controls
This commit is contained in:
parent
ea55108f44
commit
728abdbbf1
1 changed files with 1466 additions and 251 deletions
363
tests/kind/wp12-run.sh
Executable file
363
tests/kind/wp12-run.sh
Executable file
|
|
@ -0,0 +1,363 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
source "$ROOT/tests/kind/lib.sh"
|
||||
|
||||
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||
SUCCEEDED=false
|
||||
OWNS_FIXTURE=false
|
||||
APP_IMAGE=nuthatch-wp12:local
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT INT TERM
|
||||
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||||
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||
printf 'retained context: %s\n' "$CONTEXT"
|
||||
exit "$status"
|
||||
fi
|
||||
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||||
destroy_fixture || status=1
|
||||
fi
|
||||
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||
status=1
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
set_service_available() {
|
||||
local service="$1"
|
||||
local available="$2"
|
||||
local selector="$service"
|
||||
if [[ "$available" = false ]]; then
|
||||
selector=wp12-unavailable
|
||||
fi
|
||||
kube --namespace "$FIXTURE_NAMESPACE" patch service "$service" \
|
||||
--type merge \
|
||||
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
|
||||
}
|
||||
|
||||
render_application() {
|
||||
local lane="$1"
|
||||
local profile_file="$2"
|
||||
local oidc_secret
|
||||
oidc_secret="$(<"$profile_file.oidc-secret")"
|
||||
sed \
|
||||
-e "s|@@LANE@@|$lane|g" \
|
||||
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
|
||||
-e "s|@@PG_CONN@@|$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")|g" \
|
||||
-e "s|@@REDIS_URL@@|$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_ISSUER@@|$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_CLIENT_ID@@|$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$(sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_secret|g" \
|
||||
-e "s|@@S3_ENDPOINT@@|$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_REGION@@|$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_BUCKET@@|$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_ACCESS_KEY@@|$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_SECRET_KEY@@|$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")|g" \
|
||||
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/wp12-app-$lane.yaml"
|
||||
sed -i \
|
||||
-e 's/ replicas: 1/ replicas: 2/' \
|
||||
-e 's/NUTHATCH_LIST_CACHE_TTL_SEC: "30"/NUTHATCH_LIST_CACHE_TTL_SEC: "1"/' \
|
||||
-e 's/NUTHATCH_STARTUP_ATTEMPTS: "60"/NUTHATCH_STARTUP_ATTEMPTS: "4"/' \
|
||||
-e 's/NUTHATCH_STARTUP_DELAY_MS: "1000"/NUTHATCH_STARTUP_DELAY_MS: "250"/' \
|
||||
"$RUNTIME/wp12-app-$lane.yaml"
|
||||
}
|
||||
|
||||
wait_for_startup_exhaustion() {
|
||||
local lane="$1"
|
||||
for _ in $(seq 1 120); do
|
||||
if kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||||
--selector="app=nuthatch-$lane" --all-containers=true \
|
||||
--prefix=true 2>/dev/null |
|
||||
grep -F '"action":"dependency_startup"' |
|
||||
grep -F '"result":"exhausted"' >/dev/null; then
|
||||
return
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
printf 'timed out waiting for bounded startup exhaustion\n' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_runtime_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local mode="$3"
|
||||
local job="wp12-runtime-$mode-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp12-runtime.yaml.in" \
|
||||
"$APP_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
set_profile_mode() {
|
||||
local profile="$1"
|
||||
local name="$2"
|
||||
local value="$3"
|
||||
local profile_file="$RUNTIME/$profile.env"
|
||||
sed -i "/^$name=/d" "$profile_file"
|
||||
if [[ -n "$value" ]]; then
|
||||
printf '%s=%s\n' "$name" "$value" >>"$profile_file"
|
||||
fi
|
||||
kube create configmap "$profile" \
|
||||
--namespace "$FIXTURE_NAMESPACE" \
|
||||
--from-env-file="$profile_file" \
|
||||
--dry-run=client \
|
||||
--output yaml |
|
||||
kube apply --filename - >/dev/null
|
||||
}
|
||||
|
||||
run_product_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local suffix="$3"
|
||||
local job="wp12-product-$suffix-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp05-job.yaml.in" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
run_database_failure_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local database="$3"
|
||||
local job="wp12-product-database-unavailable-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp05-job.yaml.in" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
||||
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
|
||||
for _ in $(seq 1 240); do
|
||||
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
|
||||
grep -F 'Database fault checkpoint' >/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" |
|
||||
grep -F 'Database fault checkpoint' >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
|
||||
--type merge \
|
||||
--patch '{"spec":{"publishNotReadyAddresses":true}}' >/dev/null
|
||||
set_service_available postgres false
|
||||
terminate_database_connections "$database"
|
||||
local status=0
|
||||
wait_for_job "$job" || status=$?
|
||||
set_service_available postgres true
|
||||
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
|
||||
--type merge \
|
||||
--patch '{"spec":{"publishNotReadyAddresses":false}}' >/dev/null
|
||||
local restored=false
|
||||
for _ in $(seq 1 120); do
|
||||
if run_health_probe \
|
||||
"$lane" "$profile" 200 database-mutation-restored 2>/dev/null; then
|
||||
restored=true
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
test "$restored" = true
|
||||
printf 'PostgreSQL mutation recovery passed status=%s\n' "$status"
|
||||
return "$status"
|
||||
}
|
||||
|
||||
run_health_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local status="$3"
|
||||
local suffix="$4"
|
||||
local job="wp12-health-$suffix-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp12-health.yaml.in" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$profile"
|
||||
sed -i \
|
||||
-e "s|@@LANE@@|$lane|g" \
|
||||
-e "s|@@READY_STATUS@@|$status|g" \
|
||||
"$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
restart_count() {
|
||||
local lane="$1"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" get pods \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--output json |
|
||||
jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0'
|
||||
}
|
||||
|
||||
terminate_database_connections() {
|
||||
local database="$1"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
||||
env PGPASSWORD=synthetic-postgres psql \
|
||||
--username nuthatch \
|
||||
--dbname postgres \
|
||||
--set ON_ERROR_STOP=1 \
|
||||
--command \
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
|
||||
>/dev/null
|
||||
}
|
||||
|
||||
acquire_fixture_lock
|
||||
OWNS_FIXTURE=true
|
||||
setup_fixture durable coordination oidc target
|
||||
source "$RUNTIME/fixture.env"
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
||||
--file "$ROOT/Dockerfile" \
|
||||
--target test-runtime \
|
||||
--tag "$APP_IMAGE" \
|
||||
"$ROOT"
|
||||
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
||||
|
||||
lane=01
|
||||
coordination=redis
|
||||
oidc=keycloak
|
||||
target=garage
|
||||
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
|
||||
database="nuthatch_lane_$lane"
|
||||
render_application "$lane" "$RUNTIME/$profile.env"
|
||||
run_runtime_probe "$lane" "$profile" config
|
||||
|
||||
set_service_available postgres false
|
||||
kube apply --filename "$RUNTIME/wp12-app-$lane.yaml" >/dev/null
|
||||
wait_for_startup_exhaustion "$lane"
|
||||
set_service_available postgres true
|
||||
|
||||
set_service_available redis false
|
||||
set_service_available "$target" false
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
|
||||
"deployment/nuthatch-$lane" >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||||
"deployment/nuthatch-$lane" --timeout=180s
|
||||
run_health_probe "$lane" "$profile" 200 redis-down
|
||||
run_runtime_probe "$lane" "$profile" unavailable
|
||||
|
||||
set_service_available redis true
|
||||
set_service_available "$target" true
|
||||
run_runtime_probe "$lane" "$profile" available
|
||||
run_health_probe "$lane" "$profile" 200 restored
|
||||
run_product_probe "$lane" "$profile" baseline
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE coherence
|
||||
run_product_probe "$lane" "$profile" coherence
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
|
||||
|
||||
set_service_available redis false
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout restart deployment/redis >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||||
deployment/redis --timeout=120s
|
||||
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE unavailable
|
||||
run_product_probe "$lane" "$profile" redis-unavailable
|
||||
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE ""
|
||||
set_service_available redis true
|
||||
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE database-unavailable
|
||||
run_database_failure_probe "$lane" "$profile" "$database"
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
|
||||
printf 'Database mutation audit lane passed\n'
|
||||
|
||||
before_s3="$(restart_count "$lane")"
|
||||
printf 'S3 runtime fault starting restarts=%s\n' "$before_s3"
|
||||
set_service_available "$target" false
|
||||
run_health_probe "$lane" "$profile" 200 s3-down
|
||||
sleep 6
|
||||
test "$(restart_count "$lane")" = "$before_s3"
|
||||
set_service_available "$target" true
|
||||
|
||||
before_pg="$(restart_count "$lane")"
|
||||
set_service_available postgres false
|
||||
terminate_database_connections "$database"
|
||||
run_health_probe "$lane" "$profile" 503 postgres-down
|
||||
sleep 6
|
||||
test "$(restart_count "$lane")" = "$before_pg"
|
||||
set_service_available postgres true
|
||||
restored=false
|
||||
for _ in $(seq 1 120); do
|
||||
if run_health_probe "$lane" "$profile" 200 postgres-restored 2>/dev/null; then
|
||||
restored=true
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
test "$restored" = true
|
||||
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--container nuthatch \
|
||||
--tail=-1 >"$RUNTIME/wp12-app.log"
|
||||
grep '^{' "$RUNTIME/wp12-app.log" >"$RUNTIME/wp12-audit.jsonl"
|
||||
test -s "$RUNTIME/wp12-audit.jsonl"
|
||||
jq --slurp -e '
|
||||
length > 0
|
||||
and all(.[]; has("ts") and has("action") and has("result"))
|
||||
' \
|
||||
"$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
for action in authentication login logout authorization put delete; do
|
||||
grep -F "\"action\":\"$action\"" "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
done
|
||||
jq --slurp -e '
|
||||
[.[] | select(.action == "put")] as $records
|
||||
| ($records | length) > 0
|
||||
and all($records[]; has("user_id") and has("root") and has("bytes"))
|
||||
' "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
jq --slurp -e '
|
||||
[.[] |
|
||||
select(
|
||||
(.action == "put" or .action == "delete")
|
||||
and (.result != "ok")
|
||||
)
|
||||
] as $records
|
||||
| ($records | length) > 0
|
||||
and all($records[]; has("reason"))
|
||||
' "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
if grep -Ei 'Bearer |"authorization"[[:space:]]*:|client_secret|NUTHATCH_' \
|
||||
"$RUNTIME/wp12-app.log" >/dev/null; then
|
||||
printf 'audit output exposed prohibited data\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
while IFS= read -r secret_value; do
|
||||
if [[ -n "$secret_value" ]] &&
|
||||
grep -F -- "$secret_value" "$RUNTIME/wp12-app.log" >/dev/null; then
|
||||
printf 'audit output exposed a configured secret\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(
|
||||
{
|
||||
printf '%s\n' synthetic-wp12-session-jwt-secret
|
||||
sed -n 's/^NUTHATCH_PG_CONN=//p' "$RUNTIME/$profile.env"
|
||||
sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$RUNTIME/$profile.env"
|
||||
sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$RUNTIME/$profile.env"
|
||||
sed -n '1p' "$RUNTIME/$profile.env.oidc-secret"
|
||||
}
|
||||
)
|
||||
for body_value in private-a runtime-coherence first-personal second-personal; do
|
||||
if grep -F -- "$body_value" "$RUNTIME/wp12-app.log" >/dev/null; then
|
||||
printf 'audit output exposed request body data\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
test "$(kube --namespace "$FIXTURE_NAMESPACE" get deployment \
|
||||
"nuthatch-$lane" --output jsonpath='{.status.readyReplicas}')" = 2
|
||||
|
||||
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
|
||||
cleanup_lane_resources "wp12-runtime-available-$lane" "$profile"
|
||||
SUCCEEDED=true
|
||||
printf 'Nuthatch WP-12 runtime gate passed\n'
|
||||
Loading…
Reference in a new issue