363 lines
12 KiB
Shell
363 lines
12 KiB
Shell
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||
|
|
source "$ROOT/tests/kind/lib.sh"
|
||
|
|
|
||
|
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||
|
|
SUCCEEDED=false
|
||
|
|
OWNS_FIXTURE=false
|
||
|
|
APP_IMAGE=nuthatch-wp12:local
|
||
|
|
|
||
|
|
cleanup() {
|
||
|
|
local status=$?
|
||
|
|
trap - EXIT INT TERM
|
||
|
|
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||
|
|
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||
|
|
printf 'retained context: %s\n' "$CONTEXT"
|
||
|
|
exit "$status"
|
||
|
|
fi
|
||
|
|
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||
|
|
destroy_fixture || status=1
|
||
|
|
fi
|
||
|
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||
|
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||
|
|
status=1
|
||
|
|
fi
|
||
|
|
exit "$status"
|
||
|
|
}
|
||
|
|
|
||
|
|
trap cleanup EXIT
|
||
|
|
trap 'exit 130' INT
|
||
|
|
trap 'exit 143' TERM
|
||
|
|
|
||
|
|
set_service_available() {
|
||
|
|
local service="$1"
|
||
|
|
local available="$2"
|
||
|
|
local selector="$service"
|
||
|
|
if [[ "$available" = false ]]; then
|
||
|
|
selector=wp12-unavailable
|
||
|
|
fi
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" patch service "$service" \
|
||
|
|
--type merge \
|
||
|
|
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
|
||
|
|
}
|
||
|
|
|
||
|
|
render_application() {
|
||
|
|
local lane="$1"
|
||
|
|
local profile_file="$2"
|
||
|
|
local oidc_secret
|
||
|
|
oidc_secret="$(<"$profile_file.oidc-secret")"
|
||
|
|
sed \
|
||
|
|
-e "s|@@LANE@@|$lane|g" \
|
||
|
|
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
|
||
|
|
-e "s|@@PG_CONN@@|$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@REDIS_URL@@|$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@OIDC_ISSUER@@|$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@OIDC_CLIENT_ID@@|$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$(sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_secret|g" \
|
||
|
|
-e "s|@@S3_ENDPOINT@@|$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@S3_REGION@@|$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@S3_BUCKET@@|$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@S3_ACCESS_KEY@@|$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")|g" \
|
||
|
|
-e "s|@@S3_SECRET_KEY@@|$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")|g" \
|
||
|
|
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/wp12-app-$lane.yaml"
|
||
|
|
sed -i \
|
||
|
|
-e 's/ replicas: 1/ replicas: 2/' \
|
||
|
|
-e 's/NUTHATCH_LIST_CACHE_TTL_SEC: "30"/NUTHATCH_LIST_CACHE_TTL_SEC: "1"/' \
|
||
|
|
-e 's/NUTHATCH_STARTUP_ATTEMPTS: "60"/NUTHATCH_STARTUP_ATTEMPTS: "4"/' \
|
||
|
|
-e 's/NUTHATCH_STARTUP_DELAY_MS: "1000"/NUTHATCH_STARTUP_DELAY_MS: "250"/' \
|
||
|
|
"$RUNTIME/wp12-app-$lane.yaml"
|
||
|
|
}
|
||
|
|
|
||
|
|
wait_for_startup_exhaustion() {
|
||
|
|
local lane="$1"
|
||
|
|
for _ in $(seq 1 120); do
|
||
|
|
if kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||
|
|
--selector="app=nuthatch-$lane" --all-containers=true \
|
||
|
|
--prefix=true 2>/dev/null |
|
||
|
|
grep -F '"action":"dependency_startup"' |
|
||
|
|
grep -F '"result":"exhausted"' >/dev/null; then
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
sleep 0.25
|
||
|
|
done
|
||
|
|
printf 'timed out waiting for bounded startup exhaustion\n' >&2
|
||
|
|
return 1
|
||
|
|
}
|
||
|
|
|
||
|
|
run_runtime_probe() {
|
||
|
|
local lane="$1"
|
||
|
|
local profile="$2"
|
||
|
|
local mode="$3"
|
||
|
|
local job="wp12-runtime-$mode-$lane"
|
||
|
|
render_lane_job \
|
||
|
|
"$job" \
|
||
|
|
"$TESTBED/wp12-runtime.yaml.in" \
|
||
|
|
"$APP_IMAGE" \
|
||
|
|
"$profile"
|
||
|
|
sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
|
||
|
|
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||
|
|
}
|
||
|
|
|
||
|
|
set_profile_mode() {
|
||
|
|
local profile="$1"
|
||
|
|
local name="$2"
|
||
|
|
local value="$3"
|
||
|
|
local profile_file="$RUNTIME/$profile.env"
|
||
|
|
sed -i "/^$name=/d" "$profile_file"
|
||
|
|
if [[ -n "$value" ]]; then
|
||
|
|
printf '%s=%s\n' "$name" "$value" >>"$profile_file"
|
||
|
|
fi
|
||
|
|
kube create configmap "$profile" \
|
||
|
|
--namespace "$FIXTURE_NAMESPACE" \
|
||
|
|
--from-env-file="$profile_file" \
|
||
|
|
--dry-run=client \
|
||
|
|
--output yaml |
|
||
|
|
kube apply --filename - >/dev/null
|
||
|
|
}
|
||
|
|
|
||
|
|
run_product_probe() {
|
||
|
|
local lane="$1"
|
||
|
|
local profile="$2"
|
||
|
|
local suffix="$3"
|
||
|
|
local job="wp12-product-$suffix-$lane"
|
||
|
|
render_lane_job \
|
||
|
|
"$job" \
|
||
|
|
"$TESTBED/wp05-job.yaml.in" \
|
||
|
|
"$CURL_LOCAL_IMAGE" \
|
||
|
|
"$profile"
|
||
|
|
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
||
|
|
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||
|
|
}
|
||
|
|
|
||
|
|
run_database_failure_probe() {
|
||
|
|
local lane="$1"
|
||
|
|
local profile="$2"
|
||
|
|
local database="$3"
|
||
|
|
local job="wp12-product-database-unavailable-$lane"
|
||
|
|
render_lane_job \
|
||
|
|
"$job" \
|
||
|
|
"$TESTBED/wp05-job.yaml.in" \
|
||
|
|
"$CURL_LOCAL_IMAGE" \
|
||
|
|
"$profile"
|
||
|
|
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
||
|
|
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
|
||
|
|
for _ in $(seq 1 240); do
|
||
|
|
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
|
||
|
|
grep -F 'Database fault checkpoint' >/dev/null; then
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
sleep 0.25
|
||
|
|
done
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" |
|
||
|
|
grep -F 'Database fault checkpoint' >/dev/null
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
|
||
|
|
--type merge \
|
||
|
|
--patch '{"spec":{"publishNotReadyAddresses":true}}' >/dev/null
|
||
|
|
set_service_available postgres false
|
||
|
|
terminate_database_connections "$database"
|
||
|
|
local status=0
|
||
|
|
wait_for_job "$job" || status=$?
|
||
|
|
set_service_available postgres true
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
|
||
|
|
--type merge \
|
||
|
|
--patch '{"spec":{"publishNotReadyAddresses":false}}' >/dev/null
|
||
|
|
local restored=false
|
||
|
|
for _ in $(seq 1 120); do
|
||
|
|
if run_health_probe \
|
||
|
|
"$lane" "$profile" 200 database-mutation-restored 2>/dev/null; then
|
||
|
|
restored=true
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
sleep 0.25
|
||
|
|
done
|
||
|
|
test "$restored" = true
|
||
|
|
printf 'PostgreSQL mutation recovery passed status=%s\n' "$status"
|
||
|
|
return "$status"
|
||
|
|
}
|
||
|
|
|
||
|
|
run_health_probe() {
|
||
|
|
local lane="$1"
|
||
|
|
local profile="$2"
|
||
|
|
local status="$3"
|
||
|
|
local suffix="$4"
|
||
|
|
local job="wp12-health-$suffix-$lane"
|
||
|
|
render_lane_job \
|
||
|
|
"$job" \
|
||
|
|
"$TESTBED/wp12-health.yaml.in" \
|
||
|
|
"$CURL_LOCAL_IMAGE" \
|
||
|
|
"$profile"
|
||
|
|
sed -i \
|
||
|
|
-e "s|@@LANE@@|$lane|g" \
|
||
|
|
-e "s|@@READY_STATUS@@|$status|g" \
|
||
|
|
"$RUNTIME/$job.yaml"
|
||
|
|
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||
|
|
}
|
||
|
|
|
||
|
|
restart_count() {
|
||
|
|
local lane="$1"
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" get pods \
|
||
|
|
--selector="app=nuthatch-$lane" \
|
||
|
|
--output json |
|
||
|
|
jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0'
|
||
|
|
}
|
||
|
|
|
||
|
|
terminate_database_connections() {
|
||
|
|
local database="$1"
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
||
|
|
env PGPASSWORD=synthetic-postgres psql \
|
||
|
|
--username nuthatch \
|
||
|
|
--dbname postgres \
|
||
|
|
--set ON_ERROR_STOP=1 \
|
||
|
|
--command \
|
||
|
|
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
|
||
|
|
>/dev/null
|
||
|
|
}
|
||
|
|
|
||
|
|
acquire_fixture_lock
|
||
|
|
OWNS_FIXTURE=true
|
||
|
|
setup_fixture durable coordination oidc target
|
||
|
|
source "$RUNTIME/fixture.env"
|
||
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
||
|
|
--file "$ROOT/Dockerfile" \
|
||
|
|
--target test-runtime \
|
||
|
|
--tag "$APP_IMAGE" \
|
||
|
|
"$ROOT"
|
||
|
|
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
||
|
|
|
||
|
|
lane=01
|
||
|
|
coordination=redis
|
||
|
|
oidc=keycloak
|
||
|
|
target=garage
|
||
|
|
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
|
||
|
|
database="nuthatch_lane_$lane"
|
||
|
|
render_application "$lane" "$RUNTIME/$profile.env"
|
||
|
|
run_runtime_probe "$lane" "$profile" config
|
||
|
|
|
||
|
|
set_service_available postgres false
|
||
|
|
kube apply --filename "$RUNTIME/wp12-app-$lane.yaml" >/dev/null
|
||
|
|
wait_for_startup_exhaustion "$lane"
|
||
|
|
set_service_available postgres true
|
||
|
|
|
||
|
|
set_service_available redis false
|
||
|
|
set_service_available "$target" false
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
|
||
|
|
"deployment/nuthatch-$lane" >/dev/null
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||
|
|
"deployment/nuthatch-$lane" --timeout=180s
|
||
|
|
run_health_probe "$lane" "$profile" 200 redis-down
|
||
|
|
run_runtime_probe "$lane" "$profile" unavailable
|
||
|
|
|
||
|
|
set_service_available redis true
|
||
|
|
set_service_available "$target" true
|
||
|
|
run_runtime_probe "$lane" "$profile" available
|
||
|
|
run_health_probe "$lane" "$profile" 200 restored
|
||
|
|
run_product_probe "$lane" "$profile" baseline
|
||
|
|
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE coherence
|
||
|
|
run_product_probe "$lane" "$profile" coherence
|
||
|
|
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
|
||
|
|
|
||
|
|
set_service_available redis false
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout restart deployment/redis >/dev/null
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||
|
|
deployment/redis --timeout=120s
|
||
|
|
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE unavailable
|
||
|
|
run_product_probe "$lane" "$profile" redis-unavailable
|
||
|
|
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE ""
|
||
|
|
set_service_available redis true
|
||
|
|
|
||
|
|
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE database-unavailable
|
||
|
|
run_database_failure_probe "$lane" "$profile" "$database"
|
||
|
|
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
|
||
|
|
printf 'Database mutation audit lane passed\n'
|
||
|
|
|
||
|
|
before_s3="$(restart_count "$lane")"
|
||
|
|
printf 'S3 runtime fault starting restarts=%s\n' "$before_s3"
|
||
|
|
set_service_available "$target" false
|
||
|
|
run_health_probe "$lane" "$profile" 200 s3-down
|
||
|
|
sleep 6
|
||
|
|
test "$(restart_count "$lane")" = "$before_s3"
|
||
|
|
set_service_available "$target" true
|
||
|
|
|
||
|
|
before_pg="$(restart_count "$lane")"
|
||
|
|
set_service_available postgres false
|
||
|
|
terminate_database_connections "$database"
|
||
|
|
run_health_probe "$lane" "$profile" 503 postgres-down
|
||
|
|
sleep 6
|
||
|
|
test "$(restart_count "$lane")" = "$before_pg"
|
||
|
|
set_service_available postgres true
|
||
|
|
restored=false
|
||
|
|
for _ in $(seq 1 120); do
|
||
|
|
if run_health_probe "$lane" "$profile" 200 postgres-restored 2>/dev/null; then
|
||
|
|
restored=true
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
sleep 0.25
|
||
|
|
done
|
||
|
|
test "$restored" = true
|
||
|
|
|
||
|
|
kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||
|
|
--selector="app=nuthatch-$lane" \
|
||
|
|
--container nuthatch \
|
||
|
|
--tail=-1 >"$RUNTIME/wp12-app.log"
|
||
|
|
grep '^{' "$RUNTIME/wp12-app.log" >"$RUNTIME/wp12-audit.jsonl"
|
||
|
|
test -s "$RUNTIME/wp12-audit.jsonl"
|
||
|
|
jq --slurp -e '
|
||
|
|
length > 0
|
||
|
|
and all(.[]; has("ts") and has("action") and has("result"))
|
||
|
|
' \
|
||
|
|
"$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||
|
|
for action in authentication login logout authorization put delete; do
|
||
|
|
grep -F "\"action\":\"$action\"" "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||
|
|
done
|
||
|
|
jq --slurp -e '
|
||
|
|
[.[] | select(.action == "put")] as $records
|
||
|
|
| ($records | length) > 0
|
||
|
|
and all($records[]; has("user_id") and has("root") and has("bytes"))
|
||
|
|
' "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||
|
|
jq --slurp -e '
|
||
|
|
[.[] |
|
||
|
|
select(
|
||
|
|
(.action == "put" or .action == "delete")
|
||
|
|
and (.result != "ok")
|
||
|
|
)
|
||
|
|
] as $records
|
||
|
|
| ($records | length) > 0
|
||
|
|
and all($records[]; has("reason"))
|
||
|
|
' "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||
|
|
if grep -Ei 'Bearer |"authorization"[[:space:]]*:|client_secret|NUTHATCH_' \
|
||
|
|
"$RUNTIME/wp12-app.log" >/dev/null; then
|
||
|
|
printf 'audit output exposed prohibited data\n' >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
while IFS= read -r secret_value; do
|
||
|
|
if [[ -n "$secret_value" ]] &&
|
||
|
|
grep -F -- "$secret_value" "$RUNTIME/wp12-app.log" >/dev/null; then
|
||
|
|
printf 'audit output exposed a configured secret\n' >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
done < <(
|
||
|
|
{
|
||
|
|
printf '%s\n' synthetic-wp12-session-jwt-secret
|
||
|
|
sed -n 's/^NUTHATCH_PG_CONN=//p' "$RUNTIME/$profile.env"
|
||
|
|
sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$RUNTIME/$profile.env"
|
||
|
|
sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$RUNTIME/$profile.env"
|
||
|
|
sed -n '1p' "$RUNTIME/$profile.env.oidc-secret"
|
||
|
|
}
|
||
|
|
)
|
||
|
|
for body_value in private-a runtime-coherence first-personal second-personal; do
|
||
|
|
if grep -F -- "$body_value" "$RUNTIME/wp12-app.log" >/dev/null; then
|
||
|
|
printf 'audit output exposed request body data\n' >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
test "$(kube --namespace "$FIXTURE_NAMESPACE" get deployment \
|
||
|
|
"nuthatch-$lane" --output jsonpath='{.status.readyReplicas}')" = 2
|
||
|
|
|
||
|
|
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
|
||
|
|
cleanup_lane_resources "wp12-runtime-available-$lane" "$profile"
|
||
|
|
SUCCEEDED=true
|
||
|
|
printf 'Nuthatch WP-12 runtime gate passed\n'
|