Complete runtime resilience and audit controls
This commit is contained in:
parent
ea55108f44
commit
728abdbbf1
5 changed files with 1466 additions and 251 deletions
|
|
@ -362,6 +362,100 @@ spec:
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_RUNTIME_MODE:-}" = coherence ]; then
|
||||
csrf_b="$(login "$NUTHATCH_OIDC_USER_B" b)"
|
||||
test -n "$csrf_b"
|
||||
for _ in 1 2 3 4; do
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared" >/tmp/shared-warm.json
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared&prefix=runtime" \
|
||||
>/tmp/shared-prefix-warm.json
|
||||
done
|
||||
printf runtime-coherence >/tmp/runtime-body
|
||||
runtime_size="$(wc -c </tmp/runtime-body | tr -d ' ')"
|
||||
runtime_digest="$(
|
||||
sha256sum /tmp/runtime-body | cut -d' ' -f1
|
||||
)"
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $runtime_size" \
|
||||
--header "X-Content-SHA256: $runtime_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/runtime-body \
|
||||
"$base/api/upload?root=shared&path=runtime%2Fcoherence.txt&operation_id=74000000-0000-4000-8000-000000000001" \
|
||||
| grep -F '"result":"succeeded"'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared" |
|
||||
grep -F 'runtime'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared&prefix=runtime" |
|
||||
grep -F 'coherence.txt'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "{\"root\":\"shared\",\"path\":\"runtime/coherence.txt\",\"operation_id\":\"74000000-0000-4000-8000-000000000002\",\"csrf\":\"$csrf\"}" \
|
||||
"$base/api/delete" |
|
||||
grep -F '"result":"succeeded"'
|
||||
if curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared&prefix=runtime" |
|
||||
grep -F 'coherence.txt'; then
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared" >/dev/null
|
||||
printf 'Cross-user cache coherence and TTL probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_RUNTIME_MODE:-}" = database-unavailable ]; then
|
||||
printf 'Database fault checkpoint\n'
|
||||
sleep 10
|
||||
printf x >/tmp/database-body
|
||||
database_digest="$(
|
||||
sha256sum /tmp/database-body | cut -d' ' -f1
|
||||
)"
|
||||
test "$(
|
||||
curl --silent --output /tmp/database-upload.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 1' \
|
||||
--header "X-Content-SHA256: $database_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/database-body \
|
||||
"$base/api/upload?root=personal&path=database-unavailable.txt&operation_id=75000000-0000-4000-8000-000000000001"
|
||||
)" = 503
|
||||
grep -F '"error":"mutation_rejected"' \
|
||||
/tmp/database-upload.json
|
||||
test "$(
|
||||
curl --silent --output /tmp/database-delete.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "{\"root\":\"personal\",\"path\":\"cross-user.txt\",\"operation_id\":\"75000000-0000-4000-8000-000000000002\",\"csrf\":\"$csrf\"}" \
|
||||
"$base/api/delete"
|
||||
)" = 503
|
||||
grep -F '"error":"mutation_rejected"' \
|
||||
/tmp/database-delete.json
|
||||
printf 'Database-unavailable mutation probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_PRODUCT_MODE:-}" = timeout ]; then
|
||||
curl --insecure --fail --silent --show-error \
|
||||
--get \
|
||||
|
|
@ -1204,6 +1298,16 @@ spec:
|
|||
fi
|
||||
|
||||
if [ "${NUTHATCH_STREAMING_MODE:-}" = unavailable ]; then
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=personal" |
|
||||
grep -F 'cross-user.txt'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=personal&q=CROSS-USER" |
|
||||
grep -F 'cross-user.txt'
|
||||
printf x >/tmp/tiny
|
||||
tiny_digest="$(sha256sum /tmp/tiny | cut -d' ' -f1)"
|
||||
test "$(
|
||||
|
|
@ -1223,11 +1327,11 @@ spec:
|
|||
curl --silent --output /tmp/unavailable-download.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin"
|
||||
"$base/download?root=personal&path=cross-user.txt"
|
||||
)" = 503
|
||||
grep -F '"error":"throttle_unavailable"' \
|
||||
/tmp/unavailable-download.json
|
||||
printf 'Unavailable coordination probes passed\n'
|
||||
printf 'Redis-unavailable product probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -1370,7 +1474,7 @@ spec:
|
|||
|
||||
if curl --insecure --fail --silent \
|
||||
--cookie "$cookies" \
|
||||
--header 'X-CSRF-Token: invalid' \
|
||||
--header 'X-CSRF-Token: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' \
|
||||
--request POST \
|
||||
"$base/logout" >/dev/null; then
|
||||
exit 1
|
||||
|
|
@ -1468,6 +1572,12 @@ spec:
|
|||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_PRODUCT_MODE
|
||||
optional: true
|
||||
- name: NUTHATCH_RUNTIME_MODE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_RUNTIME_MODE
|
||||
optional: true
|
||||
- name: NUTHATCH_IDEMPOTENCY_IP_A
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
|
|
|
|||
|
|
@ -340,6 +340,12 @@ while IFS=$'\t' read -r lane coordination oidc target; do
|
|||
"UPDATE mutations SET request_deadline_at = now() + interval '60 seconds', worker_lease_until = now() + interval '60 seconds' WHERE operation_id = 'recovery-after-dispatch'; SELECT 1" \
|
||||
>/dev/null
|
||||
deploy_recovery "$lane" "$profile"
|
||||
before_pg_restarts="$(
|
||||
kube --namespace "$FIXTURE_NAMESPACE" get pods \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--output json |
|
||||
jq '[.items[].status.containerStatuses[].restartCount] | add'
|
||||
)"
|
||||
set_service_available postgres false
|
||||
terminate_database_connections "$database"
|
||||
sleep 6
|
||||
|
|
@ -348,7 +354,7 @@ while IFS=$'\t' read -r lane coordination oidc target; do
|
|||
--selector="app=nuthatch-$lane" \
|
||||
--output json |
|
||||
jq '[.items[].status.containerStatuses[].restartCount] | add'
|
||||
)" -gt 0
|
||||
)" = "$before_pg_restarts"
|
||||
set_service_available postgres true
|
||||
wait_for_value "$database" "SELECT 1" 1 "PostgreSQL recovery"
|
||||
pg_value \
|
||||
|
|
|
|||
39
tests/kind/wp12-health.yaml.in
Normal file
39
tests/kind/wp12-health.yaml.in
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: @@JOB_NAME@@
|
||||
namespace: nuthatch-testbed
|
||||
spec:
|
||||
activeDeadlineSeconds: 60
|
||||
backoffLimit: 0
|
||||
template:
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: health
|
||||
image: @@CURL_IMAGE@@
|
||||
imagePullPolicy: Never
|
||||
command: [/bin/sh, -ec]
|
||||
args:
|
||||
- |
|
||||
base="https://nuthatch-@@LANE@@:8080"
|
||||
test "$(
|
||||
curl --insecure --silent --output /tmp/health \
|
||||
--write-out '%{http_code}' "$base/healthz"
|
||||
)" = 200
|
||||
grep -F '"ok":true' /tmp/health
|
||||
test "$(
|
||||
curl --insecure --silent --output /tmp/ready \
|
||||
--write-out '%{http_code}' "$base/readyz"
|
||||
)" = "@@READY_STATUS@@"
|
||||
curl --insecure --silent --output /dev/null "$base/app"
|
||||
test "$(
|
||||
curl --insecure --silent --output /dev/null \
|
||||
--write-out '%{http_code}' "$base/api/session"
|
||||
)" = 303
|
||||
printf 'runtime health semantics passed\n'
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 16Mi}
|
||||
limits: {cpu: 250m, memory: 64Mi}
|
||||
363
tests/kind/wp12-run.sh
Executable file
363
tests/kind/wp12-run.sh
Executable file
|
|
@ -0,0 +1,363 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
source "$ROOT/tests/kind/lib.sh"
|
||||
|
||||
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||
SUCCEEDED=false
|
||||
OWNS_FIXTURE=false
|
||||
APP_IMAGE=nuthatch-wp12:local
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT INT TERM
|
||||
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||||
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||
printf 'retained context: %s\n' "$CONTEXT"
|
||||
exit "$status"
|
||||
fi
|
||||
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||||
destroy_fixture || status=1
|
||||
fi
|
||||
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||
status=1
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
set_service_available() {
|
||||
local service="$1"
|
||||
local available="$2"
|
||||
local selector="$service"
|
||||
if [[ "$available" = false ]]; then
|
||||
selector=wp12-unavailable
|
||||
fi
|
||||
kube --namespace "$FIXTURE_NAMESPACE" patch service "$service" \
|
||||
--type merge \
|
||||
--patch "{\"spec\":{\"selector\":{\"app\":\"$selector\"}}}" >/dev/null
|
||||
}
|
||||
|
||||
render_application() {
|
||||
local lane="$1"
|
||||
local profile_file="$2"
|
||||
local oidc_secret
|
||||
oidc_secret="$(<"$profile_file.oidc-secret")"
|
||||
sed \
|
||||
-e "s|@@LANE@@|$lane|g" \
|
||||
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
|
||||
-e "s|@@PG_CONN@@|$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")|g" \
|
||||
-e "s|@@REDIS_URL@@|$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_ISSUER@@|$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_CLIENT_ID@@|$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$(sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file")|g" \
|
||||
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_secret|g" \
|
||||
-e "s|@@S3_ENDPOINT@@|$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_REGION@@|$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_BUCKET@@|$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_ACCESS_KEY@@|$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")|g" \
|
||||
-e "s|@@S3_SECRET_KEY@@|$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")|g" \
|
||||
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/wp12-app-$lane.yaml"
|
||||
sed -i \
|
||||
-e 's/ replicas: 1/ replicas: 2/' \
|
||||
-e 's/NUTHATCH_LIST_CACHE_TTL_SEC: "30"/NUTHATCH_LIST_CACHE_TTL_SEC: "1"/' \
|
||||
-e 's/NUTHATCH_STARTUP_ATTEMPTS: "60"/NUTHATCH_STARTUP_ATTEMPTS: "4"/' \
|
||||
-e 's/NUTHATCH_STARTUP_DELAY_MS: "1000"/NUTHATCH_STARTUP_DELAY_MS: "250"/' \
|
||||
"$RUNTIME/wp12-app-$lane.yaml"
|
||||
}
|
||||
|
||||
wait_for_startup_exhaustion() {
|
||||
local lane="$1"
|
||||
for _ in $(seq 1 120); do
|
||||
if kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||||
--selector="app=nuthatch-$lane" --all-containers=true \
|
||||
--prefix=true 2>/dev/null |
|
||||
grep -F '"action":"dependency_startup"' |
|
||||
grep -F '"result":"exhausted"' >/dev/null; then
|
||||
return
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
printf 'timed out waiting for bounded startup exhaustion\n' >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
run_runtime_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local mode="$3"
|
||||
local job="wp12-runtime-$mode-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp12-runtime.yaml.in" \
|
||||
"$APP_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@MODE@@|$mode|g" "$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
set_profile_mode() {
|
||||
local profile="$1"
|
||||
local name="$2"
|
||||
local value="$3"
|
||||
local profile_file="$RUNTIME/$profile.env"
|
||||
sed -i "/^$name=/d" "$profile_file"
|
||||
if [[ -n "$value" ]]; then
|
||||
printf '%s=%s\n' "$name" "$value" >>"$profile_file"
|
||||
fi
|
||||
kube create configmap "$profile" \
|
||||
--namespace "$FIXTURE_NAMESPACE" \
|
||||
--from-env-file="$profile_file" \
|
||||
--dry-run=client \
|
||||
--output yaml |
|
||||
kube apply --filename - >/dev/null
|
||||
}
|
||||
|
||||
run_product_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local suffix="$3"
|
||||
local job="wp12-product-$suffix-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp05-job.yaml.in" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
run_database_failure_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local database="$3"
|
||||
local job="wp12-product-database-unavailable-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp05-job.yaml.in" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
||||
kube apply --filename "$RUNTIME/$job.yaml" >/dev/null
|
||||
for _ in $(seq 1 240); do
|
||||
if kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" 2>/dev/null |
|
||||
grep -F 'Database fault checkpoint' >/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs "job/$job" |
|
||||
grep -F 'Database fault checkpoint' >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
|
||||
--type merge \
|
||||
--patch '{"spec":{"publishNotReadyAddresses":true}}' >/dev/null
|
||||
set_service_available postgres false
|
||||
terminate_database_connections "$database"
|
||||
local status=0
|
||||
wait_for_job "$job" || status=$?
|
||||
set_service_available postgres true
|
||||
kube --namespace "$FIXTURE_NAMESPACE" patch service "nuthatch-$lane" \
|
||||
--type merge \
|
||||
--patch '{"spec":{"publishNotReadyAddresses":false}}' >/dev/null
|
||||
local restored=false
|
||||
for _ in $(seq 1 120); do
|
||||
if run_health_probe \
|
||||
"$lane" "$profile" 200 database-mutation-restored 2>/dev/null; then
|
||||
restored=true
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
test "$restored" = true
|
||||
printf 'PostgreSQL mutation recovery passed status=%s\n' "$status"
|
||||
return "$status"
|
||||
}
|
||||
|
||||
run_health_probe() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local status="$3"
|
||||
local suffix="$4"
|
||||
local job="wp12-health-$suffix-$lane"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp12-health.yaml.in" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$profile"
|
||||
sed -i \
|
||||
-e "s|@@LANE@@|$lane|g" \
|
||||
-e "s|@@READY_STATUS@@|$status|g" \
|
||||
"$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
restart_count() {
|
||||
local lane="$1"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" get pods \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--output json |
|
||||
jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0'
|
||||
}
|
||||
|
||||
terminate_database_connections() {
|
||||
local database="$1"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
|
||||
env PGPASSWORD=synthetic-postgres psql \
|
||||
--username nuthatch \
|
||||
--dbname postgres \
|
||||
--set ON_ERROR_STOP=1 \
|
||||
--command \
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$database' AND pid <> pg_backend_pid()" \
|
||||
>/dev/null
|
||||
}
|
||||
|
||||
acquire_fixture_lock
|
||||
OWNS_FIXTURE=true
|
||||
setup_fixture durable coordination oidc target
|
||||
source "$RUNTIME/fixture.env"
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
||||
--file "$ROOT/Dockerfile" \
|
||||
--target test-runtime \
|
||||
--tag "$APP_IMAGE" \
|
||||
"$ROOT"
|
||||
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
||||
|
||||
lane=01
|
||||
coordination=redis
|
||||
oidc=keycloak
|
||||
target=garage
|
||||
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
|
||||
database="nuthatch_lane_$lane"
|
||||
render_application "$lane" "$RUNTIME/$profile.env"
|
||||
run_runtime_probe "$lane" "$profile" config
|
||||
|
||||
set_service_available postgres false
|
||||
kube apply --filename "$RUNTIME/wp12-app-$lane.yaml" >/dev/null
|
||||
wait_for_startup_exhaustion "$lane"
|
||||
set_service_available postgres true
|
||||
|
||||
set_service_available redis false
|
||||
set_service_available "$target" false
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout restart \
|
||||
"deployment/nuthatch-$lane" >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||||
"deployment/nuthatch-$lane" --timeout=180s
|
||||
run_health_probe "$lane" "$profile" 200 redis-down
|
||||
run_runtime_probe "$lane" "$profile" unavailable
|
||||
|
||||
set_service_available redis true
|
||||
set_service_available "$target" true
|
||||
run_runtime_probe "$lane" "$profile" available
|
||||
run_health_probe "$lane" "$profile" 200 restored
|
||||
run_product_probe "$lane" "$profile" baseline
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE coherence
|
||||
run_product_probe "$lane" "$profile" coherence
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
|
||||
|
||||
set_service_available redis false
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout restart deployment/redis >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||||
deployment/redis --timeout=120s
|
||||
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE unavailable
|
||||
run_product_probe "$lane" "$profile" redis-unavailable
|
||||
set_profile_mode "$profile" NUTHATCH_STREAMING_MODE ""
|
||||
set_service_available redis true
|
||||
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE database-unavailable
|
||||
run_database_failure_probe "$lane" "$profile" "$database"
|
||||
set_profile_mode "$profile" NUTHATCH_RUNTIME_MODE ""
|
||||
printf 'Database mutation audit lane passed\n'
|
||||
|
||||
before_s3="$(restart_count "$lane")"
|
||||
printf 'S3 runtime fault starting restarts=%s\n' "$before_s3"
|
||||
set_service_available "$target" false
|
||||
run_health_probe "$lane" "$profile" 200 s3-down
|
||||
sleep 6
|
||||
test "$(restart_count "$lane")" = "$before_s3"
|
||||
set_service_available "$target" true
|
||||
|
||||
before_pg="$(restart_count "$lane")"
|
||||
set_service_available postgres false
|
||||
terminate_database_connections "$database"
|
||||
run_health_probe "$lane" "$profile" 503 postgres-down
|
||||
sleep 6
|
||||
test "$(restart_count "$lane")" = "$before_pg"
|
||||
set_service_available postgres true
|
||||
restored=false
|
||||
for _ in $(seq 1 120); do
|
||||
if run_health_probe "$lane" "$profile" 200 postgres-restored 2>/dev/null; then
|
||||
restored=true
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
test "$restored" = true
|
||||
|
||||
kube --namespace "$FIXTURE_NAMESPACE" logs \
|
||||
--selector="app=nuthatch-$lane" \
|
||||
--container nuthatch \
|
||||
--tail=-1 >"$RUNTIME/wp12-app.log"
|
||||
grep '^{' "$RUNTIME/wp12-app.log" >"$RUNTIME/wp12-audit.jsonl"
|
||||
test -s "$RUNTIME/wp12-audit.jsonl"
|
||||
jq --slurp -e '
|
||||
length > 0
|
||||
and all(.[]; has("ts") and has("action") and has("result"))
|
||||
' \
|
||||
"$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
for action in authentication login logout authorization put delete; do
|
||||
grep -F "\"action\":\"$action\"" "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
done
|
||||
jq --slurp -e '
|
||||
[.[] | select(.action == "put")] as $records
|
||||
| ($records | length) > 0
|
||||
and all($records[]; has("user_id") and has("root") and has("bytes"))
|
||||
' "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
jq --slurp -e '
|
||||
[.[] |
|
||||
select(
|
||||
(.action == "put" or .action == "delete")
|
||||
and (.result != "ok")
|
||||
)
|
||||
] as $records
|
||||
| ($records | length) > 0
|
||||
and all($records[]; has("reason"))
|
||||
' "$RUNTIME/wp12-audit.jsonl" >/dev/null
|
||||
if grep -Ei 'Bearer |"authorization"[[:space:]]*:|client_secret|NUTHATCH_' \
|
||||
"$RUNTIME/wp12-app.log" >/dev/null; then
|
||||
printf 'audit output exposed prohibited data\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
while IFS= read -r secret_value; do
|
||||
if [[ -n "$secret_value" ]] &&
|
||||
grep -F -- "$secret_value" "$RUNTIME/wp12-app.log" >/dev/null; then
|
||||
printf 'audit output exposed a configured secret\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(
|
||||
{
|
||||
printf '%s\n' synthetic-wp12-session-jwt-secret
|
||||
sed -n 's/^NUTHATCH_PG_CONN=//p' "$RUNTIME/$profile.env"
|
||||
sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$RUNTIME/$profile.env"
|
||||
sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$RUNTIME/$profile.env"
|
||||
sed -n '1p' "$RUNTIME/$profile.env.oidc-secret"
|
||||
}
|
||||
)
|
||||
for body_value in private-a runtime-coherence first-personal second-personal; do
|
||||
if grep -F -- "$body_value" "$RUNTIME/wp12-app.log" >/dev/null; then
|
||||
printf 'audit output exposed request body data\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
test "$(kube --namespace "$FIXTURE_NAMESPACE" get deployment \
|
||||
"nuthatch-$lane" --output jsonpath='{.status.readyReplicas}')" = 2
|
||||
|
||||
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
|
||||
cleanup_lane_resources "wp12-runtime-available-$lane" "$profile"
|
||||
SUCCEEDED=true
|
||||
printf 'Nuthatch WP-12 runtime gate passed\n'
|
||||
32
tests/kind/wp12-runtime.yaml.in
Normal file
32
tests/kind/wp12-runtime.yaml.in
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: @@JOB_NAME@@
|
||||
namespace: nuthatch-testbed
|
||||
spec:
|
||||
activeDeadlineSeconds: 120
|
||||
backoffLimit: 0
|
||||
template:
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: runtime
|
||||
image: @@TEST_IMAGE@@
|
||||
imagePullPolicy: Never
|
||||
command: [nuthatch-wp12-runtime, "@@MODE@@"]
|
||||
envFrom:
|
||||
- configMapRef: {name: @@PROFILE_NAME@@}
|
||||
env:
|
||||
- {name: NUTHATCH_LIST_CACHE_TTL_SEC, value: "1"}
|
||||
- {name: NUTHATCH_JWT_SECRET, value: synthetic-wp12-session-jwt-secret}
|
||||
- {name: NUTHATCH_PUBLIC_BASE_URL, value: "https://wp12.invalid"}
|
||||
- {name: NUTHATCH_OIDC_ISSUER, value: "https://issuer.invalid"}
|
||||
- {name: NUTHATCH_OIDC_CLIENT_ID, value: wp12}
|
||||
- {name: NUTHATCH_OIDC_ADDITIONAL_AUDIENCES, value: ""}
|
||||
- {name: NUTHATCH_OIDC_CLIENT_SECRET, value: synthetic-wp12-oidc-client-secret}
|
||||
- {name: NUTHATCH_OIDC_REDIRECT_URI, value: "https://wp12.invalid/oidc/callback"}
|
||||
resources:
|
||||
requests: {cpu: 50m, memory: 32Mi}
|
||||
limits: {cpu: "1", memory: 128Mi}
|
||||
Loading…
Reference in a new issue