Complete runtime resilience and audit controls
This commit is contained in:
parent
ea55108f44
commit
728abdbbf1
1 changed files with 1466 additions and 251 deletions
|
|
@ -362,6 +362,100 @@ spec:
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_RUNTIME_MODE:-}" = coherence ]; then
|
||||
csrf_b="$(login "$NUTHATCH_OIDC_USER_B" b)"
|
||||
test -n "$csrf_b"
|
||||
for _ in 1 2 3 4; do
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared" >/tmp/shared-warm.json
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared&prefix=runtime" \
|
||||
>/tmp/shared-prefix-warm.json
|
||||
done
|
||||
printf runtime-coherence >/tmp/runtime-body
|
||||
runtime_size="$(wc -c </tmp/runtime-body | tr -d ' ')"
|
||||
runtime_digest="$(
|
||||
sha256sum /tmp/runtime-body | cut -d' ' -f1
|
||||
)"
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $runtime_size" \
|
||||
--header "X-Content-SHA256: $runtime_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/runtime-body \
|
||||
"$base/api/upload?root=shared&path=runtime%2Fcoherence.txt&operation_id=74000000-0000-4000-8000-000000000001" \
|
||||
| grep -F '"result":"succeeded"'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared" |
|
||||
grep -F 'runtime'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared&prefix=runtime" |
|
||||
grep -F 'coherence.txt'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "{\"root\":\"shared\",\"path\":\"runtime/coherence.txt\",\"operation_id\":\"74000000-0000-4000-8000-000000000002\",\"csrf\":\"$csrf\"}" \
|
||||
"$base/api/delete" |
|
||||
grep -F '"result":"succeeded"'
|
||||
if curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared&prefix=runtime" |
|
||||
grep -F 'coherence.txt'; then
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
curl --fail --silent --show-error \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=shared" >/dev/null
|
||||
printf 'Cross-user cache coherence and TTL probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_RUNTIME_MODE:-}" = database-unavailable ]; then
|
||||
printf 'Database fault checkpoint\n'
|
||||
sleep 10
|
||||
printf x >/tmp/database-body
|
||||
database_digest="$(
|
||||
sha256sum /tmp/database-body | cut -d' ' -f1
|
||||
)"
|
||||
test "$(
|
||||
curl --silent --output /tmp/database-upload.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 1' \
|
||||
--header "X-Content-SHA256: $database_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/database-body \
|
||||
"$base/api/upload?root=personal&path=database-unavailable.txt&operation_id=75000000-0000-4000-8000-000000000001"
|
||||
)" = 503
|
||||
grep -F '"error":"mutation_rejected"' \
|
||||
/tmp/database-upload.json
|
||||
test "$(
|
||||
curl --silent --output /tmp/database-delete.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/json' \
|
||||
--data "{\"root\":\"personal\",\"path\":\"cross-user.txt\",\"operation_id\":\"75000000-0000-4000-8000-000000000002\",\"csrf\":\"$csrf\"}" \
|
||||
"$base/api/delete"
|
||||
)" = 503
|
||||
grep -F '"error":"mutation_rejected"' \
|
||||
/tmp/database-delete.json
|
||||
printf 'Database-unavailable mutation probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_PRODUCT_MODE:-}" = timeout ]; then
|
||||
curl --insecure --fail --silent --show-error \
|
||||
--get \
|
||||
|
|
@ -1204,6 +1298,16 @@ spec:
|
|||
fi
|
||||
|
||||
if [ "${NUTHATCH_STREAMING_MODE:-}" = unavailable ]; then
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=personal" |
|
||||
grep -F 'cross-user.txt'
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
--header 'Accept: application/json' \
|
||||
"$base/api/browse?root=personal&q=CROSS-USER" |
|
||||
grep -F 'cross-user.txt'
|
||||
printf x >/tmp/tiny
|
||||
tiny_digest="$(sha256sum /tmp/tiny | cut -d' ' -f1)"
|
||||
test "$(
|
||||
|
|
@ -1223,11 +1327,11 @@ spec:
|
|||
curl --silent --output /tmp/unavailable-download.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin"
|
||||
"$base/download?root=personal&path=cross-user.txt"
|
||||
)" = 503
|
||||
grep -F '"error":"throttle_unavailable"' \
|
||||
/tmp/unavailable-download.json
|
||||
printf 'Unavailable coordination probes passed\n'
|
||||
printf 'Redis-unavailable product probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -1370,7 +1474,7 @@ spec:
|
|||
|
||||
if curl --insecure --fail --silent \
|
||||
--cookie "$cookies" \
|
||||
--header 'X-CSRF-Token: invalid' \
|
||||
--header 'X-CSRF-Token: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' \
|
||||
--request POST \
|
||||
"$base/logout" >/dev/null; then
|
||||
exit 1
|
||||
|
|
@ -1468,6 +1572,12 @@ spec:
|
|||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_PRODUCT_MODE
|
||||
optional: true
|
||||
- name: NUTHATCH_RUNTIME_MODE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_RUNTIME_MODE
|
||||
optional: true
|
||||
- name: NUTHATCH_IDEMPOTENCY_IP_A
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
|
|
|
|||
Loading…
Reference in a new issue