Stream object transfers with shared byte throttling
This commit is contained in:
parent
a4e615096c
commit
4b044b1456
2 changed files with 1228 additions and 105 deletions
|
|
@ -362,6 +362,374 @@ spec:
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_STREAMING_MODE:-}" = large ]; then
|
||||
large_bytes="${NUTHATCH_STREAMING_LARGE_BYTES:?}"
|
||||
printf 'Preparing %s-byte streaming object\n' "$large_bytes"
|
||||
large_digest="$(
|
||||
dd if=/dev/zero bs=1048576 \
|
||||
count="$((large_bytes / 1048576))" status=none |
|
||||
sha256sum |
|
||||
cut -d' ' -f1
|
||||
)"
|
||||
printf 'Uploading constrained-memory streaming object\n'
|
||||
status="$(
|
||||
dd if=/dev/zero bs=1048576 \
|
||||
count="$((large_bytes / 1048576))" status=none |
|
||||
curl --silent --show-error \
|
||||
--output /tmp/large-upload.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $large_bytes" \
|
||||
--header "X-Content-SHA256: $large_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--request POST \
|
||||
--upload-file - \
|
||||
"$base/upload?root=personal&path=streaming-large.bin&operation_id=70000000-0000-4000-8000-000000000001"
|
||||
)"
|
||||
printf 'Large upload status=%s\n' "$status"
|
||||
test "$status" = 200
|
||||
grep -F '"result":"succeeded"' /tmp/large-upload.json
|
||||
printf 'Downloading constrained-memory streaming object\n'
|
||||
download_digest="$(
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin" |
|
||||
sha256sum |
|
||||
cut -d' ' -f1
|
||||
)"
|
||||
printf 'Large download digest expected=%s actual=%s\n' \
|
||||
"$large_digest" "$download_digest"
|
||||
test "$download_digest" = "$large_digest"
|
||||
|
||||
printf x >/tmp/tiny
|
||||
tiny_digest="$(sha256sum /tmp/tiny | cut -d' ' -f1)"
|
||||
for attempt in first replay; do
|
||||
status="$(
|
||||
curl --silent --show-error \
|
||||
--output "/tmp/prior-desired-$attempt.json" \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $large_bytes" \
|
||||
--header "X-Content-SHA256: $large_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/tiny \
|
||||
"$base/upload?root=personal&path=streaming-large.bin&operation_id=70000000-0000-4000-8000-000000000007"
|
||||
)"
|
||||
test "$status" = 200
|
||||
grep -F '"result":"succeeded"' \
|
||||
"/tmp/prior-desired-$attempt.json"
|
||||
done
|
||||
status="$(
|
||||
curl --silent --show-error \
|
||||
--output /tmp/size-mismatch.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 2' \
|
||||
--header "X-Content-SHA256: $tiny_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/tiny \
|
||||
"$base/upload?root=personal&path=size-mismatch.bin&operation_id=70000000-0000-4000-8000-000000000002"
|
||||
)"
|
||||
printf 'Size mismatch status=%s body=' "$status"
|
||||
cat /tmp/size-mismatch.json
|
||||
printf '\n'
|
||||
test "$status" = 400
|
||||
grep -F '"error":"size_mismatch"' /tmp/size-mismatch.json
|
||||
status="$(
|
||||
curl --silent --show-error \
|
||||
--output /tmp/identity-mismatch.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 1' \
|
||||
--header "X-Content-SHA256: $(printf '%064d' 0)" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/tiny \
|
||||
"$base/upload?root=personal&path=identity-mismatch.bin&operation_id=70000000-0000-4000-8000-000000000003"
|
||||
)"
|
||||
printf 'Identity mismatch status=%s body=' "$status"
|
||||
cat /tmp/identity-mismatch.json
|
||||
printf '\n'
|
||||
test "$status" = 400
|
||||
grep -F '"error":"content_identity_mismatch"' \
|
||||
/tmp/identity-mismatch.json
|
||||
status="$(
|
||||
set +o pipefail
|
||||
dd if=/dev/zero bs=1048576 count=160 status=none |
|
||||
curl --silent --show-error \
|
||||
--max-time 5 \
|
||||
--limit-rate 64k \
|
||||
--output /tmp/too-large.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $NUTHATCH_STREAMING_OVER_LIMIT_BYTES" \
|
||||
--header "X-Content-SHA256: $tiny_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--request POST \
|
||||
--upload-file - \
|
||||
"$base/upload?root=personal&path=too-large.bin&operation_id=70000000-0000-4000-8000-000000000004"
|
||||
)"
|
||||
printf 'Over-limit status=%s body=' "$status"
|
||||
cat /tmp/too-large.json
|
||||
printf '\n'
|
||||
test "$status" = 413
|
||||
grep -F '"error":"object_too_large"' /tmp/too-large.json
|
||||
|
||||
if dd if=/dev/zero bs=1048576 \
|
||||
count="$((large_bytes / 1048576))" status=none |
|
||||
curl --silent --show-error \
|
||||
--max-time 1 \
|
||||
--limit-rate 64k \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $large_bytes" \
|
||||
--header "X-Content-SHA256: $large_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--request POST \
|
||||
--upload-file - \
|
||||
"$base/upload?root=personal&path=aborted-upload.bin&operation_id=70000000-0000-4000-8000-000000000005" \
|
||||
>/tmp/aborted-upload.json; then
|
||||
exit 1
|
||||
fi
|
||||
aborted_upload_status="$(
|
||||
curl --silent --output /dev/null \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=aborted-upload.bin"
|
||||
)"
|
||||
printf 'Aborted upload lookup status=%s\n' \
|
||||
"$aborted_upload_status"
|
||||
test "$aborted_upload_status" = 404
|
||||
for retry in $(seq 1 20); do
|
||||
status="$(
|
||||
curl --silent --show-error \
|
||||
--output /tmp/aborted-upload-replay.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $large_bytes" \
|
||||
--header "X-Content-SHA256: $large_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/tiny \
|
||||
"$base/upload?root=personal&path=aborted-upload.bin&operation_id=70000000-0000-4000-8000-000000000005"
|
||||
)"
|
||||
test "$status" = 200
|
||||
if grep -F '"result":"failed"' \
|
||||
/tmp/aborted-upload-replay.json >/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
grep -F '"result":"failed"' /tmp/aborted-upload-replay.json
|
||||
if dd if=/dev/zero bs=1048576 \
|
||||
count="$((large_bytes / 1048576))" status=none |
|
||||
curl --silent --show-error \
|
||||
--max-time 1 \
|
||||
--limit-rate 64k \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header "X-Declared-Size: $large_bytes" \
|
||||
--header "X-Content-SHA256: $(printf '%064d' 0)" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--request POST \
|
||||
--upload-file - \
|
||||
"$base/upload?root=personal&path=streaming-large.bin&operation_id=70000000-0000-4000-8000-000000000006" \
|
||||
>/tmp/aborted-overwrite.json; then
|
||||
exit 1
|
||||
fi
|
||||
preserved_digest="$(
|
||||
curl --fail --silent --show-error \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin" |
|
||||
sha256sum |
|
||||
cut -d' ' -f1
|
||||
)"
|
||||
test "$preserved_digest" = "$large_digest"
|
||||
if curl --silent --show-error \
|
||||
--max-time 1 \
|
||||
--limit-rate 64k \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin" \
|
||||
>/tmp/aborted-download; then
|
||||
exit 1
|
||||
fi
|
||||
aborted_bytes="$(wc -c </tmp/aborted-download | tr -d ' ')"
|
||||
test "$aborted_bytes" -gt 0
|
||||
test "$aborted_bytes" -lt "$large_bytes"
|
||||
curl --fail --silent --show-error "$base/readyz" |
|
||||
grep -F '"ready":true'
|
||||
printf 'Large streaming and abort probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_STREAMING_MODE:-}" = throttle ]; then
|
||||
csrf_b="$(login "$NUTHATCH_OIDC_USER_B" b)"
|
||||
dd if=/dev/zero of=/tmp/joint-upload bs=65536 \
|
||||
count=1 status=none
|
||||
joint_digest="$(
|
||||
sha256sum /tmp/joint-upload | cut -d' ' -f1
|
||||
)"
|
||||
status="$(
|
||||
curl --silent --show-error \
|
||||
--output /tmp/joint-upload.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 65536' \
|
||||
--header "X-Content-SHA256: $joint_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/joint-upload \
|
||||
"$base/upload?root=personal&path=joint-throttle.bin&operation_id=70000000-0000-4000-8000-000000000013"
|
||||
)"
|
||||
test "$status" = 200
|
||||
grep -F '"result":"succeeded"' /tmp/joint-upload.json
|
||||
dd if=/dev/zero of=/tmp/throttle-upload bs=1048576 \
|
||||
count=1 status=none
|
||||
upload_digest="$(
|
||||
sha256sum /tmp/throttle-upload | cut -d' ' -f1
|
||||
)"
|
||||
status="$(
|
||||
curl --silent --show-error \
|
||||
--output /tmp/throttle-upload.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 1048576' \
|
||||
--header "X-Content-SHA256: $upload_digest" \
|
||||
--header "X-CSRF-Token: $csrf_b" \
|
||||
--data-binary @/tmp/throttle-upload \
|
||||
"$base/upload?root=personal&path=throttled-upload.bin&operation_id=71000000-0000-4000-8000-000000000001"
|
||||
)"
|
||||
test "$status" = 429
|
||||
grep -F '"error":"bandwidth_exceeded"' /tmp/throttle-upload.json
|
||||
printf x >/tmp/tiny
|
||||
test "$(
|
||||
curl --silent --output /tmp/prestart-upload.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie /tmp/cookies-b \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 1' \
|
||||
--header "X-Content-SHA256: $upload_digest" \
|
||||
--header "X-CSRF-Token: $csrf_b" \
|
||||
--data-binary @/tmp/tiny \
|
||||
"$base/upload?root=personal&path=prestart.bin&operation_id=71000000-0000-4000-8000-000000000002"
|
||||
)" = 429
|
||||
|
||||
curl --silent --show-error --cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin" \
|
||||
>/tmp/parallel-a &
|
||||
first_pid=$!
|
||||
curl --silent --show-error --cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin" \
|
||||
>/tmp/parallel-b &
|
||||
second_pid=$!
|
||||
first_ok=true
|
||||
second_ok=true
|
||||
wait "$first_pid" || first_ok=false
|
||||
wait "$second_pid" || second_ok=false
|
||||
test "$first_ok" = false
|
||||
test "$second_ok" = false
|
||||
parallel_bytes="$(
|
||||
wc -c /tmp/parallel-a /tmp/parallel-b |
|
||||
awk 'END {print $1}'
|
||||
)"
|
||||
printf 'Parallel downloads moved %s bytes with burst %s\n' \
|
||||
"$parallel_bytes" \
|
||||
"$NUTHATCH_STREAMING_THROTTLE_BURST_BYTES"
|
||||
test "$parallel_bytes" -gt 0
|
||||
joint_limit="$(
|
||||
expr "$NUTHATCH_STREAMING_THROTTLE_BURST_BYTES" - 65536 + 4096
|
||||
)"
|
||||
test "$parallel_bytes" -le "$joint_limit"
|
||||
prestart_status="$(
|
||||
curl --silent --output /tmp/prestart-download.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin"
|
||||
)"
|
||||
printf 'Post-exhaustion download status=%s\n' "$prestart_status"
|
||||
test "$prestart_status" = 429
|
||||
printf 'Shared incremental throttle probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_STREAMING_MODE:-}" = refill ]; then
|
||||
dd if=/dev/zero of=/tmp/refill-upload bs=65536 \
|
||||
count=1 status=none
|
||||
refill_digest="$(
|
||||
sha256sum /tmp/refill-upload | cut -d' ' -f1
|
||||
)"
|
||||
for suffix in initial refilled; do
|
||||
if [ "$suffix" = refilled ]; then
|
||||
sleep 2
|
||||
refill_operation=72000000-0000-4000-8000-000000000002
|
||||
else
|
||||
refill_operation=72000000-0000-4000-8000-000000000001
|
||||
fi
|
||||
status="$(
|
||||
curl --silent --show-error \
|
||||
--output "/tmp/refill-$suffix.json" \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 65536' \
|
||||
--header "X-Content-SHA256: $refill_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/refill-upload \
|
||||
"$base/upload?root=personal&path=refill-$suffix.bin&operation_id=$refill_operation"
|
||||
)"
|
||||
test "$status" = 200
|
||||
grep -F '"result":"succeeded"' "/tmp/refill-$suffix.json"
|
||||
if [ "$suffix" = initial ]; then
|
||||
test "$(
|
||||
curl --silent --output /tmp/refill-exhausted.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 65536' \
|
||||
--header "X-Content-SHA256: $refill_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/refill-upload \
|
||||
"$base/upload?root=personal&path=refill-exhausted.bin&operation_id=72000000-0000-4000-8000-000000000003"
|
||||
)" = 429
|
||||
fi
|
||||
done
|
||||
printf 'Token-bucket exhaustion and refill probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${NUTHATCH_STREAMING_MODE:-}" = unavailable ]; then
|
||||
printf x >/tmp/tiny
|
||||
tiny_digest="$(sha256sum /tmp/tiny | cut -d' ' -f1)"
|
||||
test "$(
|
||||
curl --silent --output /tmp/unavailable-upload.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
--header 'Content-Type: application/octet-stream' \
|
||||
--header 'X-Declared-Size: 1' \
|
||||
--header "X-Content-SHA256: $tiny_digest" \
|
||||
--header "X-CSRF-Token: $csrf" \
|
||||
--data-binary @/tmp/tiny \
|
||||
"$base/upload?root=personal&path=unavailable.bin&operation_id=72000000-0000-4000-8000-000000000001"
|
||||
)" = 503
|
||||
grep -F '"error":"throttle_unavailable"' \
|
||||
/tmp/unavailable-upload.json
|
||||
test "$(
|
||||
curl --silent --output /tmp/unavailable-download.json \
|
||||
--write-out '%{http_code}' \
|
||||
--cookie "$cookies" \
|
||||
"$base/download?root=personal&path=streaming-large.bin"
|
||||
)" = 503
|
||||
grep -F '"error":"throttle_unavailable"' \
|
||||
/tmp/unavailable-download.json
|
||||
printf 'Unavailable coordination probes passed\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
exercise_root() {
|
||||
root="$1"
|
||||
suffix="$2"
|
||||
|
|
@ -563,6 +931,30 @@ spec:
|
|||
configMapKeyRef:
|
||||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_OIDC_REDIRECT_URI
|
||||
- name: NUTHATCH_STREAMING_MODE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_STREAMING_MODE
|
||||
optional: true
|
||||
- name: NUTHATCH_STREAMING_LARGE_BYTES
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_STREAMING_LARGE_BYTES
|
||||
optional: true
|
||||
- name: NUTHATCH_STREAMING_OVER_LIMIT_BYTES
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_STREAMING_OVER_LIMIT_BYTES
|
||||
optional: true
|
||||
- name: NUTHATCH_STREAMING_THROTTLE_BURST_BYTES
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: @@PROFILE_NAME@@
|
||||
key: NUTHATCH_STREAMING_THROTTLE_BURST_BYTES
|
||||
optional: true
|
||||
- name: NUTHATCH_OIDC_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
|
|
|||
217
tests/kind/wp07-run.sh
Executable file
217
tests/kind/wp07-run.sh
Executable file
|
|
@ -0,0 +1,217 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
source "$ROOT/tests/kind/lib.sh"
|
||||
|
||||
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||
SUCCEEDED=false
|
||||
OWNS_FIXTURE=false
|
||||
APP_IMAGE=nuthatch-wp07:local
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT INT TERM
|
||||
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||||
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||
printf 'retained context: %s\n' "$CONTEXT"
|
||||
exit "$status"
|
||||
fi
|
||||
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||||
destroy_fixture || status=1
|
||||
fi
|
||||
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||
status=1
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
render_application() {
|
||||
local lane="$1"
|
||||
local profile_file="$2"
|
||||
local rate="$3"
|
||||
local burst="$4"
|
||||
local pg_conn
|
||||
local redis_url
|
||||
local oidc_issuer
|
||||
local oidc_client_id
|
||||
local oidc_additional_audiences
|
||||
local oidc_client_secret
|
||||
local s3_endpoint
|
||||
local s3_region
|
||||
local s3_bucket
|
||||
local s3_access_key
|
||||
local s3_secret_key
|
||||
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
|
||||
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
|
||||
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
|
||||
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
|
||||
oidc_additional_audiences="$(
|
||||
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
|
||||
)"
|
||||
oidc_client_secret="$(<"$profile_file.oidc-secret")"
|
||||
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
|
||||
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
|
||||
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
|
||||
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
|
||||
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
|
||||
sed \
|
||||
-e "s|@@LANE@@|$lane|g" \
|
||||
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
|
||||
-e "s|@@PG_CONN@@|$pg_conn|g" \
|
||||
-e "s|@@REDIS_URL@@|$redis_url|g" \
|
||||
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
|
||||
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
|
||||
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
|
||||
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
|
||||
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
|
||||
-e "s|@@S3_REGION@@|$s3_region|g" \
|
||||
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
|
||||
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
|
||||
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
|
||||
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/wp07-app-$lane.yaml"
|
||||
sed -i \
|
||||
"/NUTHATCH_S3_PATH_STYLE/a\\ NUTHATCH_MAX_OBJECT_BYTES: \"201326592\"\\n NUTHATCH_THROTTLE_BYTES_PER_SEC: \"$rate\"\\n NUTHATCH_THROTTLE_BURST_BYTES: \"$burst\"" \
|
||||
"$RUNTIME/wp07-app-$lane.yaml"
|
||||
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/wp07-app-$lane.yaml"
|
||||
sed -i \
|
||||
's/limits: {cpu: "2", memory: 512Mi}/limits: {cpu: "2", memory: 128Mi}/' \
|
||||
"$RUNTIME/wp07-app-$lane.yaml"
|
||||
sed -i \
|
||||
'/ envFrom:/i\ env:\n - {name: GHCRTS, value: "-N2 -A1m -M96m"}' \
|
||||
"$RUNTIME/wp07-app-$lane.yaml"
|
||||
}
|
||||
|
||||
cleanup_application() {
|
||||
local lane="$1"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
||||
"deployment/nuthatch-$lane" \
|
||||
"service/nuthatch-$lane" \
|
||||
"configmap/nuthatch-$lane" \
|
||||
"secret/nuthatch-$lane" \
|
||||
--ignore-not-found >/dev/null
|
||||
rm -f "$RUNTIME/wp07-app-$lane.yaml"
|
||||
}
|
||||
|
||||
set_streaming_profile() {
|
||||
local profile="$1"
|
||||
local mode="$2"
|
||||
local throttle_burst=262144
|
||||
local profile_file="$RUNTIME/$profile.env"
|
||||
if [[ "$mode" = refill ]]; then
|
||||
throttle_burst=65536
|
||||
fi
|
||||
sed -i '/^NUTHATCH_STREAMING_/d' "$profile_file"
|
||||
{
|
||||
printf 'NUTHATCH_STREAMING_MODE=%s\n' "$mode"
|
||||
printf 'NUTHATCH_STREAMING_LARGE_BYTES=167772160\n'
|
||||
printf 'NUTHATCH_STREAMING_OVER_LIMIT_BYTES=201326593\n'
|
||||
printf 'NUTHATCH_STREAMING_THROTTLE_BURST_BYTES=%s\n' "$throttle_burst"
|
||||
} >> "$profile_file"
|
||||
kube create configmap "$profile" \
|
||||
--namespace "$FIXTURE_NAMESPACE" \
|
||||
--from-env-file="$profile_file" \
|
||||
--dry-run=client \
|
||||
--output yaml |
|
||||
kube apply --filename - >/dev/null
|
||||
}
|
||||
|
||||
run_streaming_job() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local mode="$3"
|
||||
local job="wp07-$mode-$lane"
|
||||
set_streaming_profile "$profile" "$mode"
|
||||
render_lane_job \
|
||||
"$job" \
|
||||
"$TESTBED/wp05-job.yaml.in" \
|
||||
"$CURL_LOCAL_IMAGE" \
|
||||
"$profile"
|
||||
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
||||
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
||||
}
|
||||
|
||||
scale_coordination() {
|
||||
local provider="$1"
|
||||
local replicas="$2"
|
||||
kube --namespace "$FIXTURE_NAMESPACE" scale \
|
||||
"deployment/$provider" \
|
||||
--replicas="$replicas" >/dev/null
|
||||
if [[ "$replicas" = 0 ]]; then
|
||||
kube --namespace "$FIXTURE_NAMESPACE" wait \
|
||||
--for=delete \
|
||||
"pod" \
|
||||
--selector="app=$provider" \
|
||||
--timeout=120s >/dev/null
|
||||
else
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||||
"deployment/$provider" \
|
||||
--timeout=180s >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
deploy_application() {
|
||||
local lane="$1"
|
||||
local profile="$2"
|
||||
local rate="$3"
|
||||
local burst="$4"
|
||||
cleanup_application "$lane"
|
||||
render_application "$lane" "$RUNTIME/$profile.env" "$rate" "$burst"
|
||||
kube apply --filename "$RUNTIME/wp07-app-$lane.yaml" >/dev/null
|
||||
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
||||
"deployment/nuthatch-$lane" --timeout=180s
|
||||
}
|
||||
|
||||
acquire_fixture_lock
|
||||
OWNS_FIXTURE=true
|
||||
setup_fixture durable coordination oidc target
|
||||
source "$RUNTIME/fixture.env"
|
||||
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
||||
--file "$ROOT/Dockerfile" \
|
||||
--tag "$APP_IMAGE" \
|
||||
"$ROOT"
|
||||
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
||||
|
||||
while IFS=$'\t' read -r lane coordination oidc target; do
|
||||
if [[ -n "${NUTHATCH_WP07_LANE:-}" ]] &&
|
||||
[[ "$lane" != "$NUTHATCH_WP07_LANE" ]]; then
|
||||
continue
|
||||
fi
|
||||
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
|
||||
printf 'Running WP07 lane %s: coordination=%s oidc=%s target=%s\n' \
|
||||
"$lane" "$coordination" "$oidc" "$target"
|
||||
|
||||
deploy_application "$lane" "$profile" 536870912 536870912
|
||||
run_streaming_job "$lane" "$profile" large
|
||||
|
||||
cleanup_application "$lane"
|
||||
scale_coordination "$coordination" 0
|
||||
scale_coordination "$coordination" 1
|
||||
deploy_application "$lane" "$profile" 1 262144
|
||||
run_streaming_job "$lane" "$profile" throttle
|
||||
|
||||
deploy_application "$lane" "$profile" 65536 65536
|
||||
run_streaming_job "$lane" "$profile" refill
|
||||
|
||||
deploy_application "$lane" "$profile" 536870912 536870912
|
||||
scale_coordination "$coordination" 0
|
||||
run_streaming_job "$lane" "$profile" unavailable
|
||||
cleanup_application "$lane"
|
||||
scale_coordination "$coordination" 1
|
||||
|
||||
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
|
||||
for mode in large throttle refill unavailable; do
|
||||
cleanup_lane_resources "wp07-$mode-$lane" "$profile"
|
||||
done
|
||||
if [[ "${NUTHATCH_WP07_FIRST_LANE_ONLY:-0}" = 1 ]]; then
|
||||
break
|
||||
fi
|
||||
done < <(expand_fixture_matrix coordination oidc target)
|
||||
|
||||
SUCCEEDED=true
|
||||
printf 'Nuthatch WP-07 streaming gate passed\n'
|
||||
Loading…
Reference in a new issue