ms2k
15eb0038d2
Use descriptive test and executable names so implementation surfaces do not depend on journal organization. Consolidate the unreleased migration history into the authoritative initial schema because no deployed database requires an upgrade path.
217 lines
7 KiB
Shell
Executable file
217 lines
7 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
source "$ROOT/tests/kind/lib.sh"
|
|
|
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
|
SUCCEEDED=false
|
|
OWNS_FIXTURE=false
|
|
APP_IMAGE=nuthatch-streaming:local
|
|
|
|
cleanup() {
|
|
local status=$?
|
|
trap - EXIT INT TERM
|
|
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
|
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
|
printf 'retained context: %s\n' "$CONTEXT"
|
|
exit "$status"
|
|
fi
|
|
if [[ "$OWNS_FIXTURE" = true ]]; then
|
|
destroy_fixture || status=1
|
|
fi
|
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
|
status=1
|
|
fi
|
|
exit "$status"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
render_application() {
|
|
local lane="$1"
|
|
local profile_file="$2"
|
|
local rate="$3"
|
|
local burst="$4"
|
|
local pg_conn
|
|
local redis_url
|
|
local oidc_issuer
|
|
local oidc_client_id
|
|
local oidc_additional_audiences
|
|
local oidc_client_secret
|
|
local s3_endpoint
|
|
local s3_region
|
|
local s3_bucket
|
|
local s3_access_key
|
|
local s3_secret_key
|
|
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
|
|
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
|
|
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
|
|
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
|
|
oidc_additional_audiences="$(
|
|
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
|
|
)"
|
|
oidc_client_secret="$(<"$profile_file.oidc-secret")"
|
|
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
|
|
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
|
|
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
|
|
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
|
|
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
|
|
sed \
|
|
-e "s|@@LANE@@|$lane|g" \
|
|
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
|
|
-e "s|@@PG_CONN@@|$pg_conn|g" \
|
|
-e "s|@@REDIS_URL@@|$redis_url|g" \
|
|
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
|
|
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
|
|
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
|
|
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
|
|
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
|
|
-e "s|@@S3_REGION@@|$s3_region|g" \
|
|
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
|
|
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
|
|
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
|
|
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/streaming-app-$lane.yaml"
|
|
sed -i \
|
|
"/NUTHATCH_S3_PATH_STYLE/a\\ NUTHATCH_MAX_OBJECT_BYTES: \"201326592\"\\n NUTHATCH_THROTTLE_BYTES_PER_SEC: \"$rate\"\\n NUTHATCH_THROTTLE_BURST_BYTES: \"$burst\"" \
|
|
"$RUNTIME/streaming-app-$lane.yaml"
|
|
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/streaming-app-$lane.yaml"
|
|
sed -i \
|
|
's/limits: {cpu: "2", memory: 512Mi}/limits: {cpu: "2", memory: 128Mi}/' \
|
|
"$RUNTIME/streaming-app-$lane.yaml"
|
|
sed -i \
|
|
'/ envFrom:/i\ env:\n - {name: GHCRTS, value: "-N2 -A1m -M96m"}' \
|
|
"$RUNTIME/streaming-app-$lane.yaml"
|
|
}
|
|
|
|
cleanup_application() {
|
|
local lane="$1"
|
|
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
|
"deployment/nuthatch-$lane" \
|
|
"service/nuthatch-$lane" \
|
|
"configmap/nuthatch-$lane" \
|
|
"secret/nuthatch-$lane" \
|
|
--ignore-not-found >/dev/null
|
|
rm -f "$RUNTIME/streaming-app-$lane.yaml"
|
|
}
|
|
|
|
set_streaming_profile() {
|
|
local profile="$1"
|
|
local mode="$2"
|
|
local throttle_burst=262144
|
|
local profile_file="$RUNTIME/$profile.env"
|
|
if [[ "$mode" = refill ]]; then
|
|
throttle_burst=65536
|
|
fi
|
|
sed -i '/^NUTHATCH_STREAMING_/d' "$profile_file"
|
|
{
|
|
printf 'NUTHATCH_STREAMING_MODE=%s\n' "$mode"
|
|
printf 'NUTHATCH_STREAMING_LARGE_BYTES=167772160\n'
|
|
printf 'NUTHATCH_STREAMING_OVER_LIMIT_BYTES=201326593\n'
|
|
printf 'NUTHATCH_STREAMING_THROTTLE_BURST_BYTES=%s\n' "$throttle_burst"
|
|
} >> "$profile_file"
|
|
kube create configmap "$profile" \
|
|
--namespace "$FIXTURE_NAMESPACE" \
|
|
--from-env-file="$profile_file" \
|
|
--dry-run=client \
|
|
--output yaml |
|
|
kube apply --filename - >/dev/null
|
|
}
|
|
|
|
run_streaming_job() {
|
|
local lane="$1"
|
|
local profile="$2"
|
|
local mode="$3"
|
|
local job="streaming-$mode-$lane"
|
|
set_streaming_profile "$profile" "$mode"
|
|
render_lane_job \
|
|
"$job" \
|
|
"$TESTBED/product-matrix-job.yaml.in" \
|
|
"$CURL_LOCAL_IMAGE" \
|
|
"$profile"
|
|
sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
|
|
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
|
}
|
|
|
|
scale_coordination() {
|
|
local provider="$1"
|
|
local replicas="$2"
|
|
kube --namespace "$FIXTURE_NAMESPACE" scale \
|
|
"deployment/$provider" \
|
|
--replicas="$replicas" >/dev/null
|
|
if [[ "$replicas" = 0 ]]; then
|
|
kube --namespace "$FIXTURE_NAMESPACE" wait \
|
|
--for=delete \
|
|
"pod" \
|
|
--selector="app=$provider" \
|
|
--timeout=120s >/dev/null
|
|
else
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
|
"deployment/$provider" \
|
|
--timeout=180s >/dev/null
|
|
fi
|
|
}
|
|
|
|
deploy_application() {
|
|
local lane="$1"
|
|
local profile="$2"
|
|
local rate="$3"
|
|
local burst="$4"
|
|
cleanup_application "$lane"
|
|
render_application "$lane" "$RUNTIME/$profile.env" "$rate" "$burst"
|
|
kube apply --filename "$RUNTIME/streaming-app-$lane.yaml" >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
|
"deployment/nuthatch-$lane" --timeout=180s
|
|
}
|
|
|
|
acquire_fixture_lock
|
|
OWNS_FIXTURE=true
|
|
setup_fixture durable coordination oidc target
|
|
source "$RUNTIME/fixture.env"
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
|
--file "$ROOT/Dockerfile" \
|
|
--tag "$APP_IMAGE" \
|
|
"$ROOT"
|
|
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
|
|
|
while IFS=$'\t' read -r lane coordination oidc target; do
|
|
if [[ -n "${NUTHATCH_STREAMING_LANE:-}" ]] &&
|
|
[[ "$lane" != "$NUTHATCH_STREAMING_LANE" ]]; then
|
|
continue
|
|
fi
|
|
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
|
|
printf 'Running streaming lane %s: coordination=%s oidc=%s target=%s\n' \
|
|
"$lane" "$coordination" "$oidc" "$target"
|
|
|
|
deploy_application "$lane" "$profile" 536870912 536870912
|
|
run_streaming_job "$lane" "$profile" large
|
|
|
|
cleanup_application "$lane"
|
|
scale_coordination "$coordination" 0
|
|
scale_coordination "$coordination" 1
|
|
deploy_application "$lane" "$profile" 1 262144
|
|
run_streaming_job "$lane" "$profile" throttle
|
|
|
|
deploy_application "$lane" "$profile" 65536 65536
|
|
run_streaming_job "$lane" "$profile" refill
|
|
|
|
deploy_application "$lane" "$profile" 536870912 536870912
|
|
scale_coordination "$coordination" 0
|
|
run_streaming_job "$lane" "$profile" unavailable
|
|
cleanup_application "$lane"
|
|
scale_coordination "$coordination" 1
|
|
|
|
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
|
|
for mode in large throttle refill unavailable; do
|
|
cleanup_lane_resources "streaming-$mode-$lane" "$profile"
|
|
done
|
|
if [[ "${NUTHATCH_STREAMING_FIRST_LANE_ONLY:-0}" = 1 ]]; then
|
|
break
|
|
fi
|
|
done < <(expand_fixture_matrix coordination oidc target)
|
|
|
|
SUCCEEDED=true
|
|
printf 'Nuthatch streaming gate passed\n'
|