195 lines
6.5 KiB
Shell
Executable file
195 lines
6.5 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
source "$ROOT/tests/kind/lib.sh"
|
|
|
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
|
SUCCEEDED=false
|
|
OWNS_FIXTURE=false
|
|
APP_IMAGE=nuthatch-idempotency:local
|
|
|
|
cleanup() {
|
|
local status=$?
|
|
trap - EXIT INT TERM
|
|
if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
|
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
|
printf 'retained context: %s\n' "$CONTEXT"
|
|
exit "$status"
|
|
fi
|
|
if [[ "$OWNS_FIXTURE" = true ]]; then
|
|
destroy_fixture || status=1
|
|
fi
|
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
|
status=1
|
|
fi
|
|
exit "$status"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
render_application() {
|
|
local lane="$1"
|
|
local profile_file="$2"
|
|
local pg_conn
|
|
local redis_url
|
|
local oidc_issuer
|
|
local oidc_client_id
|
|
local oidc_additional_audiences
|
|
local oidc_client_secret
|
|
local s3_endpoint
|
|
local s3_region
|
|
local s3_bucket
|
|
local s3_access_key
|
|
local s3_secret_key
|
|
pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
|
|
redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
|
|
oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
|
|
oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
|
|
oidc_additional_audiences="$(
|
|
sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
|
|
)"
|
|
oidc_client_secret="$(<"$profile_file.oidc-secret")"
|
|
s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
|
|
s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
|
|
s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
|
|
s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
|
|
s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
|
|
sed \
|
|
-e "s|@@LANE@@|$lane|g" \
|
|
-e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
|
|
-e "s|@@PG_CONN@@|$pg_conn|g" \
|
|
-e "s|@@REDIS_URL@@|$redis_url|g" \
|
|
-e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
|
|
-e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
|
|
-e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
|
|
-e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
|
|
-e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
|
|
-e "s|@@S3_REGION@@|$s3_region|g" \
|
|
-e "s|@@S3_BUCKET@@|$s3_bucket|g" \
|
|
-e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
|
|
-e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
|
|
"$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/idempotency-app-$lane.yaml"
|
|
sed -i \
|
|
's/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "2"/' \
|
|
"$RUNTIME/idempotency-app-$lane.yaml"
|
|
sed -i 's/ replicas: 1/ replicas: 2/' "$RUNTIME/idempotency-app-$lane.yaml"
|
|
}
|
|
|
|
cleanup_application() {
|
|
local lane="$1"
|
|
kube --namespace "$FIXTURE_NAMESPACE" delete \
|
|
"deployment/nuthatch-$lane" \
|
|
"service/nuthatch-$lane" \
|
|
service/nuthatch-idempotency-a \
|
|
service/nuthatch-idempotency-b \
|
|
"configmap/nuthatch-$lane" \
|
|
"secret/nuthatch-$lane" \
|
|
--ignore-not-found >/dev/null
|
|
rm -f "$RUNTIME/idempotency-app-$lane.yaml"
|
|
}
|
|
|
|
create_process_services() {
|
|
local lane="$1"
|
|
local profile="$2"
|
|
local pods
|
|
mapfile -t pods < <(
|
|
kube --namespace "$FIXTURE_NAMESPACE" get pods \
|
|
--selector="app=nuthatch-$lane" \
|
|
--field-selector=status.phase=Running \
|
|
--output jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
|
|
sort
|
|
)
|
|
test "${#pods[@]}" = 2
|
|
kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[0]}" \
|
|
nuthatch-idempotency-process=a >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[1]}" \
|
|
nuthatch-idempotency-process=b >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[0]}" \
|
|
--name=nuthatch-idempotency-a \
|
|
--port=8080 \
|
|
--target-port=8080 >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[1]}" \
|
|
--name=nuthatch-idempotency-b \
|
|
--port=8080 \
|
|
--target-port=8080 >/dev/null
|
|
local ip_a
|
|
local ip_b
|
|
ip_a="$(
|
|
kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-a \
|
|
--output jsonpath='{.spec.clusterIP}'
|
|
)"
|
|
ip_b="$(
|
|
kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-b \
|
|
--output jsonpath='{.spec.clusterIP}'
|
|
)"
|
|
local profile_file="$RUNTIME/$profile.env"
|
|
sed -i '/^NUTHATCH_IDEMPOTENCY_/d' "$profile_file"
|
|
{
|
|
printf 'NUTHATCH_IDEMPOTENCY_MODE=verify\n'
|
|
printf 'NUTHATCH_IDEMPOTENCY_IP_A=%s\n' "$ip_a"
|
|
printf 'NUTHATCH_IDEMPOTENCY_IP_B=%s\n' "$ip_b"
|
|
} >>"$profile_file"
|
|
kube create configmap "$profile" \
|
|
--namespace "$FIXTURE_NAMESPACE" \
|
|
--from-env-file="$profile_file" \
|
|
--dry-run=client \
|
|
--output yaml |
|
|
kube apply --filename - >/dev/null
|
|
}
|
|
|
|
acquire_fixture_lock
|
|
OWNS_FIXTURE=true
|
|
setup_fixture durable coordination oidc target
|
|
source "$RUNTIME/fixture.env"
|
|
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
|
|
--file "$ROOT/Dockerfile" \
|
|
--target test-runtime \
|
|
--tag "$APP_IMAGE" \
|
|
"$ROOT"
|
|
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
|
|
|
|
while IFS=$'\t' read -r lane coordination oidc target; do
|
|
if [[ -n "${NUTHATCH_IDEMPOTENCY_LANE:-}" ]] &&
|
|
[[ "$lane" != "$NUTHATCH_IDEMPOTENCY_LANE" ]]; then
|
|
continue
|
|
fi
|
|
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
|
|
printf 'Running idempotency lane %s: coordination=%s oidc=%s target=%s\n' \
|
|
"$lane" "$coordination" "$oidc" "$target"
|
|
|
|
cleanup_application "$lane"
|
|
render_application "$lane" "$RUNTIME/$profile.env"
|
|
kube apply --filename "$RUNTIME/idempotency-app-$lane.yaml" >/dev/null
|
|
kube --namespace "$FIXTURE_NAMESPACE" rollout status \
|
|
"deployment/nuthatch-$lane" --timeout=180s
|
|
create_process_services "$lane" "$profile"
|
|
|
|
job="idempotency-epoch-$lane"
|
|
render_lane_job \
|
|
"$job" \
|
|
"$TESTBED/application-probe-job.yaml.in" \
|
|
"$APP_IMAGE" \
|
|
"$profile"
|
|
sed -i \
|
|
-e "s|@@LANE@@|$lane|g" \
|
|
-e 's|@@COMPONENT@@|idempotency|g' \
|
|
-e 's|@@PROBE@@|nuthatch-idempotency-probe|g' \
|
|
-e 's|@@PROBE_ARGS@@||g' \
|
|
-e 's|@@DEADLINE_SECONDS@@|300|g' \
|
|
"$RUNTIME/$job.yaml"
|
|
run_job_manifest "$job" "$RUNTIME/$job.yaml"
|
|
|
|
cleanup_application "$lane"
|
|
cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
|
|
cleanup_lane_resources "$job" "$profile"
|
|
if [[ "${NUTHATCH_IDEMPOTENCY_FIRST_LANE_ONLY:-0}" = 1 ]]; then
|
|
break
|
|
fi
|
|
done < <(expand_fixture_matrix coordination oidc target)
|
|
|
|
SUCCEEDED=true
|
|
printf 'Nuthatch idempotency gate passed\n'
|