nuthatch/tests/kind/idempotency-run.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"

USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-idempotency:local

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
    printf 'retained context: %s\n' "$CONTEXT"
    exit "$status"
  fi
  if [[ "$OWNS_FIXTURE" = true ]]; then
    destroy_fixture || status=1
  fi
  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
    printf 'user kubeconfig changed unexpectedly\n' >&2
    status=1
  fi
  exit "$status"
}

trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

render_application() {
  local lane="$1"
  local profile_file="$2"
  local pg_conn
  local redis_url
  local oidc_issuer
  local oidc_client_id
  local oidc_additional_audiences
  local oidc_client_secret
  local s3_endpoint
  local s3_region
  local s3_bucket
  local s3_access_key
  local s3_secret_key
  pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
  redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
  oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
  oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
  oidc_additional_audiences="$(
    sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
  )"
  oidc_client_secret="$(<"$profile_file.oidc-secret")"
  s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
  s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
  s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
  s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
  s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
  sed \
    -e "s|@@LANE@@|$lane|g" \
    -e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
    -e "s|@@PG_CONN@@|$pg_conn|g" \
    -e "s|@@REDIS_URL@@|$redis_url|g" \
    -e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
    -e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
    -e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
    -e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
    -e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
    -e "s|@@S3_REGION@@|$s3_region|g" \
    -e "s|@@S3_BUCKET@@|$s3_bucket|g" \
    -e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
    -e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
    "$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/idempotency-app-$lane.yaml"
  sed -i \
    's/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "2"/' \
    "$RUNTIME/idempotency-app-$lane.yaml"
  sed -i 's/  replicas: 1/  replicas: 2/' "$RUNTIME/idempotency-app-$lane.yaml"
}

cleanup_application() {
  local lane="$1"
  kube --namespace "$FIXTURE_NAMESPACE" delete \
    "deployment/nuthatch-$lane" \
    "service/nuthatch-$lane" \
    service/nuthatch-idempotency-a \
    service/nuthatch-idempotency-b \
    "configmap/nuthatch-$lane" \
    "secret/nuthatch-$lane" \
    --ignore-not-found >/dev/null
  rm -f "$RUNTIME/idempotency-app-$lane.yaml"
}

create_process_services() {
  local lane="$1"
  local profile="$2"
  local pods
  mapfile -t pods < <(
    kube --namespace "$FIXTURE_NAMESPACE" get pods \
      --selector="app=nuthatch-$lane" \
      --field-selector=status.phase=Running \
      --output jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
      sort
  )
  test "${#pods[@]}" = 2
  kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[0]}" \
    nuthatch-idempotency-process=a >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[1]}" \
    nuthatch-idempotency-process=b >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[0]}" \
    --name=nuthatch-idempotency-a \
    --port=8080 \
    --target-port=8080 >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[1]}" \
    --name=nuthatch-idempotency-b \
    --port=8080 \
    --target-port=8080 >/dev/null
  local ip_a
  local ip_b
  ip_a="$(
    kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-a \
      --output jsonpath='{.spec.clusterIP}'
  )"
  ip_b="$(
    kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-idempotency-b \
      --output jsonpath='{.spec.clusterIP}'
  )"
  local profile_file="$RUNTIME/$profile.env"
  sed -i '/^NUTHATCH_IDEMPOTENCY_/d' "$profile_file"
  {
    printf 'NUTHATCH_IDEMPOTENCY_MODE=verify\n'
    printf 'NUTHATCH_IDEMPOTENCY_IP_A=%s\n' "$ip_a"
    printf 'NUTHATCH_IDEMPOTENCY_IP_B=%s\n' "$ip_b"
  } >>"$profile_file"
  kube create configmap "$profile" \
    --namespace "$FIXTURE_NAMESPACE" \
    --from-env-file="$profile_file" \
    --dry-run=client \
    --output yaml |
    kube apply --filename - >/dev/null
}

acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
  --file "$ROOT/Dockerfile" \
  --tag "$APP_IMAGE" \
  "$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"

while IFS=$'\t' read -r lane coordination oidc target; do
  if [[ -n "${NUTHATCH_IDEMPOTENCY_LANE:-}" ]] &&
     [[ "$lane" != "$NUTHATCH_IDEMPOTENCY_LANE" ]]; then
    continue
  fi
  profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
  printf 'Running idempotency lane %s: coordination=%s oidc=%s target=%s\n' \
    "$lane" "$coordination" "$oidc" "$target"

  cleanup_application "$lane"
  render_application "$lane" "$RUNTIME/$profile.env"
  kube apply --filename "$RUNTIME/idempotency-app-$lane.yaml" >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" rollout status \
    "deployment/nuthatch-$lane" --timeout=180s
  create_process_services "$lane" "$profile"

  job="idempotency-epoch-$lane"
  render_lane_job \
    "$job" \
    "$TESTBED/product-matrix-job.yaml.in" \
    "$CURL_LOCAL_IMAGE" \
    "$profile"
  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
  run_job_manifest "$job" "$RUNTIME/$job.yaml"

  cleanup_application "$lane"
  cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
  cleanup_lane_resources "$job" "$profile"
  if [[ "${NUTHATCH_IDEMPOTENCY_FIRST_LANE_ONLY:-0}" = 1 ]]; then
    break
  fi
done < <(expand_fixture_matrix coordination oidc target)

SUCCEEDED=true
printf 'Nuthatch idempotency gate passed\n'