nuthatch/tests/kind/streaming-run.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"

USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
APP_IMAGE=nuthatch-streaming:local

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
    printf 'retained context: %s\n' "$CONTEXT"
    exit "$status"
  fi
  if [[ "$OWNS_FIXTURE" = true ]]; then
    destroy_fixture || status=1
  fi
  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
    printf 'user kubeconfig changed unexpectedly\n' >&2
    status=1
  fi
  exit "$status"
}

trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

render_application() {
  local lane="$1"
  local profile_file="$2"
  local rate="$3"
  local burst="$4"
  local pg_conn
  local redis_url
  local oidc_issuer
  local oidc_client_id
  local oidc_additional_audiences
  local oidc_client_secret
  local s3_endpoint
  local s3_region
  local s3_bucket
  local s3_access_key
  local s3_secret_key
  pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
  redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
  oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
  oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
  oidc_additional_audiences="$(
    sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
  )"
  oidc_client_secret="$(<"$profile_file.oidc-secret")"
  s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
  s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
  s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
  s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
  s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
  sed \
    -e "s|@@LANE@@|$lane|g" \
    -e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
    -e "s|@@PG_CONN@@|$pg_conn|g" \
    -e "s|@@REDIS_URL@@|$redis_url|g" \
    -e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
    -e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
    -e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
    -e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
    -e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
    -e "s|@@S3_REGION@@|$s3_region|g" \
    -e "s|@@S3_BUCKET@@|$s3_bucket|g" \
    -e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
    -e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
    "$ROOT/deploy/kubernetes/nuthatch.yaml.in" > "$RUNTIME/streaming-app-$lane.yaml"
  sed -i \
    "/NUTHATCH_S3_PATH_STYLE/a\\  NUTHATCH_MAX_OBJECT_BYTES: \"201326592\"\\n  NUTHATCH_THROTTLE_BYTES_PER_SEC: \"$rate\"\\n  NUTHATCH_THROTTLE_BURST_BYTES: \"$burst\"" \
    "$RUNTIME/streaming-app-$lane.yaml"
  sed -i 's/  replicas: 1/  replicas: 2/' "$RUNTIME/streaming-app-$lane.yaml"
  sed -i \
    's/limits: {cpu: "2", memory: 512Mi}/limits: {cpu: "2", memory: 128Mi}/' \
    "$RUNTIME/streaming-app-$lane.yaml"
  sed -i \
    '/          envFrom:/i\          env:\n            - {name: GHCRTS, value: "-N2 -A1m -M96m"}' \
    "$RUNTIME/streaming-app-$lane.yaml"
}

cleanup_application() {
  local lane="$1"
  kube --namespace "$FIXTURE_NAMESPACE" delete \
    "deployment/nuthatch-$lane" \
    "service/nuthatch-$lane" \
    "configmap/nuthatch-$lane" \
    "secret/nuthatch-$lane" \
    --ignore-not-found >/dev/null
  rm -f "$RUNTIME/streaming-app-$lane.yaml"
}

set_streaming_profile() {
  local profile="$1"
  local mode="$2"
  local throttle_burst=262144
  local profile_file="$RUNTIME/$profile.env"
  if [[ "$mode" = refill ]]; then
    throttle_burst=65536
  fi
  sed -i '/^NUTHATCH_STREAMING_/d' "$profile_file"
  {
    printf 'NUTHATCH_STREAMING_MODE=%s\n' "$mode"
    printf 'NUTHATCH_STREAMING_LARGE_BYTES=167772160\n'
    printf 'NUTHATCH_STREAMING_OVER_LIMIT_BYTES=201326593\n'
    printf 'NUTHATCH_STREAMING_THROTTLE_BURST_BYTES=%s\n' "$throttle_burst"
  } >> "$profile_file"
  kube create configmap "$profile" \
    --namespace "$FIXTURE_NAMESPACE" \
    --from-env-file="$profile_file" \
    --dry-run=client \
    --output yaml |
    kube apply --filename - >/dev/null
}

run_streaming_job() {
  local lane="$1"
  local profile="$2"
  local mode="$3"
  local job="streaming-$mode-$lane"
  set_streaming_profile "$profile" "$mode"
  render_lane_job \
    "$job" \
    "$TESTBED/application-probe-job.yaml.in" \
    "$APP_IMAGE" \
    "$profile"
  sed -i \
    -e "s|@@LANE@@|$lane|g" \
    -e 's|@@COMPONENT@@|streaming|g' \
    -e 's|@@PROBE@@|nuthatch-streaming-probe|g' \
    -e "s|@@PROBE_ARGS@@|\"$mode\"|g" \
    -e 's|@@DEADLINE_SECONDS@@|600|g' \
    "$RUNTIME/$job.yaml"
  run_job_manifest "$job" "$RUNTIME/$job.yaml"
}

scale_coordination() {
  local provider="$1"
  local replicas="$2"
  kube --namespace "$FIXTURE_NAMESPACE" scale \
    "deployment/$provider" \
    --replicas="$replicas" >/dev/null
  if [[ "$replicas" = 0 ]]; then
    kube --namespace "$FIXTURE_NAMESPACE" wait \
      --for=delete \
      "pod" \
      --selector="app=$provider" \
      --timeout=120s >/dev/null
  else
    kube --namespace "$FIXTURE_NAMESPACE" rollout status \
      "deployment/$provider" \
      --timeout=180s >/dev/null
  fi
}

deploy_application() {
  local lane="$1"
  local profile="$2"
  local rate="$3"
  local burst="$4"
  cleanup_application "$lane"
  render_application "$lane" "$RUNTIME/$profile.env" "$rate" "$burst"
  kube apply --filename "$RUNTIME/streaming-app-$lane.yaml" >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" rollout status \
    "deployment/nuthatch-$lane" --timeout=180s
}

acquire_fixture_lock
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target
source "$RUNTIME/fixture.env"
"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
  --file "$ROOT/Dockerfile" \
  --target test-runtime \
  --tag "$APP_IMAGE" \
  "$ROOT"
load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"

while IFS=$'\t' read -r lane coordination oidc target; do
  if [[ -n "${NUTHATCH_STREAMING_LANE:-}" ]] &&
     [[ "$lane" != "$NUTHATCH_STREAMING_LANE" ]]; then
    continue
  fi
  profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
  printf 'Running streaming lane %s: coordination=%s oidc=%s target=%s\n' \
    "$lane" "$coordination" "$oidc" "$target"

  deploy_application "$lane" "$profile" 536870912 536870912
  run_streaming_job "$lane" "$profile" large

  cleanup_application "$lane"
  scale_coordination "$coordination" 0
  scale_coordination "$coordination" 1
  deploy_application "$lane" "$profile" 1 262144
  run_streaming_job "$lane" "$profile" throttle

  cleanup_application "$lane"
  scale_coordination "$coordination" 0
  scale_coordination "$coordination" 1
  deploy_application "$lane" "$profile" 1 65536
  run_streaming_job "$lane" "$profile" refill

  deploy_application "$lane" "$profile" 536870912 536870912
  scale_coordination "$coordination" 0
  run_streaming_job "$lane" "$profile" unavailable
  cleanup_application "$lane"
  scale_coordination "$coordination" 1

  cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
  for mode in large throttle refill unavailable; do
    cleanup_lane_resources "streaming-$mode-$lane" "$profile"
  done
  if [[ "${NUTHATCH_STREAMING_FIRST_LANE_ONLY:-0}" = 1 ]]; then
    break
  fi
done < <(expand_fixture_matrix coordination oidc target)

SUCCEEDED=true
printf 'Nuthatch streaming gate passed\n'