remora/tests/kind/deployment/run.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
source "$ROOT/tests/kind/fixture/lib.sh"

USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
RELEASE=remora
LANE=1
PROFILE=

cleanup() {
  local status=$?
  trap - EXIT INT TERM
  rm -f \
    "$RUNTIME/deployment-values.yaml" \
    "$RUNTIME/deployment-rendered.yaml" \
    "$RUNTIME/deployment-policy-rendered.yaml" \
    "$RUNTIME/qualification-ingress.yaml"
  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
    printf 'retained context: %s\n' "$CONTEXT"
    exit "$status"
  fi
  if [[ "$OWNS_FIXTURE" = true ]]; then
    destroy_fixture || status=1
  fi
  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
    printf 'user kubeconfig changed unexpectedly\n' >&2
    status=1
  fi
  exit "$status"
}

trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

profile_value() {
  kube --namespace "$FIXTURE_NAMESPACE" get "configmap/$1" \
    --output "jsonpath={.data.$2}"
}

configure_vault_kubernetes_auth() {
  kube apply --filename - <<'EOF'
apiVersion: v1
kind: ServiceAccount
metadata:
  name: vault-reviewer
  namespace: remora-testbed
automountServiceAccountToken: false
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: remora-vault-reviewer
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
  - kind: ServiceAccount
    name: vault-reviewer
    namespace: remora-testbed
---
apiVersion: v1
kind: Secret
metadata:
  name: vault-reviewer-token
  namespace: remora-testbed
  annotations:
    kubernetes.io/service-account.name: vault-reviewer
type: kubernetes.io/service-account-token
EOF
  local deadline=$((SECONDS + 60))
  while ((SECONDS < deadline)); do
    if [[ -n "$(kube --namespace "$FIXTURE_NAMESPACE" get secret/vault-reviewer-token --output 'jsonpath={.data.token}')" ]]; then
      break
    fi
    sleep 1
  done
  kube --namespace "$FIXTURE_NAMESPACE" patch deployment/vault \
    --type strategic \
    --patch '{"spec":{"template":{"spec":{"containers":[{"name":"vault","volumeMounts":[{"name":"reviewer","mountPath":"/reviewer","readOnly":true}]}],"volumes":[{"name":"reviewer","secret":{"secretName":"vault-reviewer-token"}}]}}}}'
  wait_for_deployment vault
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
    sh -ec 'vault auth list -format=json | grep -q "\"remora-kubernetes/\"" || vault auth enable -path=remora-kubernetes kubernetes'
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
    vault write auth/remora-kubernetes/config \
      kubernetes_host=https://kubernetes.default.svc:443 \
      kubernetes_ca_cert=@/reviewer/ca.crt \
      token_reviewer_jwt=@/reviewer/token \
      disable_iss_validation=true >/dev/null
  local api_policy
  local worker_policy
  api_policy='path "secret/data/remora/lane-1/*" { capabilities = ["create", "read", "update"] }
path "secret/metadata/remora/lane-1/*" { capabilities = ["create", "read", "update", "delete", "list"] }
path "secret/delete/remora/lane-1/*" { capabilities = ["update"] }
path "secret/destroy/remora/lane-1/*" { capabilities = ["update"] }'
  worker_policy='path "secret/data/remora/lane-1/*" { capabilities = ["read"] }'
  printf '%s\n' "$api_policy" | kube --namespace "$FIXTURE_NAMESPACE" exec --stdin deployment/vault -- \
    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
    vault policy write remora-api - >/dev/null
  printf '%s\n' "$worker_policy" | kube --namespace "$FIXTURE_NAMESPACE" exec --stdin deployment/vault -- \
    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
    vault policy write remora-worker - >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
    vault write auth/remora-kubernetes/role/remora-api \
      bound_service_account_names=remora-remora-api \
      bound_service_account_namespaces=remora-testbed \
      audience=vault \
      token_policies=remora-api \
      token_ttl=1h >/dev/null
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
    vault write auth/remora-kubernetes/role/remora-worker \
      bound_service_account_names=remora-remora-worker \
      bound_service_account_namespaces=remora-testbed \
      audience=vault \
      token_policies=remora-worker \
      token_ttl=1h >/dev/null
}

create_external_secrets() {
  local database_url
  local coordination_url
  local target_access_key
  local target_secret_key
  database_url="$(profile_value "$PROFILE" DATABASE_URL)"
  coordination_url="$(profile_value "$PROFILE" REMORA_COORDINATION_URL)"
  target_access_key="$(profile_value "$PROFILE" REMORA_TARGET_ACCESS_KEY)"
  target_secret_key="$(profile_value "$PROFILE" REMORA_TARGET_SECRET_KEY)"
  kube --namespace "$FIXTURE_NAMESPACE" delete \
    secret/remora-database \
    secret/remora-coordination \
    secret/remora-web-oidc \
    secret/remora-web-session \
    secret/remora-target \
    --ignore-not-found \
    --wait=true
  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-database \
    --from-literal="url=$database_url"
  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-coordination \
    --from-literal="url=$coordination_url"
  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-web-oidc \
    --from-literal="client-secret=synthetic-web-client-secret"
  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-web-session \
    --from-literal="url=$coordination_url"
  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-target \
    --from-literal="access-key=$target_access_key" \
    --from-literal="secret-key=$target_secret_key"
}

write_helm_values() {
  local coordination_namespace
  local oidc_issuer
  local oidc_client_id
  local oidc_project_id
  local target_endpoint
  local target_bucket
  local target_prefix
  local target_region
  local vault_path
  local public_origin=http://remora.test
  coordination_namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)"
  oidc_issuer="$(profile_value "$PROFILE" REMORA_OIDC_ISSUER)"
  oidc_client_id="$(profile_value "$PROFILE" REMORA_OIDC_CLIENT_ID)"
  oidc_project_id="$(profile_value "$PROFILE" REMORA_OIDC_PROJECT_ID)"
  target_endpoint="$(profile_value "$PROFILE" REMORA_TARGET_ENDPOINT)"
  target_bucket="$(profile_value "$PROFILE" REMORA_TARGET_BUCKET)"
  target_prefix="$(profile_value "$PROFILE" REMORA_TARGET_PREFIX)"
  target_region="$(profile_value "$PROFILE" REMORA_TARGET_REGION)"
  vault_path="$(profile_value "$PROFILE" REMORA_VAULT_PATH)"
  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
    public_origin=http://127.0.0.1:4174
  fi
  cat > "$RUNTIME/deployment-values.yaml" <<EOF
api:
  image: {repository: remora-api, tag: local, digest: "", pullPolicy: Never}
worker:
  image: {repository: remora-worker, tag: local, digest: "", pullPolicy: Never}
web:
  image: {repository: remora-web, tag: local, digest: "", pullPolicy: Never}
  publicOrigin: $public_origin
  oidcClientId: remora-web
podDnsPolicy: None
podDnsConfig:
  nameservers: [10.96.0.53]
  searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
  options:
    - {name: ndots, value: "5"}
coordination:
  namespace: "$coordination_namespace"
oidc:
  issuer: "$oidc_issuer"
  audience: "$oidc_project_id"
  resourceClient: "$oidc_client_id"
  projectRoleClaims: "urn:zitadel:iam:org:project:roles,urn:zitadel:iam:org:project:$oidc_project_id:roles"
target:
  storeId: versitygw
  endpoint: "$target_endpoint"
  bucket: "$target_bucket"
  region: "$target_region"
  defaultPrefix: "$target_prefix"
vault:
  image: "$VAULT_LOCAL_IMAGE"
  address: http://vault:8200
  path: "$vault_path"
  kubernetesAuthMount: auth/remora-kubernetes
runtime:
  eventAllowedOrigins: $public_origin
  recoveryIntervalSeconds: 1
networkPolicy:
  enabled: true
  monitoringNamespaceSelector:
    matchLabels:
      kubernetes.io/metadata.name: remora-testbed
  monitoringPodSelector:
    matchLabels:
      app.kubernetes.io/name: remora-deployment-test
  dnsPodLabels:
    - fixture.remora.dev/component: controlled-dns
  dependencyPodLabels:
    - fixture.remora.dev/component: postgres
    - fixture.remora.dev/component: dragonfly
    - fixture.remora.dev/component: vault
    - fixture.remora.dev/component: zitadel
    - fixture.remora.dev/component: versitygw
  webDependencyPodLabels:
    - fixture.remora.dev/component: dragonfly
    - fixture.remora.dev/component: zitadel
observability:
  serviceMonitor: {enabled: false, interval: 30s}
  prometheusRule: {enabled: false}
EOF
  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
    cat >> "$RUNTIME/deployment-values.yaml" <<'EOF'
ingress:
  enabled: true
  className: qualification-nginx
  host: qualification-ingress
EOF
  fi
}

wait_for_queue_length() {
  local expected="$1"
  local namespace
  local deadline=$((SECONDS + 120))
  namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)"
  while ((SECONDS < deadline)); do
    if [[ "$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/dragonfly -- redis-cli LLEN "${namespace}queue")" = "$expected" ]]; then
      return
    fi
    sleep 1
  done
  printf 'coordination queue did not reach length %s\n' "$expected" >&2
  return 1
}

wait_for_available_replicas() {
  local deployment="$1"
  local expected="$2"
  local deadline=$((SECONDS + 120))
  local available
  while ((SECONDS < deadline)); do
    available="$(kube --namespace "$FIXTURE_NAMESPACE" get "deployment/$deployment" \
      --output 'jsonpath={.status.availableReplicas}')"
    if [[ "${available:-0}" = "$expected" ]]; then
      return
    fi
    sleep 1
  done
  printf '%s did not reach %s available replicas\n' "$deployment" "$expected" >&2
  return 1
}

run_deployment_test() {
  local job="$1"
  local expected_job_id="${2:-}"
  local manifest="$RUNTIME/$job.yaml"
  render_lane_job \
    "$TESTBED/deployment/job.yaml.in" \
    "$manifest" \
    "$job" \
    "$PROFILE" \
    remora-kind-deployment:local
  sed -i "s|@@EXPECTED_JOB_ID@@|$expected_job_id|g" "$manifest"
  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
    sed -i 's|@@INGRESS_URL@@|http://qualification-ingress|g' "$manifest"
  else
    sed -i 's|@@INGRESS_URL@@||g' "$manifest"
  fi
  run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
}

run_system_browser_test() {
  local job=system-browser
  local manifest="$RUNTIME/$job.yaml"
  render_lane_job \
    "$TESTBED/deployment/system-browser-job.yaml.in" \
    "$manifest" \
    "$job" \
    "$PROFILE" \
    remora-kind-web-system:local
  run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
}

install_qualification_ingress() {
  local image
  image="$(load_external_image ingress-nginx "$INGRESS_NGINX_IMAGE")"
  sed "s|@@INGRESS_IMAGE@@|$image|g" \
    "$TESTBED/deployment/qualification-ingress.yaml.in" \
    > "$RUNTIME/qualification-ingress.yaml"
  kube apply --filename "$RUNTIME/qualification-ingress.yaml"
  wait_for_deployment qualification-ingress
}

operational_restore() {
  local database="remora_lane_$LANE"
  local backup="/tmp/remora-qualification.dump"
  local expected_job_id
  expected_job_id="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username remora \
      --dbname "$database" \
      --tuples-only \
      --no-align \
      --command "SELECT job_id FROM jobs WHERE label = 'deployment-coordination-rebuild'")"
  [[ -n "$expected_job_id" ]]

  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=0
  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=300s
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres pg_dump \
      --username remora \
      --dbname "$database" \
      --format custom \
      --file "$backup"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres psql \
      --username remora \
      --dbname postgres \
      --set ON_ERROR_STOP=1 \
      --command "DROP DATABASE $database" \
      --command "CREATE DATABASE $database"
  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
    env PGPASSWORD=synthetic-postgres pg_restore \
      --username remora \
      --dbname "$database" \
      --exit-on-error \
      "$backup"

  kube --namespace "$FIXTURE_NAMESPACE" delete pod \
    --selector app=dragonfly \
    --wait=true
  wait_for_deployment dragonfly
  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=2
  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=600s
  wait_for_queue_length 1
  run_deployment_test deployment-restore "$expected_job_id"

  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=0
  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/versitygw --timeout=300s
  wait_for_available_replicas remora-remora-api 0
  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=1
  wait_for_deployment versitygw
  reset_lane_target_store versitygw "remora-lane-$LANE"
  wait_for_available_replicas remora-remora-api 2
}

rollout_remora() {
  local component
  for component in api worker web; do
    kube --namespace "$FIXTURE_NAMESPACE" rollout status \
      "deployment/remora-remora-$component" \
      --timeout=600s
  done
}

reject_chart_values() {
  local reason="$1"
  shift
  if helm template remora "$ROOT/deploy/charts/remora" \
    --namespace "$FIXTURE_NAMESPACE" \
    "$@" >/dev/null 2>&1; then
    printf 'chart accepted %s\n' "$reason" >&2
    return 1
  fi
}

validate_deployment_assets() {
  local chart="$ROOT/deploy/charts/remora"
  helm lint "$chart"
  umask 077
  helm template remora "$chart" \
    --namespace "$FIXTURE_NAMESPACE" \
    --set observability.prometheusRule.enabled=true \
    > "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q '^kind: NetworkPolicy$' "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q '^kind: ServiceAccount$' "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q '169.254.0.0/16' "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q 'cidr: 2000::/3' "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q 'absent(remora_ready{component="remora-api"})' "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q 'absent(remora_ready{component="remora-worker"})' "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q 'automountServiceAccountToken: false' "$RUNTIME/deployment-policy-rendered.yaml"
  grep -q 'readOnlyRootFilesystem: true' "$RUNTIME/deployment-policy-rendered.yaml"
  reject_chart_values "an unprotected target namespace" --set target.exclusiveWriter=false
  reject_chart_values "an unknown top-level value" --set unexpected=true
  reject_chart_values "an unknown nested value" --set runtime.unexpected=true
  reject_chart_values "a zero lease TTL" --set coordination.leaseTtlSeconds=0
  reject_chart_values "an invalid OIDC issuer" --set-string oidc.issuer=identity.invalid
  reject_chart_values "an invalid target bucket" --set-string target.bucket=two..dots
  reject_chart_values "an out-of-range source port" \
    --set-string outbound.allowedSchemePorts=https:70000
  reject_chart_values "renewal at lease expiry" \
    --set runtime.workerRenewSeconds=30
  reject_chart_values "a shared WebSession credential" \
    --set web.sessionUrlSecret.name=remora-coordination
  reject_chart_values "application-only CIDR reachability" \
    --set-string outbound.allowedCidrs=10.0.0.0/8
  reject_chart_values "an impossible disruption budget" \
    --set podDisruptionBudget.minAvailable=3
  if grep -R -E \
    '(BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|postgres://[^[:space:]]+:[^[:space:]]+@|VAULT_TOKEN[=:][^$])' \
    "$chart" "$ROOT/docs/spec" >/dev/null; then
    printf 'deployment source contains credential material\n' >&2
    return 1
  fi
}

acquire_fixture_lock
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
export REMORA_REAL_NERDCTL
validate_deployment_assets
"$REMORA_REAL_NERDCTL" --namespace default build \
  --file "$ROOT/Dockerfile" \
  --target api \
  --tag remora-api:local \
  "$ROOT"
"$REMORA_REAL_NERDCTL" --namespace default build \
  --file "$ROOT/Dockerfile" \
  --target worker \
  --tag remora-worker:local \
  "$ROOT"
"$REMORA_REAL_NERDCTL" --namespace default build \
  --file "$ROOT/Dockerfile" \
  --target web \
  --tag remora-web:local \
  "$ROOT"
"$REMORA_REAL_NERDCTL" --namespace default build \
  --file "$ROOT/Dockerfile.test" \
  --target kind-deployment-runner \
  --tag remora-kind-deployment:local \
  "$ROOT"
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
  "$REMORA_REAL_NERDCTL" --namespace default build \
    --file "$ROOT/Dockerfile.test" \
    --target kind-web-system-runner \
    --tag remora-kind-web-system:local \
    "$ROOT"
fi

OWNS_FIXTURE=true
setup_fixture durable coordination oidc target secret
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
  install_qualification_ingress
fi
load_local_image remora-api:local
load_local_image remora-worker:local
load_local_image remora-web:local
load_local_image remora-kind-deployment:local
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
  load_local_image remora-kind-web-system:local
fi

PROFILE="$(write_lane_profile "$LANE" dragonfly zitadel versitygw)"
configure_vault_kubernetes_auth
create_external_secrets
write_helm_values
helm upgrade --install "$RELEASE" "$ROOT/deploy/charts/remora" \
  --namespace "$FIXTURE_NAMESPACE" \
  --kubeconfig "$KUBECONFIG_PATH" \
  --kube-context "$CONTEXT" \
  --values "$RUNTIME/deployment-values.yaml" \
  --wait \
  --timeout 10m
rollout_remora

kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=0
kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-worker --timeout=300s
run_deployment_test deployment-smoke
wait_for_queue_length 1

if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
  operational_restore
  run_system_browser_test
else
  kube --namespace "$FIXTURE_NAMESPACE" delete pod \
    --selector app=dragonfly \
    --wait=true
  wait_for_deployment dragonfly
  wait_for_queue_length 1
fi

kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=2
rollout_remora
for component in api worker web; do
  kube --namespace "$FIXTURE_NAMESPACE" rollout restart "deployment/remora-remora-$component"
  kube --namespace "$FIXTURE_NAMESPACE" rollout status \
    "deployment/remora-remora-$component" \
    --timeout=600s
done

helm template remora "$ROOT/deploy/charts/remora" \
  --namespace "$FIXTURE_NAMESPACE" \
  --values "$RUNTIME/deployment-values.yaml" \
  > "$RUNTIME/deployment-rendered.yaml"
kube --namespace "$FIXTURE_NAMESPACE" apply \
  --dry-run=server \
  --filename "$RUNTIME/deployment-rendered.yaml" >/dev/null

SUCCEEDED=true
printf 'deployment smoke, rolling restart, and coordination rebuild passed\n'