Ship production deployment and runtime observability

tests/kind/dependencies.yaml.in

@@ -27,7 +27,7 @@ spec:
     matchLabels: {app: postgres}
   template:
     metadata:
-      labels: {app: postgres}
+      labels: {app: postgres, fixture.remora.dev/component: postgres}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -80,7 +80,7 @@ spec:
     matchLabels: {app: redis}
   template:
     metadata:
-      labels: {app: redis}
+      labels: {app: redis, fixture.remora.dev/component: redis}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -122,7 +122,7 @@ spec:
     matchLabels: {app: dragonfly}
   template:
     metadata:
-      labels: {app: dragonfly}
+      labels: {app: dragonfly, fixture.remora.dev/component: dragonfly}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -163,7 +163,7 @@ spec:
     matchLabels: {app: vault}
   template:
     metadata:
-      labels: {app: vault}
+      labels: {app: vault, fixture.remora.dev/component: vault}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -209,7 +209,7 @@ spec:
     matchLabels: {app: keycloak}
   template:
     metadata:
-      labels: {app: keycloak}
+      labels: {app: keycloak, fixture.remora.dev/component: keycloak}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -260,7 +260,7 @@ spec:
     matchLabels: {app: zitadel}
   template:
     metadata:
-      labels: {app: zitadel}
+      labels: {app: zitadel, fixture.remora.dev/component: zitadel}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -358,7 +358,7 @@ spec:
     matchLabels: {app: garage}
   template:
     metadata:
-      labels: {app: garage}
+      labels: {app: garage, fixture.remora.dev/component: garage}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -426,7 +426,7 @@ spec:
     matchLabels: {app: versitygw}
   template:
     metadata:
-      labels: {app: versitygw}
+      labels: {app: versitygw, fixture.remora.dev/component: versitygw}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false
@@ -510,7 +510,7 @@ spec:
     matchLabels: {app: controlled-dns}
   template:
     metadata:
-      labels: {app: controlled-dns}
+      labels: {app: controlled-dns, fixture.remora.dev/component: controlled-dns}
     spec:
       automountServiceAccountToken: false
       enableServiceLinks: false

tests/kind/deployment-job.yaml.in

@@ -0,0 +1,61 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+  labels:
+    app.kubernetes.io/name: remora-deployment-test
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 300
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: remora-deployment-test
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      initContainers:
+        - name: smoke
+          image: @@CURL_IMAGE@@
+          imagePullPolicy: Never
+          command: [sh, -ec]
+          args:
+            - |
+              retry="--retry 30 --retry-all-errors --retry-delay 1"
+              test "$(curl $retry --silent --output /dev/null --write-out '%{http_code}' http://remora-remora-api-metrics:9090/health/ready)" = 200
+              test "$(curl $retry --silent --output /dev/null --write-out '%{http_code}' http://remora-remora-web/health/ready)" = 200
+              curl $retry --fail --silent http://remora-remora-api-metrics:9090/metrics | grep -q remora_background_runs_total
+              curl $retry --fail --silent http://remora-remora-web/runtime-config.js | grep -q '"oidcClientId":"remora-web"'
+              test "$(curl $retry --silent --output /dev/null --write-out '%{http_code}' http://remora-remora/v1/jobs)" = 401
+              if curl --connect-timeout 2 --max-time 5 --insecure --silent --fail https://remora-remora-worker:8443/internal/v1/jobs/x/claim >/dev/null 2>&1; then
+                exit 1
+              fi
+          resources:
+            requests: {cpu: 10m, memory: 16Mi}
+            limits: {cpu: 250m, memory: 64Mi}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef:
+                name: @@PROFILE_NAME@@
+          env:
+            - {name: REMORA_DEPLOYMENT_API_URL, value: "http://remora-remora/v1"}
+          volumeMounts:
+            - {name: profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: 100m, memory: 128Mi}
+            limits: {cpu: "2", memory: 1Gi}
+      volumes:
+        - name: profile
+          configMap:
+            name: @@PROFILE_NAME@@-file

tests/kind/run-deployment.sh

@@ -0,0 +1,367 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+RELEASE=remora
+LANE=1
+PROFILE=
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  rm -f \
+    "$RUNTIME/deployment-values.yaml" \
+    "$RUNTIME/deployment-rendered.yaml" \
+    "$RUNTIME/deployment-policy-rendered.yaml"
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+profile_value() {
+  kube --namespace "$FIXTURE_NAMESPACE" get "configmap/$1" \
+    --output "jsonpath={.data.$2}"
+}
+
+configure_vault_kubernetes_auth() {
+  kube apply --filename - <<'EOF'
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: vault-reviewer
+  namespace: remora-testbed
+automountServiceAccountToken: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: remora-vault-reviewer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+  - kind: ServiceAccount
+    name: vault-reviewer
+    namespace: remora-testbed
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vault-reviewer-token
+  namespace: remora-testbed
+  annotations:
+    kubernetes.io/service-account.name: vault-reviewer
+type: kubernetes.io/service-account-token
+EOF
+  local deadline=$((SECONDS + 60))
+  while ((SECONDS < deadline)); do
+    if [[ -n "$(kube --namespace "$FIXTURE_NAMESPACE" get secret/vault-reviewer-token --output 'jsonpath={.data.token}')" ]]; then
+      break
+    fi
+    sleep 1
+  done
+  kube --namespace "$FIXTURE_NAMESPACE" patch deployment/vault \
+    --type strategic \
+    --patch '{"spec":{"template":{"spec":{"containers":[{"name":"vault","volumeMounts":[{"name":"reviewer","mountPath":"/reviewer","readOnly":true}]}],"volumes":[{"name":"reviewer","secret":{"secretName":"vault-reviewer-token"}}]}}}}'
+  wait_for_deployment vault
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
+    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
+    sh -ec 'vault auth list -format=json | grep -q "\"remora-kubernetes/\"" || vault auth enable -path=remora-kubernetes kubernetes'
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
+    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
+    vault write auth/remora-kubernetes/config \
+      kubernetes_host=https://kubernetes.default.svc:443 \
+      kubernetes_ca_cert=@/reviewer/ca.crt \
+      token_reviewer_jwt=@/reviewer/token \
+      disable_iss_validation=true >/dev/null
+  local api_policy
+  local worker_policy
+  api_policy='path "secret/data/remora/lane-1/*" { capabilities = ["create", "read", "update"] }
+path "secret/metadata/remora/lane-1/*" { capabilities = ["create", "read", "update", "delete", "list"] }
+path "secret/delete/remora/lane-1/*" { capabilities = ["update"] }
+path "secret/destroy/remora/lane-1/*" { capabilities = ["update"] }'
+  worker_policy='path "secret/data/remora/lane-1/*" { capabilities = ["read"] }'
+  printf '%s\n' "$api_policy" | kube --namespace "$FIXTURE_NAMESPACE" exec --stdin deployment/vault -- \
+    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
+    vault policy write remora-api - >/dev/null
+  printf '%s\n' "$worker_policy" | kube --namespace "$FIXTURE_NAMESPACE" exec --stdin deployment/vault -- \
+    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
+    vault policy write remora-worker - >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
+    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
+    vault write auth/remora-kubernetes/role/remora-api \
+      bound_service_account_names=remora-remora-api \
+      bound_service_account_namespaces=remora-testbed \
+      audience=vault \
+      token_policies=remora-api \
+      token_ttl=1h >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \
+    env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \
+    vault write auth/remora-kubernetes/role/remora-worker \
+      bound_service_account_names=remora-remora-worker \
+      bound_service_account_namespaces=remora-testbed \
+      audience=vault \
+      token_policies=remora-worker \
+      token_ttl=1h >/dev/null
+}
+
+create_external_secrets() {
+  local database_url
+  local coordination_url
+  local target_access_key
+  local target_secret_key
+  database_url="$(profile_value "$PROFILE" DATABASE_URL)"
+  coordination_url="$(profile_value "$PROFILE" REMORA_COORDINATION_URL)"
+  target_access_key="$(profile_value "$PROFILE" REMORA_TARGET_ACCESS_KEY)"
+  target_secret_key="$(profile_value "$PROFILE" REMORA_TARGET_SECRET_KEY)"
+  kube --namespace "$FIXTURE_NAMESPACE" delete \
+    secret/remora-database \
+    secret/remora-coordination \
+    secret/remora-target \
+    --ignore-not-found \
+    --wait=true
+  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-database \
+    --from-literal="url=$database_url"
+  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-coordination \
+    --from-literal="url=$coordination_url"
+  kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-target \
+    --from-literal="access-key=$target_access_key" \
+    --from-literal="secret-key=$target_secret_key"
+}
+
+write_helm_values() {
+  local coordination_namespace
+  local oidc_issuer
+  local oidc_client_id
+  local oidc_project_id
+  local target_endpoint
+  local target_bucket
+  local target_prefix
+  local target_region
+  local vault_path
+  coordination_namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)"
+  oidc_issuer="$(profile_value "$PROFILE" REMORA_OIDC_ISSUER)"
+  oidc_client_id="$(profile_value "$PROFILE" REMORA_OIDC_CLIENT_ID)"
+  oidc_project_id="$(profile_value "$PROFILE" REMORA_OIDC_PROJECT_ID)"
+  target_endpoint="$(profile_value "$PROFILE" REMORA_TARGET_ENDPOINT)"
+  target_bucket="$(profile_value "$PROFILE" REMORA_TARGET_BUCKET)"
+  target_prefix="$(profile_value "$PROFILE" REMORA_TARGET_PREFIX)"
+  target_region="$(profile_value "$PROFILE" REMORA_TARGET_REGION)"
+  vault_path="$(profile_value "$PROFILE" REMORA_VAULT_PATH)"
+  cat > "$RUNTIME/deployment-values.yaml" <<EOF
+api:
+  image: {repository: remora-api, tag: wp25, digest: "", pullPolicy: Never}
+worker:
+  image: {repository: remora-worker, tag: wp25, digest: "", pullPolicy: Never}
+web:
+  image: {repository: remora-web, tag: wp25, digest: "", pullPolicy: Never}
+  publicOrigin: http://remora.test
+  apiUrl: /v1/
+  oidcClientId: remora-web
+  cspConnectSrc: "'self' $oidc_issuer"
+podDnsPolicy: None
+podDnsConfig:
+  nameservers: [10.96.0.53]
+  searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+  options:
+    - {name: ndots, value: "5"}
+coordination:
+  namespace: "$coordination_namespace"
+oidc:
+  issuer: "$oidc_issuer"
+  audience: "$oidc_project_id"
+  resourceClient: "$oidc_client_id"
+  projectRoleClaims: "urn:zitadel:iam:org:project:roles,urn:zitadel:iam:org:project:$oidc_project_id:roles"
+target:
+  storeId: versitygw
+  endpoint: "$target_endpoint"
+  bucket: "$target_bucket"
+  region: "$target_region"
+  defaultPrefix: "$target_prefix"
+vault:
+  image: "$VAULT_LOCAL_IMAGE"
+  address: http://vault:8200
+  path: "$vault_path"
+  kubernetesAuthMount: auth/remora-kubernetes
+runtime:
+  eventAllowedOrigins: http://remora.test
+  recoveryIntervalSeconds: 1
+networkPolicy:
+  enabled: true
+  monitoringNamespaceSelector:
+    matchLabels:
+      kubernetes.io/metadata.name: remora-testbed
+  monitoringPodSelector:
+    matchLabels:
+      app.kubernetes.io/name: remora-deployment-test
+  dnsPodLabels:
+    - fixture.remora.dev/component: controlled-dns
+  dependencyPodLabels:
+    - fixture.remora.dev/component: postgres
+    - fixture.remora.dev/component: dragonfly
+    - fixture.remora.dev/component: vault
+    - fixture.remora.dev/component: zitadel
+    - fixture.remora.dev/component: versitygw
+observability:
+  serviceMonitor: {enabled: false, interval: 30s}
+  prometheusRule: {enabled: false}
+EOF
+}
+
+wait_for_queue_length() {
+  local expected="$1"
+  local namespace
+  local deadline=$((SECONDS + 120))
+  namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)"
+  while ((SECONDS < deadline)); do
+    if [[ "$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/dragonfly -- redis-cli LLEN "${namespace}queue")" = "$expected" ]]; then
+      return
+    fi
+    sleep 1
+  done
+  printf 'coordination queue did not reach length %s\n' "$expected" >&2
+  return 1
+}
+
+rollout_remora() {
+  local component
+  for component in api worker web; do
+    kube --namespace "$FIXTURE_NAMESPACE" rollout status \
+      "deployment/remora-remora-$component" \
+      --timeout=600s
+  done
+}
+
+validate_deployment_assets() {
+  local chart="$ROOT/deploy/charts/remora"
+  helm lint "$chart"
+  umask 077
+  helm template remora "$chart" \
+    --namespace "$FIXTURE_NAMESPACE" \
+    --set observability.prometheusRule.enabled=true \
+    > "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q '^kind: NetworkPolicy$' "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q '^kind: ServiceAccount$' "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q '169.254.0.0/16' "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q 'cidr: 2000::/3' "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q 'absent(remora_ready{component="remora-api"})' "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q 'absent(remora_ready{component="remora-worker"})' "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q 'automountServiceAccountToken: false' "$RUNTIME/deployment-policy-rendered.yaml"
+  grep -q 'readOnlyRootFilesystem: true' "$RUNTIME/deployment-policy-rendered.yaml"
+  if helm template remora "$chart" \
+    --namespace "$FIXTURE_NAMESPACE" \
+    --set target.exclusiveWriter=false >/dev/null 2>&1; then
+    printf 'chart accepted an unprotected target namespace\n' >&2
+    return 1
+  fi
+  if grep -R -E \
+    '(BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|postgres://[^[:space:]]+:[^[:space:]]+@|VAULT_TOKEN[=:][^$])' \
+    "$chart" "$ROOT/docs/spec" >/dev/null; then
+    printf 'deployment source contains credential material\n' >&2
+    return 1
+  fi
+}
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+validate_deployment_assets
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile" \
+  --target api \
+  --tag remora-api:wp25 \
+  "$ROOT"
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile" \
+  --target worker \
+  --tag remora-worker:wp25 \
+  "$ROOT"
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile" \
+  --target web \
+  --tag remora-web:wp25 \
+  "$ROOT"
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-deployment-runner \
+  --tag remora-kind-deployment:local \
+  "$ROOT"
+
+OWNS_FIXTURE=true
+setup_fixture durable coordination oidc target secret
+load_local_image remora-api:wp25
+load_local_image remora-worker:wp25
+load_local_image remora-web:wp25
+load_local_image remora-kind-deployment:local
+
+PROFILE="$(write_lane_profile "$LANE" dragonfly zitadel versitygw)"
+configure_vault_kubernetes_auth
+create_external_secrets
+write_helm_values
+helm upgrade --install "$RELEASE" "$ROOT/deploy/charts/remora" \
+  --namespace "$FIXTURE_NAMESPACE" \
+  --kubeconfig "$KUBECONFIG_PATH" \
+  --kube-context "$CONTEXT" \
+  --values "$RUNTIME/deployment-values.yaml" \
+  --wait \
+  --timeout 10m
+rollout_remora
+
+kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=0
+kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-worker --timeout=300s
+JOB=deployment-smoke
+MANIFEST="$RUNTIME/$JOB.yaml"
+render_lane_job \
+  "$TESTBED/deployment-job.yaml.in" \
+  "$MANIFEST" \
+  "$JOB" \
+  "$PROFILE" \
+  remora-kind-deployment:local
+run_job_manifest "$FIXTURE_NAMESPACE" "$JOB" "$MANIFEST"
+wait_for_queue_length 1
+
+kube --namespace "$FIXTURE_NAMESPACE" delete pod \
+  --selector app=dragonfly \
+  --wait=true
+wait_for_deployment dragonfly
+wait_for_queue_length 1
+
+kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=2
+rollout_remora
+for component in api worker web; do
+  kube --namespace "$FIXTURE_NAMESPACE" rollout restart "deployment/remora-remora-$component"
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status \
+    "deployment/remora-remora-$component" \
+    --timeout=600s
+done
+
+helm template remora "$ROOT/deploy/charts/remora" \
+  --namespace "$FIXTURE_NAMESPACE" \
+  --values "$RUNTIME/deployment-values.yaml" \
+  > "$RUNTIME/deployment-rendered.yaml"
+kube --namespace "$FIXTURE_NAMESPACE" apply \
+  --dry-run=server \
+  --filename "$RUNTIME/deployment-rendered.yaml" >/dev/null
+
+SUCCEEDED=true
+printf 'deployment smoke, rolling restart, and coordination rebuild passed\n'