#!/usr/bin/env bash set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" source "$ROOT/tests/kind/lib.sh" USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)" SUCCEEDED=false OWNS_FIXTURE=false RELEASE=remora LANE=1 PROFILE= cleanup() { local status=$? trap - EXIT INT TERM rm -f \ "$RUNTIME/deployment-values.yaml" \ "$RUNTIME/deployment-rendered.yaml" \ "$RUNTIME/deployment-policy-rendered.yaml" if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH" printf 'retained context: %s\n' "$CONTEXT" exit "$status" fi if [[ "$OWNS_FIXTURE" = true ]]; then destroy_fixture || status=1 fi if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then printf 'user kubeconfig changed unexpectedly\n' >&2 status=1 fi exit "$status" } trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM profile_value() { kube --namespace "$FIXTURE_NAMESPACE" get "configmap/$1" \ --output "jsonpath={.data.$2}" } configure_vault_kubernetes_auth() { kube apply --filename - <<'EOF' apiVersion: v1 kind: ServiceAccount metadata: name: vault-reviewer namespace: remora-testbed automountServiceAccountToken: false --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: remora-vault-reviewer roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount name: vault-reviewer namespace: remora-testbed --- apiVersion: v1 kind: Secret metadata: name: vault-reviewer-token namespace: remora-testbed annotations: kubernetes.io/service-account.name: vault-reviewer type: kubernetes.io/service-account-token EOF local deadline=$((SECONDS + 60)) while ((SECONDS < deadline)); do if [[ -n "$(kube --namespace "$FIXTURE_NAMESPACE" get secret/vault-reviewer-token --output 'jsonpath={.data.token}')" ]]; then break fi sleep 1 done kube --namespace "$FIXTURE_NAMESPACE" patch deployment/vault \ --type strategic \ --patch '{"spec":{"template":{"spec":{"containers":[{"name":"vault","volumeMounts":[{"name":"reviewer","mountPath":"/reviewer","readOnly":true}]}],"volumes":[{"name":"reviewer","secret":{"secretName":"vault-reviewer-token"}}]}}}}' wait_for_deployment vault kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \ env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \ sh -ec 'vault auth list -format=json | grep -q "\"remora-kubernetes/\"" || vault auth enable -path=remora-kubernetes kubernetes' kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \ env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \ vault write auth/remora-kubernetes/config \ kubernetes_host=https://kubernetes.default.svc:443 \ kubernetes_ca_cert=@/reviewer/ca.crt \ token_reviewer_jwt=@/reviewer/token \ disable_iss_validation=true >/dev/null local api_policy local worker_policy api_policy='path "secret/data/remora/lane-1/*" { capabilities = ["create", "read", "update"] } path "secret/metadata/remora/lane-1/*" { capabilities = ["create", "read", "update", "delete", "list"] } path "secret/delete/remora/lane-1/*" { capabilities = ["update"] } path "secret/destroy/remora/lane-1/*" { capabilities = ["update"] }' worker_policy='path "secret/data/remora/lane-1/*" { capabilities = ["read"] }' printf '%s\n' "$api_policy" | kube --namespace "$FIXTURE_NAMESPACE" exec --stdin deployment/vault -- \ env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \ vault policy write remora-api - >/dev/null printf '%s\n' "$worker_policy" | kube --namespace "$FIXTURE_NAMESPACE" exec --stdin deployment/vault -- \ env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \ vault policy write remora-worker - >/dev/null kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \ env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \ vault write auth/remora-kubernetes/role/remora-api \ bound_service_account_names=remora-remora-api \ bound_service_account_namespaces=remora-testbed \ audience=vault \ token_policies=remora-api \ token_ttl=1h >/dev/null kube --namespace "$FIXTURE_NAMESPACE" exec deployment/vault -- \ env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN=synthetic-vault-token \ vault write auth/remora-kubernetes/role/remora-worker \ bound_service_account_names=remora-remora-worker \ bound_service_account_namespaces=remora-testbed \ audience=vault \ token_policies=remora-worker \ token_ttl=1h >/dev/null } create_external_secrets() { local database_url local coordination_url local target_access_key local target_secret_key database_url="$(profile_value "$PROFILE" DATABASE_URL)" coordination_url="$(profile_value "$PROFILE" REMORA_COORDINATION_URL)" target_access_key="$(profile_value "$PROFILE" REMORA_TARGET_ACCESS_KEY)" target_secret_key="$(profile_value "$PROFILE" REMORA_TARGET_SECRET_KEY)" kube --namespace "$FIXTURE_NAMESPACE" delete \ secret/remora-database \ secret/remora-coordination \ secret/remora-target \ --ignore-not-found \ --wait=true kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-database \ --from-literal="url=$database_url" kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-coordination \ --from-literal="url=$coordination_url" kube --namespace "$FIXTURE_NAMESPACE" create secret generic remora-target \ --from-literal="access-key=$target_access_key" \ --from-literal="secret-key=$target_secret_key" } write_helm_values() { local coordination_namespace local oidc_issuer local oidc_client_id local oidc_project_id local target_endpoint local target_bucket local target_prefix local target_region local vault_path coordination_namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)" oidc_issuer="$(profile_value "$PROFILE" REMORA_OIDC_ISSUER)" oidc_client_id="$(profile_value "$PROFILE" REMORA_OIDC_CLIENT_ID)" oidc_project_id="$(profile_value "$PROFILE" REMORA_OIDC_PROJECT_ID)" target_endpoint="$(profile_value "$PROFILE" REMORA_TARGET_ENDPOINT)" target_bucket="$(profile_value "$PROFILE" REMORA_TARGET_BUCKET)" target_prefix="$(profile_value "$PROFILE" REMORA_TARGET_PREFIX)" target_region="$(profile_value "$PROFILE" REMORA_TARGET_REGION)" vault_path="$(profile_value "$PROFILE" REMORA_VAULT_PATH)" cat > "$RUNTIME/deployment-values.yaml" <&2 return 1 } rollout_remora() { local component for component in api worker web; do kube --namespace "$FIXTURE_NAMESPACE" rollout status \ "deployment/remora-remora-$component" \ --timeout=600s done } validate_deployment_assets() { local chart="$ROOT/deploy/charts/remora" helm lint "$chart" umask 077 helm template remora "$chart" \ --namespace "$FIXTURE_NAMESPACE" \ --set observability.prometheusRule.enabled=true \ > "$RUNTIME/deployment-policy-rendered.yaml" grep -q '^kind: NetworkPolicy$' "$RUNTIME/deployment-policy-rendered.yaml" grep -q '^kind: ServiceAccount$' "$RUNTIME/deployment-policy-rendered.yaml" grep -q '169.254.0.0/16' "$RUNTIME/deployment-policy-rendered.yaml" grep -q 'cidr: 2000::/3' "$RUNTIME/deployment-policy-rendered.yaml" grep -q 'absent(remora_ready{component="remora-api"})' "$RUNTIME/deployment-policy-rendered.yaml" grep -q 'absent(remora_ready{component="remora-worker"})' "$RUNTIME/deployment-policy-rendered.yaml" grep -q 'automountServiceAccountToken: false' "$RUNTIME/deployment-policy-rendered.yaml" grep -q 'readOnlyRootFilesystem: true' "$RUNTIME/deployment-policy-rendered.yaml" if helm template remora "$chart" \ --namespace "$FIXTURE_NAMESPACE" \ --set target.exclusiveWriter=false >/dev/null 2>&1; then printf 'chart accepted an unprotected target namespace\n' >&2 return 1 fi if grep -R -E \ '(BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|postgres://[^[:space:]]+:[^[:space:]]+@|VAULT_TOKEN[=:][^$])' \ "$chart" "$ROOT/docs/spec" >/dev/null; then printf 'deployment source contains credential material\n' >&2 return 1 fi } acquire_fixture_lock REMORA_REAL_NERDCTL="$(command -v nerdctl)" export REMORA_REAL_NERDCTL validate_deployment_assets "$REMORA_REAL_NERDCTL" --namespace default build \ --file "$ROOT/Dockerfile" \ --target api \ --tag remora-api:wp25 \ "$ROOT" "$REMORA_REAL_NERDCTL" --namespace default build \ --file "$ROOT/Dockerfile" \ --target worker \ --tag remora-worker:wp25 \ "$ROOT" "$REMORA_REAL_NERDCTL" --namespace default build \ --file "$ROOT/Dockerfile" \ --target web \ --tag remora-web:wp25 \ "$ROOT" "$REMORA_REAL_NERDCTL" --namespace default build \ --file "$ROOT/Dockerfile.test" \ --target kind-deployment-runner \ --tag remora-kind-deployment:local \ "$ROOT" OWNS_FIXTURE=true setup_fixture durable coordination oidc target secret load_local_image remora-api:wp25 load_local_image remora-worker:wp25 load_local_image remora-web:wp25 load_local_image remora-kind-deployment:local PROFILE="$(write_lane_profile "$LANE" dragonfly zitadel versitygw)" configure_vault_kubernetes_auth create_external_secrets write_helm_values helm upgrade --install "$RELEASE" "$ROOT/deploy/charts/remora" \ --namespace "$FIXTURE_NAMESPACE" \ --kubeconfig "$KUBECONFIG_PATH" \ --kube-context "$CONTEXT" \ --values "$RUNTIME/deployment-values.yaml" \ --wait \ --timeout 10m rollout_remora kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=0 kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-worker --timeout=300s JOB=deployment-smoke MANIFEST="$RUNTIME/$JOB.yaml" render_lane_job \ "$TESTBED/deployment-job.yaml.in" \ "$MANIFEST" \ "$JOB" \ "$PROFILE" \ remora-kind-deployment:local run_job_manifest "$FIXTURE_NAMESPACE" "$JOB" "$MANIFEST" wait_for_queue_length 1 kube --namespace "$FIXTURE_NAMESPACE" delete pod \ --selector app=dragonfly \ --wait=true wait_for_deployment dragonfly wait_for_queue_length 1 kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=2 rollout_remora for component in api worker web; do kube --namespace "$FIXTURE_NAMESPACE" rollout restart "deployment/remora-remora-$component" kube --namespace "$FIXTURE_NAMESPACE" rollout status \ "deployment/remora-remora-$component" \ --timeout=600s done helm template remora "$ROOT/deploy/charts/remora" \ --namespace "$FIXTURE_NAMESPACE" \ --values "$RUNTIME/deployment-values.yaml" \ > "$RUNTIME/deployment-rendered.yaml" kube --namespace "$FIXTURE_NAMESPACE" apply \ --dry-run=server \ --filename "$RUNTIME/deployment-rendered.yaml" >/dev/null SUCCEEDED=true printf 'deployment smoke, rolling restart, and coordination rebuild passed\n'