Expire terminal mutation identities by configured epoch

tests/kind/wp05-job.yaml.in

@@ -362,6 +362,207 @@ spec:
                 exit 1
               fi
 
+              if [ "${NUTHATCH_IDEMPOTENCY_MODE:-}" = verify ]; then
+                pod_curl() {
+                  destination="$1"
+                  shift
+                  command curl --insecure \
+                    --resolve "nuthatch-@@LANE@@:8080:$destination" "$@"
+                }
+
+                printf first-epoch >/tmp/epoch-one
+                first_size="$(wc -c </tmp/epoch-one | tr -d ' ')"
+                first_digest="$(sha256sum /tmp/epoch-one | cut -d' ' -f1)"
+                operation=73000000-0000-4000-8000-000000000001
+                mutation_url="$base/upload?root=personal&path=idempotency.txt&operation_id=$operation"
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $first_size" \
+                  --header "X-Content-SHA256: $first_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-one \
+                  "$mutation_url" >/tmp/epoch-first.json
+                grep -F '"result":"succeeded"' /tmp/epoch-first.json
+                grep -F '"idempotency_window_seconds":2' \
+                  /tmp/epoch-first.json
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $first_size" \
+                  --header "X-Content-SHA256: $first_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-one \
+                  "$mutation_url" >/tmp/epoch-replay.json
+                grep -F '"result":"succeeded"' /tmp/epoch-replay.json
+                grep -F '"idempotency_window_seconds":2' \
+                  /tmp/epoch-replay.json
+
+                printf conflict >/tmp/conflict
+                conflict_size="$(wc -c </tmp/conflict | tr -d ' ')"
+                conflict_digest="$(sha256sum /tmp/conflict | cut -d' ' -f1)"
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $conflict_size" \
+                  --header "X-Content-SHA256: $conflict_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/conflict \
+                  "$mutation_url" >/tmp/epoch-conflict.json
+                grep -F '"result":"conflict"' /tmp/epoch-conflict.json
+
+                sleep 3
+                printf second-epoch-a >/tmp/epoch-two-a
+                printf second-epoch-b >/tmp/epoch-two-b
+                second_a_size="$(wc -c </tmp/epoch-two-a | tr -d ' ')"
+                second_b_size="$(wc -c </tmp/epoch-two-b | tr -d ' ')"
+                second_a_digest="$(
+                  sha256sum /tmp/epoch-two-a | cut -d' ' -f1
+                )"
+                second_b_digest="$(
+                  sha256sum /tmp/epoch-two-b | cut -d' ' -f1
+                )"
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $second_a_size" \
+                  --header "X-Content-SHA256: $second_a_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-two-a \
+                  "$mutation_url" >/tmp/epoch-two-a.json &
+                epoch_two_a_pid=$!
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $second_b_size" \
+                  --header "X-Content-SHA256: $second_b_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-two-b \
+                  "$mutation_url" >/tmp/epoch-two-b.json &
+                epoch_two_b_pid=$!
+                wait "$epoch_two_a_pid"
+                wait "$epoch_two_b_pid"
+                test "$(
+                  grep -lF '"result":"succeeded"' \
+                    /tmp/epoch-two-a.json /tmp/epoch-two-b.json |
+                    wc -l |
+                    tr -d ' '
+                )" = 1
+                test "$(
+                  grep -lF '"result":"conflict"' \
+                    /tmp/epoch-two-a.json /tmp/epoch-two-b.json |
+                    wc -l |
+                    tr -d ' '
+                )" = 1
+                if grep -F '"result":"succeeded"' \
+                  /tmp/epoch-two-a.json >/dev/null; then
+                  epoch_two_body=/tmp/epoch-two-a
+                  epoch_two_size="$second_a_size"
+                  epoch_two_digest="$second_a_digest"
+                  epoch_two_ip="$NUTHATCH_IDEMPOTENCY_IP_B"
+                else
+                  epoch_two_body=/tmp/epoch-two-b
+                  epoch_two_size="$second_b_size"
+                  epoch_two_digest="$second_b_digest"
+                  epoch_two_ip="$NUTHATCH_IDEMPOTENCY_IP_A"
+                fi
+                pod_curl "$epoch_two_ip" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header "X-Declared-Size: $epoch_two_size" \
+                  --header "X-Content-SHA256: $epoch_two_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @"$epoch_two_body" \
+                  "$mutation_url" >/tmp/epoch-two-replay.json
+                grep -F '"result":"succeeded"' /tmp/epoch-two-replay.json
+                grep -F '"idempotency_window_seconds":2' \
+                  /tmp/epoch-two-replay.json
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  "$base/download?root=personal&path=idempotency.txt" \
+                  >/tmp/epoch-download
+                cmp "$epoch_two_body" /tmp/epoch-download
+
+                dd if=/dev/zero of=/tmp/active-body bs=1048576 count=1 \
+                  status=none
+                active_digest="$(
+                  sha256sum /tmp/active-body | cut -d' ' -f1
+                )"
+                active_operation=73000000-0000-4000-8000-000000000002
+                active_url="$base/upload?root=personal&path=active-window.bin&operation_id=$active_operation"
+                (
+                  dd if=/tmp/active-body bs=65536 count=1 status=none
+                  sleep 5
+                  dd if=/tmp/active-body bs=65536 skip=1 count=15 status=none
+                ) |
+                  pod_curl "$NUTHATCH_IDEMPOTENCY_IP_A" \
+                    --fail --silent --show-error \
+                    --cookie "$cookies" \
+                    --header 'Content-Type: application/octet-stream' \
+                    --header 'X-Declared-Size: 1048576' \
+                    --header "X-Content-SHA256: $active_digest" \
+                    --header "X-CSRF-Token: $csrf" \
+                    --request POST \
+                    --upload-file - \
+                    "$active_url" >/tmp/active-first.json &
+                active_pid=$!
+                sleep 3
+                pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                  --fail --silent --show-error \
+                  --cookie "$cookies" \
+                  --header 'Content-Type: application/octet-stream' \
+                  --header 'X-Declared-Size: 1048576' \
+                  --header "X-Content-SHA256: $active_digest" \
+                  --header "X-CSRF-Token: $csrf" \
+                  --data-binary @/tmp/epoch-one \
+                  "$active_url" >/tmp/active-replay.json
+                grep -F '"result":"pending"' /tmp/active-replay.json
+                lock_status="$(
+                  pod_curl "$NUTHATCH_IDEMPOTENCY_IP_B" \
+                    --silent --show-error \
+                    --output /tmp/active-lock.json \
+                    --write-out '%{http_code}' \
+                    --cookie "$cookies" \
+                    --header 'Content-Type: application/octet-stream' \
+                    --header "X-Declared-Size: $first_size" \
+                    --header "X-Content-SHA256: $first_digest" \
+                    --header "X-CSRF-Token: $csrf" \
+                    --data-binary @/tmp/epoch-one \
+                    "$base/upload?root=personal&path=active-window.bin&operation_id=73000000-0000-4000-8000-000000000003"
+                )"
+                test "$lock_status" = 409
+                grep -F '"error":"mutation_rejected"' /tmp/active-lock.json
+                wait "$active_pid"
+                grep -F '"result":"succeeded"' /tmp/active-first.json
+
+                printf '%s\n' \
+                  '{"epoch":1,"action":"Init"}' \
+                  '{"epoch":1,"action":"FirstRequest","observation":{"result":"pending","disposition":"first"}}' \
+                  '{"epoch":1,"action":"PrepareEffect"}' \
+                  '{"epoch":1,"action":"DispatchEffect"}' \
+                  '{"epoch":1,"action":"ExternalEffectApplied"}' \
+                  '{"epoch":1,"action":"FinalizeSuccess"}' \
+                  '{"epoch":1,"action":"ReplayRequest","observation":{"result":"succeeded","disposition":"replay"}}' \
+                  '{"epoch":1,"action":"ConflictRequest","observation":{"result":"conflict","disposition":"conflict"}}' \
+                  '{"after_epoch":1,"action":"TraceCut","reason":"terminal_identity_expired"}' \
+                  '{"epoch":2,"action":"Init"}' \
+                  '{"epoch":2,"action":"FirstRequest","observation":{"result":"pending","disposition":"first"}}' \
+                  '{"epoch":2,"action":"PrepareEffect"}' \
+                  '{"epoch":2,"action":"DispatchEffect"}' \
+                  '{"epoch":2,"action":"ExternalEffectApplied"}' \
+                  '{"epoch":2,"action":"FinalizeSuccess"}' \
+                  '{"epoch":2,"action":"ReplayRequest","observation":{"result":"succeeded","disposition":"replay"}}'
+                printf 'Cross-process idempotency epoch probes passed\n'
+                exit 0
+              fi
+
               if [ "${NUTHATCH_STREAMING_MODE:-}" = large ]; then
                 large_bytes="${NUTHATCH_STREAMING_LARGE_BYTES:?}"
                 printf 'Preparing %s-byte streaming object\n' "$large_bytes"
@@ -955,6 +1156,24 @@ spec:
                   name: @@PROFILE_NAME@@
                   key: NUTHATCH_STREAMING_THROTTLE_BURST_BYTES
                   optional: true
+            - name: NUTHATCH_IDEMPOTENCY_MODE
+              valueFrom:
+                configMapKeyRef:
+                  name: @@PROFILE_NAME@@
+                  key: NUTHATCH_IDEMPOTENCY_MODE
+                  optional: true
+            - name: NUTHATCH_IDEMPOTENCY_IP_A
+              valueFrom:
+                configMapKeyRef:
+                  name: @@PROFILE_NAME@@
+                  key: NUTHATCH_IDEMPOTENCY_IP_A
+                  optional: true
+            - name: NUTHATCH_IDEMPOTENCY_IP_B
+              valueFrom:
+                configMapKeyRef:
+                  name: @@PROFILE_NAME@@
+                  key: NUTHATCH_IDEMPOTENCY_IP_B
+                  optional: true
             - name: NUTHATCH_OIDC_CLIENT_SECRET
               valueFrom:
                 secretKeyRef:

tests/kind/wp08-run.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+APP_IMAGE=nuthatch-wp08:local
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${NUTHATCH_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+render_application() {
+  local lane="$1"
+  local profile_file="$2"
+  local pg_conn
+  local redis_url
+  local oidc_issuer
+  local oidc_client_id
+  local oidc_additional_audiences
+  local oidc_client_secret
+  local s3_endpoint
+  local s3_region
+  local s3_bucket
+  local s3_access_key
+  local s3_secret_key
+  pg_conn="$(sed -n 's/^NUTHATCH_PG_CONN=//p' "$profile_file")"
+  redis_url="$(sed -n 's/^NUTHATCH_REDIS_URL=//p' "$profile_file")"
+  oidc_issuer="$(sed -n 's/^NUTHATCH_OIDC_ISSUER=//p' "$profile_file")"
+  oidc_client_id="$(sed -n 's/^NUTHATCH_OIDC_CLIENT_ID=//p' "$profile_file")"
+  oidc_additional_audiences="$(
+    sed -n 's/^NUTHATCH_OIDC_ADDITIONAL_AUDIENCES=//p' "$profile_file"
+  )"
+  oidc_client_secret="$(<"$profile_file.oidc-secret")"
+  s3_endpoint="$(sed -n 's/^NUTHATCH_S3_ENDPOINT=//p' "$profile_file")"
+  s3_region="$(sed -n 's/^NUTHATCH_S3_REGION=//p' "$profile_file")"
+  s3_bucket="$(sed -n 's/^NUTHATCH_S3_BUCKET=//p' "$profile_file")"
+  s3_access_key="$(sed -n 's/^NUTHATCH_S3_ACCESS_KEY=//p' "$profile_file")"
+  s3_secret_key="$(sed -n 's/^NUTHATCH_S3_SECRET_KEY=//p' "$profile_file")"
+  sed \
+    -e "s|@@LANE@@|$lane|g" \
+    -e "s|@@APP_IMAGE@@|$APP_IMAGE|g" \
+    -e "s|@@PG_CONN@@|$pg_conn|g" \
+    -e "s|@@REDIS_URL@@|$redis_url|g" \
+    -e "s|@@OIDC_ISSUER@@|$oidc_issuer|g" \
+    -e "s|@@OIDC_CLIENT_ID@@|$oidc_client_id|g" \
+    -e "s|@@OIDC_ADDITIONAL_AUDIENCES@@|$oidc_additional_audiences|g" \
+    -e "s|@@OIDC_CLIENT_SECRET@@|$oidc_client_secret|g" \
+    -e "s|@@S3_ENDPOINT@@|$s3_endpoint|g" \
+    -e "s|@@S3_REGION@@|$s3_region|g" \
+    -e "s|@@S3_BUCKET@@|$s3_bucket|g" \
+    -e "s|@@S3_ACCESS_KEY@@|$s3_access_key|g" \
+    -e "s|@@S3_SECRET_KEY@@|$s3_secret_key|g" \
+    "$ROOT/deploy/kubernetes/nuthatch.yaml.in" >"$RUNTIME/wp08-app-$lane.yaml"
+  sed -i \
+    's/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "86400"/NUTHATCH_IDEMPOTENCY_WINDOW_SEC: "2"/' \
+    "$RUNTIME/wp08-app-$lane.yaml"
+  sed -i 's/  replicas: 1/  replicas: 2/' "$RUNTIME/wp08-app-$lane.yaml"
+}
+
+cleanup_application() {
+  local lane="$1"
+  kube --namespace "$FIXTURE_NAMESPACE" delete \
+    "deployment/nuthatch-$lane" \
+    "service/nuthatch-$lane" \
+    service/nuthatch-wp08-a \
+    service/nuthatch-wp08-b \
+    "configmap/nuthatch-$lane" \
+    "secret/nuthatch-$lane" \
+    --ignore-not-found >/dev/null
+  rm -f "$RUNTIME/wp08-app-$lane.yaml"
+}
+
+create_process_services() {
+  local lane="$1"
+  local profile="$2"
+  local pods
+  mapfile -t pods < <(
+    kube --namespace "$FIXTURE_NAMESPACE" get pods \
+      --selector="app=nuthatch-$lane" \
+      --field-selector=status.phase=Running \
+      --output jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' |
+      sort
+  )
+  test "${#pods[@]}" = 2
+  kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[0]}" \
+    nuthatch-wp08-process=a >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" label pod "${pods[1]}" \
+    nuthatch-wp08-process=b >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[0]}" \
+    --name=nuthatch-wp08-a \
+    --port=8080 \
+    --target-port=8080 >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" expose pod "${pods[1]}" \
+    --name=nuthatch-wp08-b \
+    --port=8080 \
+    --target-port=8080 >/dev/null
+  local ip_a
+  local ip_b
+  ip_a="$(
+    kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-wp08-a \
+      --output jsonpath='{.spec.clusterIP}'
+  )"
+  ip_b="$(
+    kube --namespace "$FIXTURE_NAMESPACE" get service nuthatch-wp08-b \
+      --output jsonpath='{.spec.clusterIP}'
+  )"
+  local profile_file="$RUNTIME/$profile.env"
+  sed -i '/^NUTHATCH_IDEMPOTENCY_/d' "$profile_file"
+  {
+    printf 'NUTHATCH_IDEMPOTENCY_MODE=verify\n'
+    printf 'NUTHATCH_IDEMPOTENCY_IP_A=%s\n' "$ip_a"
+    printf 'NUTHATCH_IDEMPOTENCY_IP_B=%s\n' "$ip_b"
+  } >>"$profile_file"
+  kube create configmap "$profile" \
+    --namespace "$FIXTURE_NAMESPACE" \
+    --from-env-file="$profile_file" \
+    --dry-run=client \
+    --output yaml |
+    kube apply --filename - >/dev/null
+}
+
+acquire_fixture_lock
+OWNS_FIXTURE=true
+setup_fixture durable coordination oidc target
+source "$RUNTIME/fixture.env"
+"$NUTHATCH_REAL_NERDCTL" --namespace "$NERDCTL_NAMESPACE" build \
+  --file "$ROOT/Dockerfile" \
+  --tag "$APP_IMAGE" \
+  "$ROOT"
+load_local_image "$APP_IMAGE" "$NERDCTL_NAMESPACE"
+
+while IFS=$'\t' read -r lane coordination oidc target; do
+  if [[ -n "${NUTHATCH_WP08_LANE:-}" ]] &&
+     [[ "$lane" != "$NUTHATCH_WP08_LANE" ]]; then
+    continue
+  fi
+  profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")"
+  printf 'Running WP08 lane %s: coordination=%s oidc=%s target=%s\n' \
+    "$lane" "$coordination" "$oidc" "$target"
+
+  cleanup_application "$lane"
+  render_application "$lane" "$RUNTIME/$profile.env"
+  kube apply --filename "$RUNTIME/wp08-app-$lane.yaml" >/dev/null
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status \
+    "deployment/nuthatch-$lane" --timeout=180s
+  create_process_services "$lane" "$profile"
+
+  job="wp08-idempotency-$lane"
+  render_lane_job \
+    "$job" \
+    "$TESTBED/wp05-job.yaml.in" \
+    "$CURL_LOCAL_IMAGE" \
+    "$profile"
+  sed -i "s|@@LANE@@|$lane|g" "$RUNTIME/$job.yaml"
+  run_job_manifest "$job" "$RUNTIME/$job.yaml"
+
+  cleanup_application "$lane"
+  cleanup_lane_state "$lane" "$coordination" "$oidc" "$target"
+  cleanup_lane_resources "$job" "$profile"
+  if [[ "${NUTHATCH_WP08_FIRST_LANE_ONLY:-0}" = 1 ]]; then
+    break
+  fi
+done < <(expand_fixture_matrix coordination oidc target)
+
+SUCCEEDED=true
+printf 'Nuthatch WP-08 idempotency gate passed\n'