ms2k/remora
SHA256
Watch
0
Code Releases Activity

Add complete system qualification evidence

Browse source
This commit is contained in:
ms2k 2026-07-18 14:09:04 +09:00
commit 6952767503
11 changed files with 1256 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

5
tests/kind/deployment-job.yaml.in
View file

@ -35,6 +35,10 @@ spec:
curl $retry --fail --silent http://remora-remora-api-metrics:9090/metrics | grep -q remora_background_runs_total
curl $retry --fail --silent http://remora-remora-web/runtime-config.js | grep -q '"oidcClientId":"remora-web"'
test "$(curl $retry --silent --output /dev/null --write-out '%{http_code}' http://remora-remora/v1/jobs)" = 401
if test -n "@@INGRESS_URL@@"; then
curl $retry --fail --silent --header 'Host: qualification-ingress' @@INGRESS_URL@@/ | grep -q '<title>Remora</title>'
test "$(curl $retry --silent --header 'Host: qualification-ingress' --output /dev/null --write-out '%{http_code}' @@INGRESS_URL@@/v1/jobs)" = 401
fi
if curl --connect-timeout 2 --max-time 5 --insecure --silent --fail https://remora-remora-worker:8443/internal/v1/jobs/x/claim >/dev/null 2>&1; then
exit 1
fi
@ -50,6 +54,7 @@ spec:
name: @@PROFILE_NAME@@
env:
- {name: REMORA_DEPLOYMENT_API_URL, value: "http://remora-remora/v1"}
- {name: REMORA_EXPECTED_JOB_ID, value: "@@EXPECTED_JOB_ID@@"}
volumeMounts:
- {name: profile, mountPath: /run/remora-fixture, readOnly: true}
resources:

29
tests/kind/lib.sh
View file

@ -860,6 +860,27 @@ cleanup_lane_target_store() {
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >&2 || return
if [[ "$provider" = versitygw ]]; then
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
/bin/sh -ec '
endpoint="$1"
bucket="$2"
while :; do
set -- $(aws --endpoint-url "$endpoint" s3api list-object-versions \
--bucket "$bucket" \
--query "[length(not_null(Versions, \`[]\`)), length(not_null(DeleteMarkers, \`[]\`))]" \
--output text)
[ "$1" -eq 0 ] && [ "$2" -eq 0 ] && break
objects="$(aws --endpoint-url "$endpoint" s3api list-object-versions \
--bucket "$bucket" \
--query "{Objects: [not_null(Versions, \`[]\`), not_null(DeleteMarkers, \`[]\`)][][].{Key:Key,VersionId:VersionId}, Quiet: \`true\`}" \
--output json)"
aws --endpoint-url "$endpoint" s3api delete-objects \
--bucket "$bucket" \
--delete "$objects" >/dev/null
done
' sh "$endpoint" "$bucket" >&2 || return
fi
kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
aws --endpoint-url "$endpoint" s3api delete-bucket --bucket "$bucket" >&2 || return
if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
@ -1108,6 +1129,9 @@ run_job_matrix() {
local manifest
select_fixture_families "${families[@]}" || return
while IFS=$'\t' read -r lane coordination oidc target; do
if [[ -n "${REMORA_FIXTURE_LANE:-}" ]] && [[ "$lane" != "$REMORA_FIXTURE_LANE" ]]; then
continue
fi
job="$job_base-$lane"
profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
manifest="$RUNTIME/$job.yaml"
@ -1163,7 +1187,8 @@ kube() {
wait_for_job() {
local namespace="$1"
local job="$2"
local deadline=$((SECONDS + 300))
local timeout="${REMORA_JOB_TIMEOUT_SECONDS:-300}"
local deadline=$((SECONDS + timeout))
local complete
local failed
while ((SECONDS < deadline)); do
@ -1180,7 +1205,7 @@ wait_for_job() {
sleep 1
done
print_job_logs "$namespace" "$job" >&2 || true
printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
printf 'job/%s did not complete within %s seconds\n' "$job" "$timeout" >&2
return 1
}

169
tests/kind/qualification-ingress.yaml.in Normal file
View file

@ -0,0 +1,169 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: qualification-ingress
namespace: remora-testbed
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: qualification-ingress
namespace: remora-testbed
rules:
- apiGroups: [""]
resources: [configmaps, pods, secrets, endpoints]
verbs: [get, list, watch]
- apiGroups: [""]
resources: [services]
verbs: [get, list, watch]
- apiGroups: [networking.k8s.io]
resources: [ingresses, ingressclasses]
verbs: [get, list, watch]
- apiGroups: [networking.k8s.io]
resources: [ingresses/status]
verbs: [update]
- apiGroups: [coordination.k8s.io]
resources: [leases]
verbs: [get, create, update]
- apiGroups: [""]
resources: [events]
verbs: [create, patch]
- apiGroups: [discovery.k8s.io]
resources: [endpointslices]
verbs: [get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: remora-qualification-ingress
rules:
- apiGroups: [""]
resources: [configmaps, endpoints, nodes, pods, secrets, namespaces]
verbs: [list, watch]
- apiGroups: [""]
resources: [nodes]
verbs: [get]
- apiGroups: [""]
resources: [services]
verbs: [get, list, watch]
- apiGroups: [networking.k8s.io]
resources: [ingresses, ingressclasses]
verbs: [get, list, watch]
- apiGroups: [networking.k8s.io]
resources: [ingresses/status]
verbs: [update]
- apiGroups: [coordination.k8s.io]
resources: [leases]
verbs: [list, watch]
- apiGroups: [""]
resources: [events]
verbs: [create, patch]
- apiGroups: [discovery.k8s.io]
resources: [endpointslices]
verbs: [get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: qualification-ingress
namespace: remora-testbed
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: qualification-ingress
subjects:
- kind: ServiceAccount
name: qualification-ingress
namespace: remora-testbed
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: remora-qualification-ingress
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: remora-qualification-ingress
subjects:
- kind: ServiceAccount
name: qualification-ingress
namespace: remora-testbed
---
apiVersion: v1
kind: ConfigMap
metadata:
name: qualification-ingress
namespace: remora-testbed
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
name: qualification-nginx
spec:
controller: remora.dev/qualification-ingress
---
apiVersion: v1
kind: Service
metadata:
name: qualification-ingress
namespace: remora-testbed
spec:
selector:
app.kubernetes.io/name: qualification-ingress
ports:
- {name: http, port: 80, targetPort: http}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: qualification-ingress
namespace: remora-testbed
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: qualification-ingress
template:
metadata:
labels:
app.kubernetes.io/name: qualification-ingress
spec:
serviceAccountName: qualification-ingress
containers:
- name: controller
image: @@INGRESS_IMAGE@@
imagePullPolicy: Never
args:
- /nginx-ingress-controller
- --election-id=qualification-ingress
- --controller-class=remora.dev/qualification-ingress
- --ingress-class=qualification-nginx
- --configmap=$(POD_NAMESPACE)/qualification-ingress
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- {name: LD_PRELOAD, value: /usr/local/lib/libmimalloc.so}
ports:
- {name: http, containerPort: 80}
readinessProbe:
httpGet: {path: /healthz, port: 10254}
initialDelaySeconds: 5
periodSeconds: 2
resources:
requests: {cpu: 100m, memory: 90Mi}
limits: {cpu: "1", memory: 256Mi}
securityContext:
allowPrivilegeEscalation: false
capabilities:
add: [NET_BIND_SERVICE]
drop: [ALL]
runAsGroup: 82
runAsNonRoot: true
runAsUser: 101
seccompProfile: {type: RuntimeDefault}

38
tests/kind/qualification-job.yaml.in Normal file
View file

@ -0,0 +1,38 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: remora-testbed
spec:
backoffLimit: 0
activeDeadlineSeconds: 3600
template:
metadata:
labels:
app.kubernetes.io/name: remora-qualification
spec:
restartPolicy: Never
automountServiceAccountToken: false
enableServiceLinks: false
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
containers:
- name: tests
image: @@TEST_IMAGE@@
imagePullPolicy: Never
envFrom:
- configMapRef:
name: @@PROFILE_NAME@@
volumeMounts:
- {name: profile, mountPath: /run/remora-fixture, readOnly: true}
resources:
requests: {cpu: "1", memory: 1Gi}
limits: {cpu: "4", memory: 12Gi}
volumes:
- name: profile
configMap:
name: @@PROFILE_NAME@@-file

168
tests/kind/run-deployment.sh
View file

@ -17,7 +17,8 @@ cleanup() {
rm -f \
"$RUNTIME/deployment-values.yaml" \
"$RUNTIME/deployment-rendered.yaml" \
"$RUNTIME/deployment-policy-rendered.yaml"
"$RUNTIME/deployment-policy-rendered.yaml" \
"$RUNTIME/qualification-ingress.yaml"
if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
@ -159,6 +160,7 @@ write_helm_values() {
local target_prefix
local target_region
local vault_path
local public_origin=http://remora.test
coordination_namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)"
oidc_issuer="$(profile_value "$PROFILE" REMORA_OIDC_ISSUER)"
oidc_client_id="$(profile_value "$PROFILE" REMORA_OIDC_CLIENT_ID)"
@ -168,6 +170,9 @@ write_helm_values() {
target_prefix="$(profile_value "$PROFILE" REMORA_TARGET_PREFIX)"
target_region="$(profile_value "$PROFILE" REMORA_TARGET_REGION)"
vault_path="$(profile_value "$PROFILE" REMORA_VAULT_PATH)"
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
public_origin=http://127.0.0.1:4174
fi
cat > "$RUNTIME/deployment-values.yaml" <<EOF
api:
image: {repository: remora-api, tag: wp25, digest: "", pullPolicy: Never}
@ -175,7 +180,7 @@ worker:
image: {repository: remora-worker, tag: wp25, digest: "", pullPolicy: Never}
web:
image: {repository: remora-web, tag: wp25, digest: "", pullPolicy: Never}
publicOrigin: http://remora.test
publicOrigin: $public_origin
apiUrl: /v1/
oidcClientId: remora-web
cspConnectSrc: "'self' $oidc_issuer"
@ -204,7 +209,7 @@ vault:
path: "$vault_path"
kubernetesAuthMount: auth/remora-kubernetes
runtime:
eventAllowedOrigins: http://remora.test
eventAllowedOrigins: $public_origin
recoveryIntervalSeconds: 1
networkPolicy:
enabled: true
@ -226,6 +231,14 @@ observability:
serviceMonitor: {enabled: false, interval: 30s}
prometheusRule: {enabled: false}
EOF
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
cat >> "$RUNTIME/deployment-values.yaml" <<'EOF'
ingress:
enabled: true
className: qualification-nginx
host: qualification-ingress
EOF
fi
}
wait_for_queue_length() {
@ -243,6 +256,117 @@ wait_for_queue_length() {
return 1
}
wait_for_available_replicas() {
local deployment="$1"
local expected="$2"
local deadline=$((SECONDS + 120))
local available
while ((SECONDS < deadline)); do
available="$(kube --namespace "$FIXTURE_NAMESPACE" get "deployment/$deployment" \
--output 'jsonpath={.status.availableReplicas}')"
if [[ "${available:-0}" = "$expected" ]]; then
return
fi
sleep 1
done
printf '%s did not reach %s available replicas\n' "$deployment" "$expected" >&2
return 1
}
run_deployment_test() {
local job="$1"
local expected_job_id="${2:-}"
local manifest="$RUNTIME/$job.yaml"
render_lane_job \
"$TESTBED/deployment-job.yaml.in" \
"$manifest" \
"$job" \
"$PROFILE" \
remora-kind-deployment:local
sed -i "s|@@EXPECTED_JOB_ID@@|$expected_job_id|g" "$manifest"
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
sed -i 's|@@INGRESS_URL@@|http://qualification-ingress|g' "$manifest"
else
sed -i 's|@@INGRESS_URL@@||g' "$manifest"
fi
run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
}
run_system_browser_test() {
local job=system-browser
local manifest="$RUNTIME/$job.yaml"
render_lane_job \
"$TESTBED/system-browser-job.yaml.in" \
"$manifest" \
"$job" \
"$PROFILE" \
remora-kind-web-system:local
run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
}
install_qualification_ingress() {
local image
image="$(load_external_image ingress-nginx "$INGRESS_NGINX_IMAGE")"
sed "s|@@INGRESS_IMAGE@@|$image|g" \
"$TESTBED/qualification-ingress.yaml.in" \
> "$RUNTIME/qualification-ingress.yaml"
kube apply --filename "$RUNTIME/qualification-ingress.yaml"
wait_for_deployment qualification-ingress
}
operational_restore() {
local database="remora_lane_$LANE"
local backup="/tmp/remora-qualification.dump"
local expected_job_id
expected_job_id="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username remora \
--dbname "$database" \
--tuples-only \
--no-align \
--command "SELECT job_id FROM jobs WHERE label = 'deployment-coordination-rebuild'")"
[[ -n "$expected_job_id" ]]
kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=0
kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=300s
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres pg_dump \
--username remora \
--dbname "$database" \
--format custom \
--file "$backup"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres psql \
--username remora \
--dbname postgres \
--set ON_ERROR_STOP=1 \
--command "DROP DATABASE $database" \
--command "CREATE DATABASE $database"
kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
env PGPASSWORD=synthetic-postgres pg_restore \
--username remora \
--dbname "$database" \
--exit-on-error \
"$backup"
kube --namespace "$FIXTURE_NAMESPACE" delete pod \
--selector app=dragonfly \
--wait=true
wait_for_deployment dragonfly
kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=2
kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=600s
wait_for_queue_length 1
run_deployment_test deployment-restore "$expected_job_id"
kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=0
kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/versitygw --timeout=300s
wait_for_available_replicas remora-remora-api 0
kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=1
wait_for_deployment versitygw
reset_lane_target_store versitygw "remora-lane-$LANE"
wait_for_available_replicas remora-remora-api 2
}
rollout_remora() {
local component
for component in api worker web; do
@ -306,13 +430,26 @@ validate_deployment_assets
--target kind-deployment-runner \
--tag remora-kind-deployment:local \
"$ROOT"
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
"$REMORA_REAL_NERDCTL" --namespace default build \
--file "$ROOT/Dockerfile.test" \
--target kind-web-system-runner \
--tag remora-kind-web-system:local \
"$ROOT"
fi
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target secret
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
install_qualification_ingress
fi
load_local_image remora-api:wp25
load_local_image remora-worker:wp25
load_local_image remora-web:wp25
load_local_image remora-kind-deployment:local
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
load_local_image remora-kind-web-system:local
fi
PROFILE="$(write_lane_profile "$LANE" dragonfly zitadel versitygw)"
configure_vault_kubernetes_auth
@ -329,22 +466,19 @@ rollout_remora
kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=0
kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-worker --timeout=300s
JOB=deployment-smoke
MANIFEST="$RUNTIME/$JOB.yaml"
render_lane_job \
"$TESTBED/deployment-job.yaml.in" \
"$MANIFEST" \
"$JOB" \
"$PROFILE" \
remora-kind-deployment:local
run_job_manifest "$FIXTURE_NAMESPACE" "$JOB" "$MANIFEST"
run_deployment_test deployment-smoke
wait_for_queue_length 1
kube --namespace "$FIXTURE_NAMESPACE" delete pod \
--selector app=dragonfly \
--wait=true
wait_for_deployment dragonfly
wait_for_queue_length 1
if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
operational_restore
run_system_browser_test
else
kube --namespace "$FIXTURE_NAMESPACE" delete pod \
--selector app=dragonfly \
--wait=true
wait_for_deployment dragonfly
wait_for_queue_length 1
fi
kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=2
rollout_remora

56
tests/kind/run-qualification.sh Executable file
View file

@ -0,0 +1,56 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
export REMORA_JOB_TIMEOUT_SECONDS=900
if [[ -n "${REMORA_FIXTURE_LANE:-}" ]] && [[ "${REMORA_QUALIFICATION_DIAGNOSTIC:-0}" != 1 ]]; then
printf 'REMORA_FIXTURE_LANE requires REMORA_QUALIFICATION_DIAGNOSTIC=1\n' >&2
exit 1
fi
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
export REMORA_REAL_NERDCTL
"$REMORA_REAL_NERDCTL" --namespace default build \
--file "$ROOT/Dockerfile.test" \
--target kind-qualification-runner \
--tag remora-kind-qualification:local \
"$ROOT"
OWNS_FIXTURE=true
setup_fixture all
load_local_image remora-kind-qualification:local
run_job_matrix \
qualification \
"$TESTBED/qualification-job.yaml.in" \
remora-kind-qualification:local \
all
SUCCEEDED=true
printf 'full provider-matrix qualification Jobs passed\n'

36
tests/kind/system-browser-job.yaml.in Normal file
View file

@ -0,0 +1,36 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: remora-testbed
labels:
app.kubernetes.io/name: remora-system-browser-test
spec:
backoffLimit: 0
activeDeadlineSeconds: 600
template:
metadata:
labels:
app.kubernetes.io/name: remora-system-browser-test
spec:
restartPolicy: Never
automountServiceAccountToken: false
enableServiceLinks: false
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
containers:
- name: tests
image: @@TEST_IMAGE@@
imagePullPolicy: Never
envFrom:
- configMapRef:
name: @@PROFILE_NAME@@
env:
- {name: REMORA_SYSTEM_BASE_URL, value: "http://127.0.0.1:4174"}
resources:
requests: {cpu: 500m, memory: 1Gi}
limits: {cpu: "4", memory: 6Gi}

1
tests/kind/versions.env
View file

@ -16,3 +16,4 @@ VERSITYGW_IMAGE=versity/versitygw@sha256:4a2c613f6541410f87983c5134cd26a88dfc23f
COREDNS_IMAGE=coredns/coredns@sha256:1ba6f47265602e2e50a9c4669e3a955e4298a0d30dc82f39293d4bf1a851e0ff
CURL_IMAGE=curlimages/curl@sha256:1ab04d023ece37e6ec991bf3306ad04e0ef0084e94a5c6b6563cfcb9563169db
AWS_CLI_IMAGE=amazon/aws-cli@sha256:10c3a6dff0fdcb3fbc86cd744709548ad041ebb6bbe14fbedc1595a5539d8d27
INGRESS_NGINX_IMAGE=registry.k8s.io/ingress-nginx/controller@sha256:de8fd8f1def14ed36f59c9838bd3c920ecb01754072e3e9f10832d62ef876cb8

54
tests/qualification/check-requirements.sh Executable file
View file

@ -0,0 +1,54 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
MATRIX="$ROOT/tests/qualification/requirements.tsv"
TLC_EVIDENCE="$ROOT/tests/qualification/tlc-evidence.tsv"
ARCHITECTURE="$ROOT/docs/spec/architecture.md"
expected="$(
{
rg -o 'FR-[0-9]+' "$ARCHITECTURE"
printf '%s\n' \
NFR-Reliability \
NFR-Observability \
NFR-Scalability \
NFR-Performance \
NFR-Security \
NFR-Deployment
} | sort -Vu
)"
actual="$(cut -f1 "$MATRIX" | sort -Vu)"
if [[ "$actual" != "$expected" ]]; then
diff -u <(printf '%s\n' "$expected") <(printf '%s\n' "$actual")
exit 1
fi
while IFS=$'\t' read -r requirement path symbol; do
[[ -n "$requirement" && -n "$path" && -n "$symbol" ]]
[[ -f "$ROOT/$path" ]]
rg --fixed-strings --quiet "$symbol" "$ROOT/$path"
done < "$MATRIX"
model_commit="$(cut -f4 "$TLC_EVIDENCE" | sort -u)"
test "$(printf '%s\n' "$model_commit" | wc -l)" = 1
git -C "$ROOT" cat-file -e "$model_commit^{commit}"
git -C "$ROOT" diff --quiet "$model_commit" -- docs/protocol
configs="$(find "$ROOT/docs/protocol" -maxdepth 1 -name '*.cfg' -printf '%f\n' | sort)"
evidence_configs="$(cut -f1 "$TLC_EVIDENCE" | sort)"
test "$configs" = "$evidence_configs"
tla_sha256="$(sha256sum "$ROOT/docs/protocol/Remora.tla" | cut -d ' ' -f1)"
while IFS=$'\t' read -r config config_sha256 evidence_tla_sha256 commit spec generated distinct depth completion; do
test "$(sha256sum "$ROOT/docs/protocol/$config" | cut -d ' ' -f1)" = "$config_sha256"
test "$evidence_tla_sha256" = "$tla_sha256"
test "$commit" = "$model_commit"
test "$(awk '/^SPECIFICATION / { print $2 }' "$ROOT/docs/protocol/$config")" = "$spec"
[[ "$generated" =~ ^[1-9][0-9]*$ ]]
[[ "$distinct" =~ ^[1-9][0-9]*$ ]]
[[ "$depth" =~ ^[1-9][0-9]*$ ]]
test "$completion" = "Model checking completed. No error has been found."
done < "$TLC_EVIDENCE"
printf 'requirement trace and complete TLC evidence binding passed\n'

26
tests/qualification/requirements.tsv Normal file
View file

@ -0,0 +1,26 @@
FR-1 crates/remora-identity/tests/token_validation.rs keycloak_resource_access_client_roles_decode
FR-2 crates/remora-identity/tests/policy.rs scope_segment_matching_rejects_textual_prefix_coincidence
FR-3 crates/remora-worker/tests/runtime.rs production_adapters_complete_one_job_per_source_kind
FR-4 crates/remora-identity/tests/policy.rs default_target_applies_bucket_and_prefix_overrides
FR-5 crates/remora-api/tests/live_events.rs live_event_replay_authorization_progress_and_resync_converge
FR-6 crates/remora-attempt/tests/mutations.rs failure_disposition_retries_with_budget_and_becomes_terminal_when_exhausted
FR-7 crates/remora-api/tests/public_api.rs public_routes_are_authenticated_replayable_authorized_and_redacted
FR-8 crates/remora-deletion/tests/lifecycle.rs soft_delete_restore_and_authorization_are_terminal_and_idempotent
FR-9 crates/remora-deletion/tests/lifecycle.rs concurrent_hard_delete_is_irreversible_exact_and_manifest_last
FR-10 crates/remora-stream/tests/kernel.rs multi_gigabyte_stream_keeps_bounded_memory
FR-10 crates/remora-result/tests/substrate.rs pages_large_ordered_inventory_without_materializing_all_rows
FR-11 crates/remora-coordination/tests/substrate.rs permit_contention_benchmark_preserves_global_bound
FR-11 crates/remora-worker/tests/runtime.rs local_limit_bounds_concurrent_attempt_tasks
FR-11 deploy/charts/remora/templates/config.yaml REMORA_GLOBAL_ATTEMPT_LIMIT
FR-11 deploy/charts/remora/templates/worker.yaml REMORA_WORKER_LOCAL_ATTEMPT_LIMIT
FR-12 crates/remora-attempt/tests/mutations.rs success_is_an_exact_fenced_commit_cas_and_terminal_state_is_immutable
FR-13 crates/remora-attempt/tests/authority.rs crash_injection_covers_every_cross_store_and_lost_response_boundary
FR-14 crates/remora-attempt/tests/recovery.rs coordination_wipes_reap_lost_attempts_and_reconstruct_dispatch
FR-15 crates/remora-outbound/tests/transport.rs redirects_reauthorize_and_strip_cross_origin_credentials
NFR-Reliability crates/remora-attempt/tests/recovery.rs coordination_partition_rolls_back_and_restart_converges
NFR-Observability crates/remora-api/tests/public_api.rs public_routes_are_authenticated_replayable_authorized_and_redacted
NFR-Observability web/src/App.browser.test.ts renders job status progress and failure
NFR-Scalability crates/remora-worker/tests/runtime.rs local_limit_bounds_concurrent_attempt_tasks
NFR-Performance crates/remora-stream/tests/kernel.rs multi_gigabyte_stream_keeps_bounded_memory
NFR-Security crates/remora-api/tests/public_api.rs public_routes_are_authenticated_replayable_authorized_and_redacted
NFR-Deployment tests/kind/run-deployment.sh operational_restore
1 FR-1 crates/remora-identity/tests/token_validation.rs keycloak_resource_access_client_roles_decode
2 FR-2 crates/remora-identity/tests/policy.rs scope_segment_matching_rejects_textual_prefix_coincidence
3 FR-3 crates/remora-worker/tests/runtime.rs production_adapters_complete_one_job_per_source_kind
4 FR-4 crates/remora-identity/tests/policy.rs default_target_applies_bucket_and_prefix_overrides
5 FR-5 crates/remora-api/tests/live_events.rs live_event_replay_authorization_progress_and_resync_converge
6 FR-6 crates/remora-attempt/tests/mutations.rs failure_disposition_retries_with_budget_and_becomes_terminal_when_exhausted
7 FR-7 crates/remora-api/tests/public_api.rs public_routes_are_authenticated_replayable_authorized_and_redacted
8 FR-8 crates/remora-deletion/tests/lifecycle.rs soft_delete_restore_and_authorization_are_terminal_and_idempotent
9 FR-9 crates/remora-deletion/tests/lifecycle.rs concurrent_hard_delete_is_irreversible_exact_and_manifest_last
10 FR-10 crates/remora-stream/tests/kernel.rs multi_gigabyte_stream_keeps_bounded_memory
11 FR-10 crates/remora-result/tests/substrate.rs pages_large_ordered_inventory_without_materializing_all_rows
12 FR-11 crates/remora-coordination/tests/substrate.rs permit_contention_benchmark_preserves_global_bound
13 FR-11 crates/remora-worker/tests/runtime.rs local_limit_bounds_concurrent_attempt_tasks
14 FR-11 deploy/charts/remora/templates/config.yaml REMORA_GLOBAL_ATTEMPT_LIMIT
15 FR-11 deploy/charts/remora/templates/worker.yaml REMORA_WORKER_LOCAL_ATTEMPT_LIMIT
16 FR-12 crates/remora-attempt/tests/mutations.rs success_is_an_exact_fenced_commit_cas_and_terminal_state_is_immutable
17 FR-13 crates/remora-attempt/tests/authority.rs crash_injection_covers_every_cross_store_and_lost_response_boundary
18 FR-14 crates/remora-attempt/tests/recovery.rs coordination_wipes_reap_lost_attempts_and_reconstruct_dispatch
19 FR-15 crates/remora-outbound/tests/transport.rs redirects_reauthorize_and_strip_cross_origin_credentials
20 NFR-Reliability crates/remora-attempt/tests/recovery.rs coordination_partition_rolls_back_and_restart_converges
21 NFR-Observability crates/remora-api/tests/public_api.rs public_routes_are_authenticated_replayable_authorized_and_redacted
22 NFR-Observability web/src/App.browser.test.ts renders job status progress and failure
23 NFR-Scalability crates/remora-worker/tests/runtime.rs local_limit_bounds_concurrent_attempt_tasks
24 NFR-Performance crates/remora-stream/tests/kernel.rs multi_gigabyte_stream_keeps_bounded_memory
25 NFR-Security crates/remora-api/tests/public_api.rs public_routes_are_authenticated_replayable_authorized_and_redacted
26 NFR-Deployment tests/kind/run-deployment.sh operational_restore

14
tests/qualification/tlc-evidence.tsv Normal file
View file

@ -0,0 +1,14 @@
Remora.cfg 541a10f9075f416447ba79bf1777933ab07c824145662dfa028a61bf6c08279d ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 Spec 31095299 3051836 32 Model checking completed. No error has been found.
RemoraApiMutation.cfg a8dfe0d53ac6703cf283694b626ff8a74ebf84c73f3098d67af825c026975309 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 Spec 14883248 354363 6 Model checking completed. No error has been found.
RemoraClaimActivation.cfg 17e8da65aed191356895e6f9db068b5cceac621def72eebe650fc8c43d748101 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 Spec 275060 32496 13 Model checking completed. No error has been found.
RemoraCombined.cfg 07ba63c59012ae1b8ff4ebfb130b3fada08f5a570fefc16d7b73d95c5ce20645 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 89489746 7186644 31 Model checking completed. No error has been found.
RemoraCredentials.cfg c27c2faa29c1517c8fe4f16b701aa25be949c46523d6fb9c1deb2cefc28b2709 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 999724 116235 26 Model checking completed. No error has been found.
RemoraDisposition.cfg f32ab6af66b11350276ba0b316c7a1d8ad5f8f0a25595684e24ae5e8f210a83d ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 179748 23946 28 Model checking completed. No error has been found.
RemoraDurableFault.cfg 1779d795ed139d18ea553a67a76a0d475363c7112db205f24d87d5624075533b ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 957130 119730 32 Model checking completed. No error has been found.
RemoraFencing.cfg 252f9d0fe071cf835f9ba2fcd62a232420f467eec7bc3ec35273987dd99127df ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 1092925873 92042852 44 Model checking completed. No error has been found.
RemoraGlobal.cfg 045ec597b4c406ba38a3c85c8b40fa8882e0c7b29b7ffa77b2d59af59f6bdf63 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 65998186 5589556 31 Model checking completed. No error has been found.
RemoraLocal.cfg 0fc8e08acf8c1324738009277c406f3c3dd9136016106945fa271c4c2e19bbd3 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 89489746 7186644 31 Model checking completed. No error has been found.
RemoraMultiShot.cfg 52c6247c22a98d13a72738247da9ae31366f9c01047ab672496f2e7ade6c1fc1 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 123778284 11568126 34 Model checking completed. No error has been found.
RemoraResume.cfg b3432788093ae86b37e68dc0afcac42f794a5de7cc9786da8bb5127a89ae2761 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 822029 103911 29 Model checking completed. No error has been found.
RemoraStress.cfg ce0c93848958f6ee9e8247536eda978bb3cc38a27571aa6b002e2163ea591adb ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 250078254 23679190 33 Model checking completed. No error has been found.
RemoraTargetFault.cfg 2389efaace2237d7dca0a55711222885db1446cac59ea2e834b32badffa8f991 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 991208 119730 32 Model checking completed. No error has been found.
1 Remora.cfg 541a10f9075f416447ba79bf1777933ab07c824145662dfa028a61bf6c08279d ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 Spec 31095299 3051836 32 Model checking completed. No error has been found.
2 RemoraApiMutation.cfg a8dfe0d53ac6703cf283694b626ff8a74ebf84c73f3098d67af825c026975309 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 Spec 14883248 354363 6 Model checking completed. No error has been found.
3 RemoraClaimActivation.cfg 17e8da65aed191356895e6f9db068b5cceac621def72eebe650fc8c43d748101 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 Spec 275060 32496 13 Model checking completed. No error has been found.
4 RemoraCombined.cfg 07ba63c59012ae1b8ff4ebfb130b3fada08f5a570fefc16d7b73d95c5ce20645 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 89489746 7186644 31 Model checking completed. No error has been found.
5 RemoraCredentials.cfg c27c2faa29c1517c8fe4f16b701aa25be949c46523d6fb9c1deb2cefc28b2709 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 999724 116235 26 Model checking completed. No error has been found.
6 RemoraDisposition.cfg f32ab6af66b11350276ba0b316c7a1d8ad5f8f0a25595684e24ae5e8f210a83d ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 179748 23946 28 Model checking completed. No error has been found.
7 RemoraDurableFault.cfg 1779d795ed139d18ea553a67a76a0d475363c7112db205f24d87d5624075533b ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 957130 119730 32 Model checking completed. No error has been found.
8 RemoraFencing.cfg 252f9d0fe071cf835f9ba2fcd62a232420f467eec7bc3ec35273987dd99127df ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 1092925873 92042852 44 Model checking completed. No error has been found.
9 RemoraGlobal.cfg 045ec597b4c406ba38a3c85c8b40fa8882e0c7b29b7ffa77b2d59af59f6bdf63 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 65998186 5589556 31 Model checking completed. No error has been found.
10 RemoraLocal.cfg 0fc8e08acf8c1324738009277c406f3c3dd9136016106945fa271c4c2e19bbd3 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 89489746 7186644 31 Model checking completed. No error has been found.
11 RemoraMultiShot.cfg 52c6247c22a98d13a72738247da9ae31366f9c01047ab672496f2e7ade6c1fc1 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 123778284 11568126 34 Model checking completed. No error has been found.
12 RemoraResume.cfg b3432788093ae86b37e68dc0afcac42f794a5de7cc9786da8bb5127a89ae2761 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 822029 103911 29 Model checking completed. No error has been found.
13 RemoraStress.cfg ce0c93848958f6ee9e8247536eda978bb3cc38a27571aa6b002e2163ea591adb ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 250078254 23679190 33 Model checking completed. No error has been found.
14 RemoraTargetFault.cfg 2389efaace2237d7dca0a55711222885db1446cac59ea2e834b32badffa8f991 ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96 45ebeacaab2034913232c5963a6f526be221eb33 SafetySpec 991208 119730 32 Model checking completed. No error has been found.