Add complete system qualification evidence

Dockerfile.test

@@ -451,6 +451,29 @@ COPY crates crates
 COPY --from=kind-hf-source-verified /tmp/remora-hf-source-tests.tar.zst /tests/remora-hf-source-tests.tar.zst
 CMD ["cargo-nextest", "nextest", "run", "--archive-file", "/tests/remora-hf-source-tests.tar.zst", "--workspace-remap", "/workspace"]
 
+FROM rust-tools AS kind-qualification-build
+WORKDIR /workspace
+COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
+COPY .config .config
+COPY contracts contracts
+COPY crates crates
+COPY docs/api docs/api
+RUN cargo nextest archive \
+    --workspace \
+    --all-features \
+    --build-jobs 4 \
+    --locked \
+    --archive-file /tmp/remora-qualification-tests.tar.zst
+
+FROM rust-tools AS kind-qualification-runner
+WORKDIR /workspace
+COPY Cargo.toml ./
+COPY contracts contracts
+COPY crates crates
+COPY docs/api docs/api
+COPY --from=kind-qualification-build /tmp/remora-qualification-tests.tar.zst /tests/remora-qualification-tests.tar.zst
+CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_OIDC_ISSUER\" && test -n \"$REMORA_TARGET_ENDPOINT\" && test -n \"$REMORA_VAULT_ADDR\" && export REMORA_COORDINATION_REQUIRE=1 REMORA_IDENTITY_REQUIRE=1 REMORA_CREDENTIALS_REQUIRE=1 REMORA_TARGET_REQUIRE=1 REMORA_RESULT_REQUIRE=1 REMORA_DELETION_REQUIRE=1 REMORA_ADMISSION_REQUIRE=1 REMORA_ATTEMPT_REQUIRE=1 REMORA_RECOVERY_REQUIRE=1 REMORA_WORKER_API_REQUIRE=1 REMORA_PUBLIC_API_REQUIRE=1 REMORA_LIVE_EVENTS_REQUIRE=1 REMORA_WORKER_REQUIRE=1 REMORA_QUALIFICATION_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-qualification-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'package(remora-credentials) and not binary(qualification_cleanup)' && cargo-nextest nextest run --archive-file /tests/remora-qualification-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'package(remora-credentials) and binary(qualification_cleanup)' && cargo-nextest nextest run --archive-file /tests/remora-qualification-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'not package(remora-credentials)'"]
+
 FROM ${NODE_IMAGE} AS node-runtime
 
 FROM ${PLAYWRIGHT_IMAGE} AS web-dependencies
@@ -473,3 +496,6 @@ CMD ["sh", "-c", "pnpm check && pnpm lint && pnpm build && pnpm test:unit && pnp
 
 FROM web-check AS web-full
 CMD ["sh", "-c", "pnpm check && pnpm lint && pnpm build && pnpm test:unit && pnpm test:component:full && pnpm test:e2e:full"]
+
+FROM web-check AS kind-web-system-runner
+CMD ["pnpm", "exec", "playwright", "test", "--config", "playwright.system.config.ts"]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-live-events check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-deployment check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
+.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-live-events check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-deployment check-requirements check-qualification-kind check-qualification check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
 
 CONTAINER_ENGINE ?= nerdctl
 CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@@ -121,3 +121,12 @@ check-hf-source:
 
 check-deployment:
 	./tests/kind/run-deployment.sh
+
+check-requirements:
+	./tests/qualification/check-requirements.sh
+
+check-qualification-kind:
+	./tests/kind/run-qualification.sh
+	REMORA_SYSTEM_QUALIFICATION=1 ./tests/kind/run-deployment.sh
+
+check-qualification: check-requirements check-full check-qualification-kind

crates/remora-api/tests/deployment.rs

@@ -42,4 +42,9 @@ async fn deployed_api_accepts_an_authorized_job_for_coordination_rebuild() {
     let body: Value = submitted.json().await.unwrap();
     serde_json::from_value::<JobResourceDto>(body.clone()).unwrap();
     assert_eq!(body["phase"], "Queued");
+    if let Ok(expected) = std::env::var("REMORA_EXPECTED_JOB_ID")
+        && !expected.is_empty()
+    {
+        assert_eq!(body["id"], expected);
+    }
 }

crates/remora-api/tests/live_events.rs

@@ -40,7 +40,9 @@ use tokio_tungstenite::{
 
 mod support;
 
-use support::{mint_provider_token, oidc_provider, required_env, revoke_remora_grants};
+use support::{
+    mint_provider_token, oidc_provider, required_env, restore_remora_grants, revoke_remora_grants,
+};
 
 type Socket = WebSocketStream<MaybeTlsStream<tokio::net::TcpStream>>;
 const BROWSER_ORIGIN: &str = "https://remora.test";
@@ -60,6 +62,7 @@ impl Harness {
         if std::env::var("REMORA_LIVE_EVENTS_REQUIRE").is_err() {
             return None;
         }
+        restore_remora_grants().await.unwrap();
         let pool = connect(
             &required_env("DATABASE_URL"),
             PoolConfig {
@@ -156,7 +159,7 @@ impl Harness {
             PublicApiConfig {
                 cursor_secret: b"live-events-fixture-cursor-secret".to_vec(),
                 member_access_ttl: Duration::from_secs(60),
-                event_retention: Duration::from_millis(500),
+                event_retention: Duration::from_secs(2),
                 event_poll_interval: Duration::from_millis(250),
                 event_heartbeat_interval: Duration::from_millis(100),
                 event_ticket_ttl: Duration::from_secs(30),
@@ -403,7 +406,7 @@ async fn live_event_replay_authorization_progress_and_resync_converge() {
         .await
         .unwrap();
     let expiring_cursor = fresh["event_cursor"].as_str().unwrap().to_owned();
-    tokio::time::sleep(Duration::from_millis(600)).await;
+    tokio::time::sleep(Duration::from_millis(2_100)).await;
     let mut expired = harness
         .connect(&harness.token, Some(&expiring_cursor))
         .await;
@@ -749,12 +752,25 @@ async fn next_frame(socket: &mut Socket) -> EventFrameDto {
 }
 
 async fn next_job_frame(socket: &mut Socket, job_id: &str) -> EventFrameDto {
+    tokio::time::timeout(Duration::from_secs(10), async {
         loop {
             let frame = next_frame(socket).await;
-        if matches!(&frame, EventFrameDto::JobChanged(event) if event.job_id.as_str() == job_id) {
+            match &frame {
+                EventFrameDto::JobChanged(event) if event.job_id.as_str() == job_id => {
                     return frame;
                 }
+                EventFrameDto::ResyncRequired(event) => {
+                    panic!(
+                        "unexpected resync while waiting for job event: {:?}",
+                        event.reason
+                    )
                 }
+                _ => {}
+            }
+        }
+    })
+    .await
+    .expect("job event timeout")
 }
 
 async fn next_job_progress(
@@ -762,6 +778,7 @@ async fn next_job_progress(
     job_id: &str,
     completed_bytes: Option<u64>,
 ) -> EventFrameDto {
+    tokio::time::timeout(Duration::from_secs(10), async {
         loop {
             let frame = next_job_frame(socket, job_id).await;
             let EventFrameDto::JobChanged(event) = &frame else {
@@ -776,9 +793,13 @@ async fn next_job_progress(
                 return frame;
             }
         }
+    })
+    .await
+    .expect("job progress timeout")
 }
 
 async fn next_deletion_frame(socket: &mut Socket, operation_id: &str) -> EventFrameDto {
+    tokio::time::timeout(Duration::from_secs(10), async {
         loop {
             let frame = next_frame(socket).await;
             if matches!(
@@ -789,15 +810,22 @@ async fn next_deletion_frame(socket: &mut Socket, operation_id: &str) -> EventFr
                 return frame;
             }
         }
+    })
+    .await
+    .expect("deletion event timeout")
 }
 
 async fn next_resync(socket: &mut Socket) -> EventFrameDto {
+    tokio::time::timeout(Duration::from_secs(10), async {
         loop {
             let frame = next_frame(socket).await;
             if matches!(frame, EventFrameDto::ResyncRequired(_)) {
                 return frame;
             }
         }
+    })
+    .await
+    .expect("resync event timeout")
 }
 
 fn assert_resync(frame: EventFrameDto, expected: ResyncReasonDto) {
@@ -807,7 +835,7 @@ fn assert_resync(frame: EventFrameDto, expected: ResyncReasonDto) {
 
 fn assert_resync_reason(frame: EventFrameDto, expected: ResyncReasonDto) -> u64 {
     let EventFrameDto::ResyncRequired(frame) = frame else {
-        panic!("expected resync_required");
+        panic!("expected resync_required, got {frame:?}");
     };
     assert_eq!(frame.reason, expected);
     frame.sequence.get()

crates/remora-api/tests/public_api.rs

@@ -35,7 +35,9 @@ use time::OffsetDateTime;
 
 mod support;
 
-use support::{mint_provider_token, oidc_provider, required_env, revoke_remora_grants};
+use support::{
+    mint_provider_token, oidc_provider, required_env, restore_remora_grants, revoke_remora_grants,
+};
 
 struct Harness {
     pool: PgPool,
@@ -51,6 +53,7 @@ impl Harness {
         if std::env::var("REMORA_PUBLIC_API_REQUIRE").is_err() {
             return None;
         }
+        restore_remora_grants().await.unwrap();
         let pool = connect(
             &required_env("DATABASE_URL"),
             PoolConfig {

crates/remora-api/tests/support/mod.rs

@@ -47,6 +47,14 @@ pub async fn revoke_remora_grants() -> Result<(), String> {
     }
 }
 
+pub async fn restore_remora_grants() -> Result<(), String> {
+    match required_env("REMORA_OIDC_PROVIDER").as_str() {
+        "keycloak" => restore_keycloak_remora_roles().await,
+        "zitadel" => restore_zitadel_remora_roles().await,
+        provider => Err(format!("unsupported OIDC provider {provider}")),
+    }
+}
+
 async fn mint_keycloak_token() -> Result<String, String> {
     let response = Client::new()
         .post(format!(
@@ -262,6 +270,103 @@ async fn revoke_keycloak_remora_roles() -> Result<(), String> {
     }
 }
 
+async fn restore_keycloak_remora_roles() -> Result<(), String> {
+    let realm = required_env("REMORA_OIDC_TENANT");
+    let client_id = required_env("REMORA_OIDC_CLIENT_ID");
+    let username = required_env("REMORA_OIDC_USERNAME");
+    let client = Client::new();
+    let admin_response = client
+        .post("http://keycloak.remora.test:8080/realms/master/protocol/openid-connect/token")
+        .form(&[
+            ("grant_type", "password"),
+            ("client_id", "admin-cli"),
+            ("username", "remora-admin"),
+            ("password", "synthetic-keycloak"),
+        ])
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    let admin_body: Value = admin_response
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let admin_token = admin_body["access_token"]
+        .as_str()
+        .ok_or_else(|| "keycloak admin token missing".to_owned())?;
+    let users: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/users?username={username}&exact=true"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let user_id = users
+        .as_array()
+        .and_then(|rows| rows.first())
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "keycloak user missing".to_owned())?;
+    let clients: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/clients?clientId={client_id}"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let client_uuid = clients
+        .as_array()
+        .and_then(|rows| rows.first())
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "keycloak client missing".to_owned())?;
+    let roles: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/clients/{client_uuid}/roles"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let storage_role = required_env("REMORA_OIDC_STORAGE_ROLE");
+    let roles = roles
+        .as_array()
+        .cloned()
+        .unwrap_or_default()
+        .into_iter()
+        .filter(|role| {
+            role["name"]
+                .as_str()
+                .is_some_and(|name| name == "remora:v1:admin" || name == storage_role)
+        })
+        .collect::<Vec<_>>();
+    let response = client
+        .post(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}"
+        ))
+        .bearer_auth(admin_token)
+        .json(&roles)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    if response.status().is_success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "keycloak role restore failed: {}",
+            response.status()
+        ))
+    }
+}
+
 async fn revoke_zitadel_remora_roles() -> Result<(), String> {
     let issuer = required_env("REMORA_OIDC_ISSUER");
     let organization = required_env("REMORA_OIDC_TENANT");
@@ -316,3 +421,64 @@ async fn revoke_zitadel_remora_roles() -> Result<(), String> {
         ))
     }
 }
+
+async fn restore_zitadel_remora_roles() -> Result<(), String> {
+    let issuer = required_env("REMORA_OIDC_ISSUER");
+    let organization = required_env("REMORA_OIDC_TENANT");
+    let user_id = required_env("REMORA_OIDC_USER_ID");
+    let project_id = required_env("REMORA_OIDC_PROJECT_ID");
+    let token = required_env("REMORA_OIDC_MGMT_TOKEN");
+    let host = issuer
+        .strip_prefix("http://")
+        .or_else(|| issuer.strip_prefix("https://"))
+        .unwrap_or(&issuer);
+    let client = Client::new();
+    let grants: Value = client
+        .post(format!("{issuer}/management/v1/users/grants/_search"))
+        .header("Host", host)
+        .bearer_auth(&token)
+        .header("x-zitadel-orgid", &organization)
+        .json(&json!({ "queries": [{ "userIdQuery": { "userId": user_id } }] }))
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let grant_id = grants["result"]
+        .as_array()
+        .and_then(|rows| {
+            rows.iter().find(|row| {
+                row["projectId"]
+                    .as_str()
+                    .is_some_and(|value| value == project_id)
+            })
+        })
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "zitadel project grant missing".to_owned())?;
+    let response = client
+        .put(format!(
+            "{issuer}/management/v1/users/{user_id}/grants/{grant_id}"
+        ))
+        .header("Host", host)
+        .bearer_auth(token)
+        .header("x-zitadel-orgid", organization)
+        .json(&json!({
+            "roleKeys": [
+                "test-user",
+                "remora:v1:admin",
+                required_env("REMORA_OIDC_STORAGE_ROLE")
+            ]
+        }))
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    if response.status().is_success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "zitadel role restore failed: {}",
+            response.status()
+        ))
+    }
+}

crates/remora-coordination/tests/substrate.rs

@@ -1,5 +1,5 @@
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use remora_coordination::{
     ActivateOptions, CoordinationError, CoordinationStore, LeaseOwner, LocalPermitGate,
@@ -164,6 +164,85 @@ async fn reserve_contention_respects_global_limit() {
         .expect("cleanup");
 }
 
+#[tokio::test]
+async fn permit_contention_benchmark_preserves_global_bound() {
+    if std::env::var("REMORA_QUALIFICATION_REQUIRE").is_err() {
+        return;
+    }
+    const TASKS: usize = 16;
+    const OPERATIONS_PER_TASK: usize = 16;
+    const GLOBAL_LIMIT: u64 = 8;
+    let Some(store) = test_store(
+        "qualification-permits",
+        GLOBAL_LIMIT,
+        Duration::from_secs(30),
+    )
+    .await
+    else {
+        return;
+    };
+    let jobs = (0..TASKS * OPERATIONS_PER_TASK)
+        .map(|index| format!("job-qualification-{index:04}"))
+        .collect::<Vec<_>>();
+    let job_refs = jobs.iter().map(String::as_str).collect::<Vec<_>>();
+    store.drop_job_state(&job_refs).await.expect("clean");
+
+    let barrier = Arc::new(Barrier::new(TASKS));
+    let started = Instant::now();
+    let mut tasks = Vec::with_capacity(TASKS);
+    for task_index in 0..TASKS {
+        let store = store.clone();
+        let barrier = Arc::clone(&barrier);
+        let worker = format!("worker-qualification-{task_index:02}");
+        let task_jobs =
+            jobs[task_index * OPERATIONS_PER_TASK..(task_index + 1) * OPERATIONS_PER_TASK].to_vec();
+        tasks.push(tokio::spawn(async move {
+            let mut durations = Vec::with_capacity(OPERATIONS_PER_TASK);
+            barrier.wait().await;
+            for job in task_jobs {
+                let operation_started = Instant::now();
+                loop {
+                    let outcome = store.reserve(&job, &worker).await.expect("reserve");
+                    if outcome.ok() {
+                        break;
+                    }
+                    assert_eq!(outcome.code, OpCode::FailFull);
+                    tokio::task::yield_now().await;
+                }
+                assert!(store.global_used().await.expect("used") <= GLOBAL_LIMIT);
+                assert!(
+                    store
+                        .free_reserve(&job, Some(&worker))
+                        .await
+                        .expect("free")
+                        .ok()
+                );
+                durations.push(operation_started.elapsed());
+            }
+            durations
+        }));
+    }
+    let mut durations = Vec::with_capacity(TASKS * OPERATIONS_PER_TASK);
+    for task in tasks {
+        durations.extend(task.await.expect("join"));
+    }
+    durations.sort_unstable();
+    let elapsed = started.elapsed();
+    let p50 = durations[durations.len() / 2];
+    let p99 = durations[durations.len() * 99 / 100];
+    let operations_per_second = durations.len() as f64 / elapsed.as_secs_f64();
+    eprintln!(
+        "permit benchmark operations={} elapsed_ms={} p50_us={} p99_us={} operations_per_second={operations_per_second:.1}",
+        durations.len(),
+        elapsed.as_millis(),
+        p50.as_micros(),
+        p99.as_micros()
+    );
+    assert!(elapsed < Duration::from_secs(30));
+    assert_eq!(store.global_used().await.expect("used"), 0);
+    store.drop_job_state(&job_refs).await.expect("cleanup");
+}
+
 #[tokio::test]
 async fn release_and_expire_preserve_reserved_only_global_slot() {
     let Some(store) = test_store("permit-reserved", 1, Duration::from_secs(10)).await else {

crates/remora-credentials/tests/qualification_cleanup.rs

@@ -0,0 +1,18 @@
+use sqlx::postgres::PgPoolOptions;
+
+#[tokio::test]
+async fn remove_standalone_credential_fixture_rows() {
+    if std::env::var("REMORA_QUALIFICATION_REQUIRE").is_err() {
+        return;
+    }
+    let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL");
+    let pool = PgPoolOptions::new()
+        .max_connections(1)
+        .connect(&database_url)
+        .await
+        .expect("connect postgres");
+    sqlx::query("TRUNCATE execution_grant_credential_pins, source_credentials CASCADE")
+        .execute(&pool)
+        .await
+        .expect("clear standalone credential fixtures");
+}

tests/kind/deployment-job.yaml.in

@@ -35,6 +35,10 @@ spec:
               curl $retry --fail --silent http://remora-remora-api-metrics:9090/metrics | grep -q remora_background_runs_total
               curl $retry --fail --silent http://remora-remora-web/runtime-config.js | grep -q '"oidcClientId":"remora-web"'
               test "$(curl $retry --silent --output /dev/null --write-out '%{http_code}' http://remora-remora/v1/jobs)" = 401
+              if test -n "@@INGRESS_URL@@"; then
+                curl $retry --fail --silent --header 'Host: qualification-ingress' @@INGRESS_URL@@/ | grep -q '<title>Remora</title>'
+                test "$(curl $retry --silent --header 'Host: qualification-ingress' --output /dev/null --write-out '%{http_code}' @@INGRESS_URL@@/v1/jobs)" = 401
+              fi
               if curl --connect-timeout 2 --max-time 5 --insecure --silent --fail https://remora-remora-worker:8443/internal/v1/jobs/x/claim >/dev/null 2>&1; then
                 exit 1
               fi
@@ -50,6 +54,7 @@ spec:
                 name: @@PROFILE_NAME@@
           env:
             - {name: REMORA_DEPLOYMENT_API_URL, value: "http://remora-remora/v1"}
+            - {name: REMORA_EXPECTED_JOB_ID, value: "@@EXPECTED_JOB_ID@@"}
           volumeMounts:
             - {name: profile, mountPath: /run/remora-fixture, readOnly: true}
           resources:

tests/kind/lib.sh

@@ -860,6 +860,27 @@ cleanup_lane_target_store() {
   fi
   kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
     aws --endpoint-url "$endpoint" s3 rm "s3://$bucket" --recursive >&2 || return
+  if [[ "$provider" = versitygw ]]; then
+    kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
+      /bin/sh -ec '
+        endpoint="$1"
+        bucket="$2"
+        while :; do
+          set -- $(aws --endpoint-url "$endpoint" s3api list-object-versions \
+            --bucket "$bucket" \
+            --query "[length(not_null(Versions, \`[]\`)), length(not_null(DeleteMarkers, \`[]\`))]" \
+            --output text)
+          [ "$1" -eq 0 ] && [ "$2" -eq 0 ] && break
+          objects="$(aws --endpoint-url "$endpoint" s3api list-object-versions \
+            --bucket "$bucket" \
+            --query "{Objects: [not_null(Versions, \`[]\`), not_null(DeleteMarkers, \`[]\`)][][].{Key:Key,VersionId:VersionId}, Quiet: \`true\`}" \
+            --output json)"
+          aws --endpoint-url "$endpoint" s3api delete-objects \
+            --bucket "$bucket" \
+            --delete "$objects" >/dev/null
+        done
+      ' sh "$endpoint" "$bucket" >&2 || return
+  fi
   kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
     aws --endpoint-url "$endpoint" s3api delete-bucket --bucket "$bucket" >&2 || return
   if kube --namespace "$FIXTURE_NAMESPACE" exec "deployment/$deployment" --container s3-readiness -- \
@@ -1108,6 +1129,9 @@ run_job_matrix() {
   local manifest
   select_fixture_families "${families[@]}" || return
   while IFS=$'\t' read -r lane coordination oidc target; do
+    if [[ -n "${REMORA_FIXTURE_LANE:-}" ]] && [[ "$lane" != "$REMORA_FIXTURE_LANE" ]]; then
+      continue
+    fi
     job="$job_base-$lane"
     profile="$(write_lane_profile "$lane" "$coordination" "$oidc" "$target")" || return
     manifest="$RUNTIME/$job.yaml"
@@ -1163,7 +1187,8 @@ kube() {
 wait_for_job() {
   local namespace="$1"
   local job="$2"
-  local deadline=$((SECONDS + 300))
+  local timeout="${REMORA_JOB_TIMEOUT_SECONDS:-300}"
+  local deadline=$((SECONDS + timeout))
   local complete
   local failed
   while ((SECONDS < deadline)); do
@@ -1180,7 +1205,7 @@ wait_for_job() {
     sleep 1
   done
   print_job_logs "$namespace" "$job" >&2 || true
-  printf 'job/%s did not complete within 300 seconds\n' "$job" >&2
+  printf 'job/%s did not complete within %s seconds\n' "$job" "$timeout" >&2
   return 1
 }
 

tests/kind/qualification-ingress.yaml.in

@@ -0,0 +1,169 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+rules:
+  - apiGroups: [""]
+    resources: [configmaps, pods, secrets, endpoints]
+    verbs: [get, list, watch]
+  - apiGroups: [""]
+    resources: [services]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses, ingressclasses]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses/status]
+    verbs: [update]
+  - apiGroups: [coordination.k8s.io]
+    resources: [leases]
+    verbs: [get, create, update]
+  - apiGroups: [""]
+    resources: [events]
+    verbs: [create, patch]
+  - apiGroups: [discovery.k8s.io]
+    resources: [endpointslices]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: remora-qualification-ingress
+rules:
+  - apiGroups: [""]
+    resources: [configmaps, endpoints, nodes, pods, secrets, namespaces]
+    verbs: [list, watch]
+  - apiGroups: [""]
+    resources: [nodes]
+    verbs: [get]
+  - apiGroups: [""]
+    resources: [services]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses, ingressclasses]
+    verbs: [get, list, watch]
+  - apiGroups: [networking.k8s.io]
+    resources: [ingresses/status]
+    verbs: [update]
+  - apiGroups: [coordination.k8s.io]
+    resources: [leases]
+    verbs: [list, watch]
+  - apiGroups: [""]
+    resources: [events]
+    verbs: [create, patch]
+  - apiGroups: [discovery.k8s.io]
+    resources: [endpointslices]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: qualification-ingress
+subjects:
+  - kind: ServiceAccount
+    name: qualification-ingress
+    namespace: remora-testbed
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: remora-qualification-ingress
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: remora-qualification-ingress
+subjects:
+  - kind: ServiceAccount
+    name: qualification-ingress
+    namespace: remora-testbed
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+---
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: qualification-nginx
+spec:
+  controller: remora.dev/qualification-ingress
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+spec:
+  selector:
+    app.kubernetes.io/name: qualification-ingress
+  ports:
+    - {name: http, port: 80, targetPort: http}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: qualification-ingress
+  namespace: remora-testbed
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: qualification-ingress
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: qualification-ingress
+    spec:
+      serviceAccountName: qualification-ingress
+      containers:
+        - name: controller
+          image: @@INGRESS_IMAGE@@
+          imagePullPolicy: Never
+          args:
+            - /nginx-ingress-controller
+            - --election-id=qualification-ingress
+            - --controller-class=remora.dev/qualification-ingress
+            - --ingress-class=qualification-nginx
+            - --configmap=$(POD_NAMESPACE)/qualification-ingress
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - {name: LD_PRELOAD, value: /usr/local/lib/libmimalloc.so}
+          ports:
+            - {name: http, containerPort: 80}
+          readinessProbe:
+            httpGet: {path: /healthz, port: 10254}
+            initialDelaySeconds: 5
+            periodSeconds: 2
+          resources:
+            requests: {cpu: 100m, memory: 90Mi}
+            limits: {cpu: "1", memory: 256Mi}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add: [NET_BIND_SERVICE]
+              drop: [ALL]
+            runAsGroup: 82
+            runAsNonRoot: true
+            runAsUser: 101
+            seccompProfile: {type: RuntimeDefault}

tests/kind/qualification-job.yaml.in

@@ -0,0 +1,38 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 3600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: remora-qualification
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef:
+                name: @@PROFILE_NAME@@
+          volumeMounts:
+            - {name: profile, mountPath: /run/remora-fixture, readOnly: true}
+          resources:
+            requests: {cpu: "1", memory: 1Gi}
+            limits: {cpu: "4", memory: 12Gi}
+      volumes:
+        - name: profile
+          configMap:
+            name: @@PROFILE_NAME@@-file

tests/kind/run-deployment.sh

@@ -17,7 +17,8 @@ cleanup() {
   rm -f \
     "$RUNTIME/deployment-values.yaml" \
     "$RUNTIME/deployment-rendered.yaml" \
-    "$RUNTIME/deployment-policy-rendered.yaml"
+    "$RUNTIME/deployment-policy-rendered.yaml" \
+    "$RUNTIME/qualification-ingress.yaml"
   if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
     printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
     printf 'retained context: %s\n' "$CONTEXT"
@@ -159,6 +160,7 @@ write_helm_values() {
   local target_prefix
   local target_region
   local vault_path
+  local public_origin=http://remora.test
   coordination_namespace="$(profile_value "$PROFILE" REMORA_COORDINATION_NAMESPACE)"
   oidc_issuer="$(profile_value "$PROFILE" REMORA_OIDC_ISSUER)"
   oidc_client_id="$(profile_value "$PROFILE" REMORA_OIDC_CLIENT_ID)"
@@ -168,6 +170,9 @@ write_helm_values() {
   target_prefix="$(profile_value "$PROFILE" REMORA_TARGET_PREFIX)"
   target_region="$(profile_value "$PROFILE" REMORA_TARGET_REGION)"
   vault_path="$(profile_value "$PROFILE" REMORA_VAULT_PATH)"
+  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+    public_origin=http://127.0.0.1:4174
+  fi
   cat > "$RUNTIME/deployment-values.yaml" <<EOF
 api:
   image: {repository: remora-api, tag: wp25, digest: "", pullPolicy: Never}
@@ -175,7 +180,7 @@ worker:
   image: {repository: remora-worker, tag: wp25, digest: "", pullPolicy: Never}
 web:
   image: {repository: remora-web, tag: wp25, digest: "", pullPolicy: Never}
-  publicOrigin: http://remora.test
+  publicOrigin: $public_origin
   apiUrl: /v1/
   oidcClientId: remora-web
   cspConnectSrc: "'self' $oidc_issuer"
@@ -204,7 +209,7 @@ vault:
   path: "$vault_path"
   kubernetesAuthMount: auth/remora-kubernetes
 runtime:
-  eventAllowedOrigins: http://remora.test
+  eventAllowedOrigins: $public_origin
   recoveryIntervalSeconds: 1
 networkPolicy:
   enabled: true
@@ -226,6 +231,14 @@ observability:
   serviceMonitor: {enabled: false, interval: 30s}
   prometheusRule: {enabled: false}
 EOF
+  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+    cat >> "$RUNTIME/deployment-values.yaml" <<'EOF'
+ingress:
+  enabled: true
+  className: qualification-nginx
+  host: qualification-ingress
+EOF
+  fi
 }
 
 wait_for_queue_length() {
@@ -243,6 +256,117 @@ wait_for_queue_length() {
   return 1
 }
 
+wait_for_available_replicas() {
+  local deployment="$1"
+  local expected="$2"
+  local deadline=$((SECONDS + 120))
+  local available
+  while ((SECONDS < deadline)); do
+    available="$(kube --namespace "$FIXTURE_NAMESPACE" get "deployment/$deployment" \
+      --output 'jsonpath={.status.availableReplicas}')"
+    if [[ "${available:-0}" = "$expected" ]]; then
+      return
+    fi
+    sleep 1
+  done
+  printf '%s did not reach %s available replicas\n' "$deployment" "$expected" >&2
+  return 1
+}
+
+run_deployment_test() {
+  local job="$1"
+  local expected_job_id="${2:-}"
+  local manifest="$RUNTIME/$job.yaml"
+  render_lane_job \
+    "$TESTBED/deployment-job.yaml.in" \
+    "$manifest" \
+    "$job" \
+    "$PROFILE" \
+    remora-kind-deployment:local
+  sed -i "s|@@EXPECTED_JOB_ID@@|$expected_job_id|g" "$manifest"
+  if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+    sed -i 's|@@INGRESS_URL@@|http://qualification-ingress|g' "$manifest"
+  else
+    sed -i 's|@@INGRESS_URL@@||g' "$manifest"
+  fi
+  run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
+}
+
+run_system_browser_test() {
+  local job=system-browser
+  local manifest="$RUNTIME/$job.yaml"
+  render_lane_job \
+    "$TESTBED/system-browser-job.yaml.in" \
+    "$manifest" \
+    "$job" \
+    "$PROFILE" \
+    remora-kind-web-system:local
+  run_job_manifest "$FIXTURE_NAMESPACE" "$job" "$manifest"
+}
+
+install_qualification_ingress() {
+  local image
+  image="$(load_external_image ingress-nginx "$INGRESS_NGINX_IMAGE")"
+  sed "s|@@INGRESS_IMAGE@@|$image|g" \
+    "$TESTBED/qualification-ingress.yaml.in" \
+    > "$RUNTIME/qualification-ingress.yaml"
+  kube apply --filename "$RUNTIME/qualification-ingress.yaml"
+  wait_for_deployment qualification-ingress
+}
+
+operational_restore() {
+  local database="remora_lane_$LANE"
+  local backup="/tmp/remora-qualification.dump"
+  local expected_job_id
+  expected_job_id="$(kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres psql \
+      --username remora \
+      --dbname "$database" \
+      --tuples-only \
+      --no-align \
+      --command "SELECT job_id FROM jobs WHERE label = 'deployment-coordination-rebuild'")"
+  [[ -n "$expected_job_id" ]]
+
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=0
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=300s
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres pg_dump \
+      --username remora \
+      --dbname "$database" \
+      --format custom \
+      --file "$backup"
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres psql \
+      --username remora \
+      --dbname postgres \
+      --set ON_ERROR_STOP=1 \
+      --command "DROP DATABASE $database" \
+      --command "CREATE DATABASE $database"
+  kube --namespace "$FIXTURE_NAMESPACE" exec deployment/postgres -- \
+    env PGPASSWORD=synthetic-postgres pg_restore \
+      --username remora \
+      --dbname "$database" \
+      --exit-on-error \
+      "$backup"
+
+  kube --namespace "$FIXTURE_NAMESPACE" delete pod \
+    --selector app=dragonfly \
+    --wait=true
+  wait_for_deployment dragonfly
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-api --replicas=2
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-api --timeout=600s
+  wait_for_queue_length 1
+  run_deployment_test deployment-restore "$expected_job_id"
+
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=0
+  kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/versitygw --timeout=300s
+  wait_for_available_replicas remora-remora-api 0
+  kube --namespace "$FIXTURE_NAMESPACE" scale deployment/versitygw --replicas=1
+  wait_for_deployment versitygw
+  reset_lane_target_store versitygw "remora-lane-$LANE"
+  wait_for_available_replicas remora-remora-api 2
+}
+
 rollout_remora() {
   local component
   for component in api worker web; do
@@ -306,13 +430,26 @@ validate_deployment_assets
   --target kind-deployment-runner \
   --tag remora-kind-deployment:local \
   "$ROOT"
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  "$REMORA_REAL_NERDCTL" --namespace default build \
+    --file "$ROOT/Dockerfile.test" \
+    --target kind-web-system-runner \
+    --tag remora-kind-web-system:local \
+    "$ROOT"
+fi
 
 OWNS_FIXTURE=true
 setup_fixture durable coordination oidc target secret
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  install_qualification_ingress
+fi
 load_local_image remora-api:wp25
 load_local_image remora-worker:wp25
 load_local_image remora-web:wp25
 load_local_image remora-kind-deployment:local
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  load_local_image remora-kind-web-system:local
+fi
 
 PROFILE="$(write_lane_profile "$LANE" dragonfly zitadel versitygw)"
 configure_vault_kubernetes_auth
@@ -329,22 +466,19 @@ rollout_remora
 
 kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=0
 kube --namespace "$FIXTURE_NAMESPACE" rollout status deployment/remora-remora-worker --timeout=300s
-JOB=deployment-smoke
-MANIFEST="$RUNTIME/$JOB.yaml"
-render_lane_job \
-  "$TESTBED/deployment-job.yaml.in" \
-  "$MANIFEST" \
-  "$JOB" \
-  "$PROFILE" \
-  remora-kind-deployment:local
-run_job_manifest "$FIXTURE_NAMESPACE" "$JOB" "$MANIFEST"
+run_deployment_test deployment-smoke
 wait_for_queue_length 1
 
+if [[ "${REMORA_SYSTEM_QUALIFICATION:-0}" = 1 ]]; then
+  operational_restore
+  run_system_browser_test
+else
   kube --namespace "$FIXTURE_NAMESPACE" delete pod \
     --selector app=dragonfly \
     --wait=true
   wait_for_deployment dragonfly
   wait_for_queue_length 1
+fi
 
 kube --namespace "$FIXTURE_NAMESPACE" scale deployment/remora-remora-worker --replicas=2
 rollout_remora

tests/kind/run-qualification.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+source "$ROOT/tests/kind/lib.sh"
+
+export REMORA_JOB_TIMEOUT_SECONDS=900
+if [[ -n "${REMORA_FIXTURE_LANE:-}" ]] && [[ "${REMORA_QUALIFICATION_DIAGNOSTIC:-0}" != 1 ]]; then
+  printf 'REMORA_FIXTURE_LANE requires REMORA_QUALIFICATION_DIAGNOSTIC=1\n' >&2
+  exit 1
+fi
+
+USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
+SUCCEEDED=false
+OWNS_FIXTURE=false
+
+cleanup() {
+  local status=$?
+  trap - EXIT INT TERM
+  if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
+    printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
+    printf 'retained context: %s\n' "$CONTEXT"
+    exit "$status"
+  fi
+  if [[ "$OWNS_FIXTURE" = true ]]; then
+    destroy_fixture || status=1
+  fi
+  if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
+    printf 'user kubeconfig changed unexpectedly\n' >&2
+    status=1
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+acquire_fixture_lock
+REMORA_REAL_NERDCTL="$(command -v nerdctl)"
+export REMORA_REAL_NERDCTL
+"$REMORA_REAL_NERDCTL" --namespace default build \
+  --file "$ROOT/Dockerfile.test" \
+  --target kind-qualification-runner \
+  --tag remora-kind-qualification:local \
+  "$ROOT"
+OWNS_FIXTURE=true
+setup_fixture all
+load_local_image remora-kind-qualification:local
+run_job_matrix \
+  qualification \
+  "$TESTBED/qualification-job.yaml.in" \
+  remora-kind-qualification:local \
+  all
+SUCCEEDED=true
+printf 'full provider-matrix qualification Jobs passed\n'

tests/kind/system-browser-job.yaml.in

@@ -0,0 +1,36 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: @@JOB_NAME@@
+  namespace: remora-testbed
+  labels:
+    app.kubernetes.io/name: remora-system-browser-test
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: remora-system-browser-test
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      dnsPolicy: None
+      dnsConfig:
+        nameservers: [10.96.0.53]
+        searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
+        options:
+          - {name: ndots, value: "5"}
+      containers:
+        - name: tests
+          image: @@TEST_IMAGE@@
+          imagePullPolicy: Never
+          envFrom:
+            - configMapRef:
+                name: @@PROFILE_NAME@@
+          env:
+            - {name: REMORA_SYSTEM_BASE_URL, value: "http://127.0.0.1:4174"}
+          resources:
+            requests: {cpu: 500m, memory: 1Gi}
+            limits: {cpu: "4", memory: 6Gi}

tests/kind/versions.env

@@ -16,3 +16,4 @@ VERSITYGW_IMAGE=versity/versitygw@sha256:4a2c613f6541410f87983c5134cd26a88dfc23f
 COREDNS_IMAGE=coredns/coredns@sha256:1ba6f47265602e2e50a9c4669e3a955e4298a0d30dc82f39293d4bf1a851e0ff
 CURL_IMAGE=curlimages/curl@sha256:1ab04d023ece37e6ec991bf3306ad04e0ef0084e94a5c6b6563cfcb9563169db
 AWS_CLI_IMAGE=amazon/aws-cli@sha256:10c3a6dff0fdcb3fbc86cd744709548ad041ebb6bbe14fbedc1595a5539d8d27
+INGRESS_NGINX_IMAGE=registry.k8s.io/ingress-nginx/controller@sha256:de8fd8f1def14ed36f59c9838bd3c920ecb01754072e3e9f10832d62ef876cb8

tests/qualification/check-requirements.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+MATRIX="$ROOT/tests/qualification/requirements.tsv"
+TLC_EVIDENCE="$ROOT/tests/qualification/tlc-evidence.tsv"
+ARCHITECTURE="$ROOT/docs/spec/architecture.md"
+
+expected="$(
+  {
+    rg -o 'FR-[0-9]+' "$ARCHITECTURE"
+    printf '%s\n' \
+      NFR-Reliability \
+      NFR-Observability \
+      NFR-Scalability \
+      NFR-Performance \
+      NFR-Security \
+      NFR-Deployment
+  } | sort -Vu
+)"
+actual="$(cut -f1 "$MATRIX" | sort -Vu)"
+if [[ "$actual" != "$expected" ]]; then
+  diff -u <(printf '%s\n' "$expected") <(printf '%s\n' "$actual")
+  exit 1
+fi
+
+while IFS=$'\t' read -r requirement path symbol; do
+  [[ -n "$requirement" && -n "$path" && -n "$symbol" ]]
+  [[ -f "$ROOT/$path" ]]
+  rg --fixed-strings --quiet "$symbol" "$ROOT/$path"
+done < "$MATRIX"
+
+model_commit="$(cut -f4 "$TLC_EVIDENCE" | sort -u)"
+test "$(printf '%s\n' "$model_commit" | wc -l)" = 1
+git -C "$ROOT" cat-file -e "$model_commit^{commit}"
+git -C "$ROOT" diff --quiet "$model_commit" -- docs/protocol
+
+configs="$(find "$ROOT/docs/protocol" -maxdepth 1 -name '*.cfg' -printf '%f\n' | sort)"
+evidence_configs="$(cut -f1 "$TLC_EVIDENCE" | sort)"
+test "$configs" = "$evidence_configs"
+
+tla_sha256="$(sha256sum "$ROOT/docs/protocol/Remora.tla" | cut -d ' ' -f1)"
+while IFS=$'\t' read -r config config_sha256 evidence_tla_sha256 commit spec generated distinct depth completion; do
+  test "$(sha256sum "$ROOT/docs/protocol/$config" | cut -d ' ' -f1)" = "$config_sha256"
+  test "$evidence_tla_sha256" = "$tla_sha256"
+  test "$commit" = "$model_commit"
+  test "$(awk '/^SPECIFICATION / { print $2 }' "$ROOT/docs/protocol/$config")" = "$spec"
+  [[ "$generated" =~ ^[1-9][0-9]*$ ]]
+  [[ "$distinct" =~ ^[1-9][0-9]*$ ]]
+  [[ "$depth" =~ ^[1-9][0-9]*$ ]]
+  test "$completion" = "Model checking completed. No error has been found."
+done < "$TLC_EVIDENCE"
+
+printf 'requirement trace and complete TLC evidence binding passed\n'

tests/qualification/requirements.tsv

@@ -0,0 +1,26 @@
+FR-1	crates/remora-identity/tests/token_validation.rs	keycloak_resource_access_client_roles_decode
+FR-2	crates/remora-identity/tests/policy.rs	scope_segment_matching_rejects_textual_prefix_coincidence
+FR-3	crates/remora-worker/tests/runtime.rs	production_adapters_complete_one_job_per_source_kind
+FR-4	crates/remora-identity/tests/policy.rs	default_target_applies_bucket_and_prefix_overrides
+FR-5	crates/remora-api/tests/live_events.rs	live_event_replay_authorization_progress_and_resync_converge
+FR-6	crates/remora-attempt/tests/mutations.rs	failure_disposition_retries_with_budget_and_becomes_terminal_when_exhausted
+FR-7	crates/remora-api/tests/public_api.rs	public_routes_are_authenticated_replayable_authorized_and_redacted
+FR-8	crates/remora-deletion/tests/lifecycle.rs	soft_delete_restore_and_authorization_are_terminal_and_idempotent
+FR-9	crates/remora-deletion/tests/lifecycle.rs	concurrent_hard_delete_is_irreversible_exact_and_manifest_last
+FR-10	crates/remora-stream/tests/kernel.rs	multi_gigabyte_stream_keeps_bounded_memory
+FR-10	crates/remora-result/tests/substrate.rs	pages_large_ordered_inventory_without_materializing_all_rows
+FR-11	crates/remora-coordination/tests/substrate.rs	permit_contention_benchmark_preserves_global_bound
+FR-11	crates/remora-worker/tests/runtime.rs	local_limit_bounds_concurrent_attempt_tasks
+FR-11	deploy/charts/remora/templates/config.yaml	REMORA_GLOBAL_ATTEMPT_LIMIT
+FR-11	deploy/charts/remora/templates/worker.yaml	REMORA_WORKER_LOCAL_ATTEMPT_LIMIT
+FR-12	crates/remora-attempt/tests/mutations.rs	success_is_an_exact_fenced_commit_cas_and_terminal_state_is_immutable
+FR-13	crates/remora-attempt/tests/authority.rs	crash_injection_covers_every_cross_store_and_lost_response_boundary
+FR-14	crates/remora-attempt/tests/recovery.rs	coordination_wipes_reap_lost_attempts_and_reconstruct_dispatch
+FR-15	crates/remora-outbound/tests/transport.rs	redirects_reauthorize_and_strip_cross_origin_credentials
+NFR-Reliability	crates/remora-attempt/tests/recovery.rs	coordination_partition_rolls_back_and_restart_converges
+NFR-Observability	crates/remora-api/tests/public_api.rs	public_routes_are_authenticated_replayable_authorized_and_redacted
+NFR-Observability	web/src/App.browser.test.ts	renders job status progress and failure
+NFR-Scalability	crates/remora-worker/tests/runtime.rs	local_limit_bounds_concurrent_attempt_tasks
+NFR-Performance	crates/remora-stream/tests/kernel.rs	multi_gigabyte_stream_keeps_bounded_memory
+NFR-Security	crates/remora-api/tests/public_api.rs	public_routes_are_authenticated_replayable_authorized_and_redacted
+NFR-Deployment	tests/kind/run-deployment.sh	operational_restore

tests/qualification/tlc-evidence.tsv

@@ -0,0 +1,14 @@
+Remora.cfg	541a10f9075f416447ba79bf1777933ab07c824145662dfa028a61bf6c08279d	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	Spec	31095299	3051836	32	Model checking completed. No error has been found.
+RemoraApiMutation.cfg	a8dfe0d53ac6703cf283694b626ff8a74ebf84c73f3098d67af825c026975309	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	Spec	14883248	354363	6	Model checking completed. No error has been found.
+RemoraClaimActivation.cfg	17e8da65aed191356895e6f9db068b5cceac621def72eebe650fc8c43d748101	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	Spec	275060	32496	13	Model checking completed. No error has been found.
+RemoraCombined.cfg	07ba63c59012ae1b8ff4ebfb130b3fada08f5a570fefc16d7b73d95c5ce20645	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	89489746	7186644	31	Model checking completed. No error has been found.
+RemoraCredentials.cfg	c27c2faa29c1517c8fe4f16b701aa25be949c46523d6fb9c1deb2cefc28b2709	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	999724	116235	26	Model checking completed. No error has been found.
+RemoraDisposition.cfg	f32ab6af66b11350276ba0b316c7a1d8ad5f8f0a25595684e24ae5e8f210a83d	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	179748	23946	28	Model checking completed. No error has been found.
+RemoraDurableFault.cfg	1779d795ed139d18ea553a67a76a0d475363c7112db205f24d87d5624075533b	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	957130	119730	32	Model checking completed. No error has been found.
+RemoraFencing.cfg	252f9d0fe071cf835f9ba2fcd62a232420f467eec7bc3ec35273987dd99127df	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	1092925873	92042852	44	Model checking completed. No error has been found.
+RemoraGlobal.cfg	045ec597b4c406ba38a3c85c8b40fa8882e0c7b29b7ffa77b2d59af59f6bdf63	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	65998186	5589556	31	Model checking completed. No error has been found.
+RemoraLocal.cfg	0fc8e08acf8c1324738009277c406f3c3dd9136016106945fa271c4c2e19bbd3	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	89489746	7186644	31	Model checking completed. No error has been found.
+RemoraMultiShot.cfg	52c6247c22a98d13a72738247da9ae31366f9c01047ab672496f2e7ade6c1fc1	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	123778284	11568126	34	Model checking completed. No error has been found.
+RemoraResume.cfg	b3432788093ae86b37e68dc0afcac42f794a5de7cc9786da8bb5127a89ae2761	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	822029	103911	29	Model checking completed. No error has been found.
+RemoraStress.cfg	ce0c93848958f6ee9e8247536eda978bb3cc38a27571aa6b002e2163ea591adb	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	250078254	23679190	33	Model checking completed. No error has been found.
+RemoraTargetFault.cfg	2389efaace2237d7dca0a55711222885db1446cac59ea2e834b32badffa8f991	ec9e4ffaaa367b1420d68e2be332c2ddee32c27970a84edd13dfaa995181cd96	45ebeacaab2034913232c5963a6f526be221eb33	SafetySpec	991208	119730	32	Model checking completed. No error has been found.

web/package.json

@@ -30,6 +30,7 @@
     "@playwright/test": "1.61.1",
     "@redocly/cli": "2.33.1",
     "@sveltejs/vite-plugin-svelte": "7.2.0",
+    "@types/node": "26.1.1",
     "@vitest/browser-playwright": "4.1.10",
     "oxlint": "1.74.0",
     "oxlint-tsgolint": "0.24.0",