Add complete system qualification evidence

crates/remora-api/tests/deployment.rs

@@ -42,4 +42,9 @@ async fn deployed_api_accepts_an_authorized_job_for_coordination_rebuild() {
     let body: Value = submitted.json().await.unwrap();
     serde_json::from_value::<JobResourceDto>(body.clone()).unwrap();
     assert_eq!(body["phase"], "Queued");
+    if let Ok(expected) = std::env::var("REMORA_EXPECTED_JOB_ID")
+        && !expected.is_empty()
+    {
+        assert_eq!(body["id"], expected);
+    }
 }

crates/remora-api/tests/live_events.rs

@@ -40,7 +40,9 @@ use tokio_tungstenite::{
 
 mod support;
 
-use support::{mint_provider_token, oidc_provider, required_env, revoke_remora_grants};
+use support::{
+    mint_provider_token, oidc_provider, required_env, restore_remora_grants, revoke_remora_grants,
+};
 
 type Socket = WebSocketStream<MaybeTlsStream<tokio::net::TcpStream>>;
 const BROWSER_ORIGIN: &str = "https://remora.test";
@@ -60,6 +62,7 @@ impl Harness {
         if std::env::var("REMORA_LIVE_EVENTS_REQUIRE").is_err() {
             return None;
         }
+        restore_remora_grants().await.unwrap();
         let pool = connect(
             &required_env("DATABASE_URL"),
             PoolConfig {
@@ -156,7 +159,7 @@ impl Harness {
             PublicApiConfig {
                 cursor_secret: b"live-events-fixture-cursor-secret".to_vec(),
                 member_access_ttl: Duration::from_secs(60),
-                event_retention: Duration::from_millis(500),
+                event_retention: Duration::from_secs(2),
                 event_poll_interval: Duration::from_millis(250),
                 event_heartbeat_interval: Duration::from_millis(100),
                 event_ticket_ttl: Duration::from_secs(30),
@@ -403,7 +406,7 @@ async fn live_event_replay_authorization_progress_and_resync_converge() {
         .await
         .unwrap();
     let expiring_cursor = fresh["event_cursor"].as_str().unwrap().to_owned();
-    tokio::time::sleep(Duration::from_millis(600)).await;
+    tokio::time::sleep(Duration::from_millis(2_100)).await;
     let mut expired = harness
         .connect(&harness.token, Some(&expiring_cursor))
         .await;
@@ -749,12 +752,25 @@ async fn next_frame(socket: &mut Socket) -> EventFrameDto {
 }
 
 async fn next_job_frame(socket: &mut Socket, job_id: &str) -> EventFrameDto {
-    loop {
-        let frame = next_frame(socket).await;
-        if matches!(&frame, EventFrameDto::JobChanged(event) if event.job_id.as_str() == job_id) {
-            return frame;
+    tokio::time::timeout(Duration::from_secs(10), async {
+        loop {
+            let frame = next_frame(socket).await;
+            match &frame {
+                EventFrameDto::JobChanged(event) if event.job_id.as_str() == job_id => {
+                    return frame;
+                }
+                EventFrameDto::ResyncRequired(event) => {
+                    panic!(
+                        "unexpected resync while waiting for job event: {:?}",
+                        event.reason
+                    )
+                }
+                _ => {}
+            }
         }
-    }
+    })
+    .await
+    .expect("job event timeout")
 }
 
 async fn next_job_progress(
@@ -762,42 +778,54 @@ async fn next_job_progress(
     job_id: &str,
     completed_bytes: Option<u64>,
 ) -> EventFrameDto {
-    loop {
-        let frame = next_job_frame(socket, job_id).await;
-        let EventFrameDto::JobChanged(event) = &frame else {
-            unreachable!()
-        };
-        let observed = event
-            .progress
-            .0
-            .as_ref()
-            .map(|progress| progress.completed_bytes.get());
-        if observed == completed_bytes {
-            return frame;
+    tokio::time::timeout(Duration::from_secs(10), async {
+        loop {
+            let frame = next_job_frame(socket, job_id).await;
+            let EventFrameDto::JobChanged(event) = &frame else {
+                unreachable!()
+            };
+            let observed = event
+                .progress
+                .0
+                .as_ref()
+                .map(|progress| progress.completed_bytes.get());
+            if observed == completed_bytes {
+                return frame;
+            }
         }
-    }
+    })
+    .await
+    .expect("job progress timeout")
 }
 
 async fn next_deletion_frame(socket: &mut Socket, operation_id: &str) -> EventFrameDto {
-    loop {
-        let frame = next_frame(socket).await;
-        if matches!(
-            &frame,
-            EventFrameDto::DeletionChanged(event)
-                if event.operation_id.as_str() == operation_id
-        ) {
-            return frame;
+    tokio::time::timeout(Duration::from_secs(10), async {
+        loop {
+            let frame = next_frame(socket).await;
+            if matches!(
+                &frame,
+                EventFrameDto::DeletionChanged(event)
+                    if event.operation_id.as_str() == operation_id
+            ) {
+                return frame;
+            }
         }
-    }
+    })
+    .await
+    .expect("deletion event timeout")
 }
 
 async fn next_resync(socket: &mut Socket) -> EventFrameDto {
-    loop {
-        let frame = next_frame(socket).await;
-        if matches!(frame, EventFrameDto::ResyncRequired(_)) {
-            return frame;
+    tokio::time::timeout(Duration::from_secs(10), async {
+        loop {
+            let frame = next_frame(socket).await;
+            if matches!(frame, EventFrameDto::ResyncRequired(_)) {
+                return frame;
+            }
         }
-    }
+    })
+    .await
+    .expect("resync event timeout")
 }
 
 fn assert_resync(frame: EventFrameDto, expected: ResyncReasonDto) {
@@ -807,7 +835,7 @@ fn assert_resync(frame: EventFrameDto, expected: ResyncReasonDto) {
 
 fn assert_resync_reason(frame: EventFrameDto, expected: ResyncReasonDto) -> u64 {
     let EventFrameDto::ResyncRequired(frame) = frame else {
-        panic!("expected resync_required");
+        panic!("expected resync_required, got {frame:?}");
     };
     assert_eq!(frame.reason, expected);
     frame.sequence.get()

crates/remora-api/tests/public_api.rs

@@ -35,7 +35,9 @@ use time::OffsetDateTime;
 
 mod support;
 
-use support::{mint_provider_token, oidc_provider, required_env, revoke_remora_grants};
+use support::{
+    mint_provider_token, oidc_provider, required_env, restore_remora_grants, revoke_remora_grants,
+};
 
 struct Harness {
     pool: PgPool,
@@ -51,6 +53,7 @@ impl Harness {
         if std::env::var("REMORA_PUBLIC_API_REQUIRE").is_err() {
             return None;
         }
+        restore_remora_grants().await.unwrap();
         let pool = connect(
             &required_env("DATABASE_URL"),
             PoolConfig {

crates/remora-api/tests/support/mod.rs

@@ -47,6 +47,14 @@ pub async fn revoke_remora_grants() -> Result<(), String> {
     }
 }
 
+pub async fn restore_remora_grants() -> Result<(), String> {
+    match required_env("REMORA_OIDC_PROVIDER").as_str() {
+        "keycloak" => restore_keycloak_remora_roles().await,
+        "zitadel" => restore_zitadel_remora_roles().await,
+        provider => Err(format!("unsupported OIDC provider {provider}")),
+    }
+}
+
 async fn mint_keycloak_token() -> Result<String, String> {
     let response = Client::new()
         .post(format!(
@@ -262,6 +270,103 @@ async fn revoke_keycloak_remora_roles() -> Result<(), String> {
     }
 }
 
+async fn restore_keycloak_remora_roles() -> Result<(), String> {
+    let realm = required_env("REMORA_OIDC_TENANT");
+    let client_id = required_env("REMORA_OIDC_CLIENT_ID");
+    let username = required_env("REMORA_OIDC_USERNAME");
+    let client = Client::new();
+    let admin_response = client
+        .post("http://keycloak.remora.test:8080/realms/master/protocol/openid-connect/token")
+        .form(&[
+            ("grant_type", "password"),
+            ("client_id", "admin-cli"),
+            ("username", "remora-admin"),
+            ("password", "synthetic-keycloak"),
+        ])
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    let admin_body: Value = admin_response
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let admin_token = admin_body["access_token"]
+        .as_str()
+        .ok_or_else(|| "keycloak admin token missing".to_owned())?;
+    let users: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/users?username={username}&exact=true"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let user_id = users
+        .as_array()
+        .and_then(|rows| rows.first())
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "keycloak user missing".to_owned())?;
+    let clients: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/clients?clientId={client_id}"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let client_uuid = clients
+        .as_array()
+        .and_then(|rows| rows.first())
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "keycloak client missing".to_owned())?;
+    let roles: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/clients/{client_uuid}/roles"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let storage_role = required_env("REMORA_OIDC_STORAGE_ROLE");
+    let roles = roles
+        .as_array()
+        .cloned()
+        .unwrap_or_default()
+        .into_iter()
+        .filter(|role| {
+            role["name"]
+                .as_str()
+                .is_some_and(|name| name == "remora:v1:admin" || name == storage_role)
+        })
+        .collect::<Vec<_>>();
+    let response = client
+        .post(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}"
+        ))
+        .bearer_auth(admin_token)
+        .json(&roles)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    if response.status().is_success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "keycloak role restore failed: {}",
+            response.status()
+        ))
+    }
+}
+
 async fn revoke_zitadel_remora_roles() -> Result<(), String> {
     let issuer = required_env("REMORA_OIDC_ISSUER");
     let organization = required_env("REMORA_OIDC_TENANT");
@@ -316,3 +421,64 @@ async fn revoke_zitadel_remora_roles() -> Result<(), String> {
         ))
     }
 }
+
+async fn restore_zitadel_remora_roles() -> Result<(), String> {
+    let issuer = required_env("REMORA_OIDC_ISSUER");
+    let organization = required_env("REMORA_OIDC_TENANT");
+    let user_id = required_env("REMORA_OIDC_USER_ID");
+    let project_id = required_env("REMORA_OIDC_PROJECT_ID");
+    let token = required_env("REMORA_OIDC_MGMT_TOKEN");
+    let host = issuer
+        .strip_prefix("http://")
+        .or_else(|| issuer.strip_prefix("https://"))
+        .unwrap_or(&issuer);
+    let client = Client::new();
+    let grants: Value = client
+        .post(format!("{issuer}/management/v1/users/grants/_search"))
+        .header("Host", host)
+        .bearer_auth(&token)
+        .header("x-zitadel-orgid", &organization)
+        .json(&json!({ "queries": [{ "userIdQuery": { "userId": user_id } }] }))
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let grant_id = grants["result"]
+        .as_array()
+        .and_then(|rows| {
+            rows.iter().find(|row| {
+                row["projectId"]
+                    .as_str()
+                    .is_some_and(|value| value == project_id)
+            })
+        })
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "zitadel project grant missing".to_owned())?;
+    let response = client
+        .put(format!(
+            "{issuer}/management/v1/users/{user_id}/grants/{grant_id}"
+        ))
+        .header("Host", host)
+        .bearer_auth(token)
+        .header("x-zitadel-orgid", organization)
+        .json(&json!({
+            "roleKeys": [
+                "test-user",
+                "remora:v1:admin",
+                required_env("REMORA_OIDC_STORAGE_ROLE")
+            ]
+        }))
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    if response.status().is_success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "zitadel role restore failed: {}",
+            response.status()
+        ))
+    }
+}

crates/remora-coordination/tests/substrate.rs

@@ -1,5 +1,5 @@
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use remora_coordination::{
     ActivateOptions, CoordinationError, CoordinationStore, LeaseOwner, LocalPermitGate,
@@ -164,6 +164,85 @@ async fn reserve_contention_respects_global_limit() {
         .expect("cleanup");
 }
 
+#[tokio::test]
+async fn permit_contention_benchmark_preserves_global_bound() {
+    if std::env::var("REMORA_QUALIFICATION_REQUIRE").is_err() {
+        return;
+    }
+    const TASKS: usize = 16;
+    const OPERATIONS_PER_TASK: usize = 16;
+    const GLOBAL_LIMIT: u64 = 8;
+    let Some(store) = test_store(
+        "qualification-permits",
+        GLOBAL_LIMIT,
+        Duration::from_secs(30),
+    )
+    .await
+    else {
+        return;
+    };
+    let jobs = (0..TASKS * OPERATIONS_PER_TASK)
+        .map(|index| format!("job-qualification-{index:04}"))
+        .collect::<Vec<_>>();
+    let job_refs = jobs.iter().map(String::as_str).collect::<Vec<_>>();
+    store.drop_job_state(&job_refs).await.expect("clean");
+
+    let barrier = Arc::new(Barrier::new(TASKS));
+    let started = Instant::now();
+    let mut tasks = Vec::with_capacity(TASKS);
+    for task_index in 0..TASKS {
+        let store = store.clone();
+        let barrier = Arc::clone(&barrier);
+        let worker = format!("worker-qualification-{task_index:02}");
+        let task_jobs =
+            jobs[task_index * OPERATIONS_PER_TASK..(task_index + 1) * OPERATIONS_PER_TASK].to_vec();
+        tasks.push(tokio::spawn(async move {
+            let mut durations = Vec::with_capacity(OPERATIONS_PER_TASK);
+            barrier.wait().await;
+            for job in task_jobs {
+                let operation_started = Instant::now();
+                loop {
+                    let outcome = store.reserve(&job, &worker).await.expect("reserve");
+                    if outcome.ok() {
+                        break;
+                    }
+                    assert_eq!(outcome.code, OpCode::FailFull);
+                    tokio::task::yield_now().await;
+                }
+                assert!(store.global_used().await.expect("used") <= GLOBAL_LIMIT);
+                assert!(
+                    store
+                        .free_reserve(&job, Some(&worker))
+                        .await
+                        .expect("free")
+                        .ok()
+                );
+                durations.push(operation_started.elapsed());
+            }
+            durations
+        }));
+    }
+    let mut durations = Vec::with_capacity(TASKS * OPERATIONS_PER_TASK);
+    for task in tasks {
+        durations.extend(task.await.expect("join"));
+    }
+    durations.sort_unstable();
+    let elapsed = started.elapsed();
+    let p50 = durations[durations.len() / 2];
+    let p99 = durations[durations.len() * 99 / 100];
+    let operations_per_second = durations.len() as f64 / elapsed.as_secs_f64();
+    eprintln!(
+        "permit benchmark operations={} elapsed_ms={} p50_us={} p99_us={} operations_per_second={operations_per_second:.1}",
+        durations.len(),
+        elapsed.as_millis(),
+        p50.as_micros(),
+        p99.as_micros()
+    );
+    assert!(elapsed < Duration::from_secs(30));
+    assert_eq!(store.global_used().await.expect("used"), 0);
+    store.drop_job_state(&job_refs).await.expect("cleanup");
+}
+
 #[tokio::test]
 async fn release_and_expire_preserve_reserved_only_global_slot() {
     let Some(store) = test_store("permit-reserved", 1, Duration::from_secs(10)).await else {

crates/remora-credentials/tests/qualification_cleanup.rs

@@ -0,0 +1,18 @@
+use sqlx::postgres::PgPoolOptions;
+
+#[tokio::test]
+async fn remove_standalone_credential_fixture_rows() {
+    if std::env::var("REMORA_QUALIFICATION_REQUIRE").is_err() {
+        return;
+    }
+    let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL");
+    let pool = PgPoolOptions::new()
+        .max_connections(1)
+        .connect(&database_url)
+        .await
+        .expect("connect postgres");
+    sqlx::query("TRUNCATE execution_grant_credential_pins, source_credentials CASCADE")
+        .execute(&pool)
+        .await
+        .expect("clear standalone credential fixtures");
+}