Add complete system qualification evidence

crates/remora-coordination/tests/substrate.rs

@@ -1,5 +1,5 @@
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use remora_coordination::{
     ActivateOptions, CoordinationError, CoordinationStore, LeaseOwner, LocalPermitGate,
@@ -164,6 +164,85 @@ async fn reserve_contention_respects_global_limit() {
         .expect("cleanup");
 }
 
+#[tokio::test]
+async fn permit_contention_benchmark_preserves_global_bound() {
+    if std::env::var("REMORA_QUALIFICATION_REQUIRE").is_err() {
+        return;
+    }
+    const TASKS: usize = 16;
+    const OPERATIONS_PER_TASK: usize = 16;
+    const GLOBAL_LIMIT: u64 = 8;
+    let Some(store) = test_store(
+        "qualification-permits",
+        GLOBAL_LIMIT,
+        Duration::from_secs(30),
+    )
+    .await
+    else {
+        return;
+    };
+    let jobs = (0..TASKS * OPERATIONS_PER_TASK)
+        .map(|index| format!("job-qualification-{index:04}"))
+        .collect::<Vec<_>>();
+    let job_refs = jobs.iter().map(String::as_str).collect::<Vec<_>>();
+    store.drop_job_state(&job_refs).await.expect("clean");
+
+    let barrier = Arc::new(Barrier::new(TASKS));
+    let started = Instant::now();
+    let mut tasks = Vec::with_capacity(TASKS);
+    for task_index in 0..TASKS {
+        let store = store.clone();
+        let barrier = Arc::clone(&barrier);
+        let worker = format!("worker-qualification-{task_index:02}");
+        let task_jobs =
+            jobs[task_index * OPERATIONS_PER_TASK..(task_index + 1) * OPERATIONS_PER_TASK].to_vec();
+        tasks.push(tokio::spawn(async move {
+            let mut durations = Vec::with_capacity(OPERATIONS_PER_TASK);
+            barrier.wait().await;
+            for job in task_jobs {
+                let operation_started = Instant::now();
+                loop {
+                    let outcome = store.reserve(&job, &worker).await.expect("reserve");
+                    if outcome.ok() {
+                        break;
+                    }
+                    assert_eq!(outcome.code, OpCode::FailFull);
+                    tokio::task::yield_now().await;
+                }
+                assert!(store.global_used().await.expect("used") <= GLOBAL_LIMIT);
+                assert!(
+                    store
+                        .free_reserve(&job, Some(&worker))
+                        .await
+                        .expect("free")
+                        .ok()
+                );
+                durations.push(operation_started.elapsed());
+            }
+            durations
+        }));
+    }
+    let mut durations = Vec::with_capacity(TASKS * OPERATIONS_PER_TASK);
+    for task in tasks {
+        durations.extend(task.await.expect("join"));
+    }
+    durations.sort_unstable();
+    let elapsed = started.elapsed();
+    let p50 = durations[durations.len() / 2];
+    let p99 = durations[durations.len() * 99 / 100];
+    let operations_per_second = durations.len() as f64 / elapsed.as_secs_f64();
+    eprintln!(
+        "permit benchmark operations={} elapsed_ms={} p50_us={} p99_us={} operations_per_second={operations_per_second:.1}",
+        durations.len(),
+        elapsed.as_millis(),
+        p50.as_micros(),
+        p99.as_micros()
+    );
+    assert!(elapsed < Duration::from_secs(30));
+    assert_eq!(store.global_used().await.expect("used"), 0);
+    store.drop_job_state(&job_refs).await.expect("cleanup");
+}
+
 #[tokio::test]
 async fn release_and_expire_preserve_reserved_only_global_slot() {
     let Some(store) = test_store("permit-reserved", 1, Duration::from_secs(10)).await else {