Implement authorized live event replay

crates/remora-api/tests/live_events.rs

@@ -0,0 +1,773 @@
+use std::{net::TcpListener, time::Duration};
+
+use actix_web::{App, HttpServer, web};
+use futures_util::{SinkExt as _, StreamExt as _};
+use remora_admission::{AdmissionConfig, AdmissionService, migrate as admission_migrate};
+use remora_api::{
+    PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,
+    migrate as api_migrate, prune_live_event_history,
+};
+use remora_attempt::{AttemptConfig, AttemptService, migrate as attempt_migrate};
+use remora_contracts::{EventFrameDto, ResyncReasonDto};
+use remora_coordination::{
+    ActivateOptions, CoordinationStore, LeaseOwner, ProgressValue, StoreConfig,
+};
+use remora_credentials::{
+    CredentialRegistry, VaultClient, VaultConfig, migrate as credentials_migrate,
+};
+use remora_deletion::{
+    DeletionConfig, DeletionService, TargetRegistry, migrate as deletion_migrate,
+};
+use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
+use remora_durable::{PoolConfig, connect, enqueue, migrate as durable_migrate};
+use remora_identity::{DefaultTarget, migrate as identity_migrate};
+use remora_result::migrate as result_migrate;
+use reqwest::{Client, StatusCode};
+use serde_json::{Value, json};
+use sqlx::PgPool;
+use time::OffsetDateTime;
+use tokio_tungstenite::{
+    MaybeTlsStream, WebSocketStream, connect_async,
+    tungstenite::{
+        Message,
+        client::IntoClientRequest,
+        http::{HeaderValue, header::AUTHORIZATION},
+    },
+};
+
+mod support;
+
+use support::{mint_provider_token, oidc_provider, required_env, revoke_remora_grants};
+
+type Socket = WebSocketStream<MaybeTlsStream<tokio::net::TcpStream>>;
+
+struct Harness {
+    pool: PgPool,
+    coordination: CoordinationStore,
+    client: Client,
+    token: String,
+    http_base: String,
+    ws_base: String,
+    server: actix_web::dev::ServerHandle,
+}
+
+impl Harness {
+    async fn new() -> Option<Self> {
+        if std::env::var("REMORA_LIVE_EVENTS_REQUIRE").is_err() {
+            return None;
+        }
+        let pool = connect(
+            &required_env("DATABASE_URL"),
+            PoolConfig {
+                max_connections: 20,
+                acquire_timeout: Duration::from_secs(10),
+                idle_timeout: Some(Duration::from_secs(30)),
+            },
+        )
+        .await
+        .unwrap();
+        durable_migrate(&pool).await.unwrap();
+        identity_migrate(&pool).await.unwrap();
+        credentials_migrate(&pool).await.unwrap();
+        admission_migrate(&pool).await.unwrap();
+        attempt_migrate(&pool).await.unwrap();
+        result_migrate(&pool).await.unwrap();
+        deletion_migrate(&pool).await.unwrap();
+        api_migrate(&pool).await.unwrap();
+        sqlx::query("TRUNCATE TABLE jobs CASCADE")
+            .execute(&pool)
+            .await
+            .unwrap();
+        sqlx::query("TRUNCATE TABLE outbox_messages RESTART IDENTITY")
+            .execute(&pool)
+            .await
+            .unwrap();
+        sqlx::query(
+            "UPDATE live_event_history_state
+             SET floor_outbox_id = 0, progress_poll_not_before = '-infinity'
+             WHERE singleton",
+        )
+        .execute(&pool)
+        .await
+        .unwrap();
+
+        let coordination = CoordinationStore::connect(
+            &required_env("REMORA_COORDINATION_URL"),
+            format!(
+                "{}wp22:{}:",
+                required_env("REMORA_COORDINATION_NAMESPACE"),
+                nonce()
+            ),
+            StoreConfig {
+                global_limit: 16,
+                lease_ttl: Duration::from_secs(10),
+                reserve_ttl: Duration::from_secs(10),
+                progress_ttl: Duration::from_secs(10),
+                cancel_ttl: None,
+            },
+        )
+        .await
+        .unwrap();
+        let attempts = AttemptService::new(
+            pool.clone(),
+            coordination.clone(),
+            AttemptConfig {
+                max_attempts: 3,
+                local_limit: 8,
+            },
+        )
+        .unwrap();
+        let admission = AdmissionService::new(
+            pool.clone(),
+            AdmissionConfig {
+                default_target: DefaultTarget {
+                    store: TargetStoreId::parse("target-main").unwrap(),
+                    bucket: BucketName::parse("target-bucket").unwrap(),
+                    prefix: TargetPrefix::parse("models", ["_remora"]).unwrap(),
+                },
+                reserved_target_segments: vec!["_remora".into()],
+                policy_version: "live-events-v1".into(),
+            },
+        )
+        .unwrap();
+        let vault = VaultClient::new(VaultConfig {
+            address: "http://127.0.0.1:8200".into(),
+            token: "synthetic-unused-token".into(),
+            mount: "secret".into(),
+            path_prefix: "unused".into(),
+            timeout: Duration::from_secs(1),
+        })
+        .unwrap();
+        let credentials = CredentialRegistry::new(pool.clone(), vault);
+        let targets = TargetRegistry::default();
+        let deletion =
+            DeletionService::new(pool.clone(), targets.clone(), DeletionConfig::default()).unwrap();
+        let state = PublicApiState::new(
+            pool.clone(),
+            PublicApiServices {
+                identity: oidc_provider(),
+                admission,
+                attempts,
+                credentials,
+                deletion,
+                targets,
+            },
+            PublicApiConfig {
+                cursor_secret: b"live-events-fixture-cursor-secret".to_vec(),
+                member_access_ttl: Duration::from_secs(60),
+                event_retention: Duration::from_millis(500),
+                event_poll_interval: Duration::from_millis(250),
+                event_heartbeat_interval: Duration::from_millis(100),
+            },
+        )
+        .unwrap();
+        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = HttpServer::new(move || {
+            App::new()
+                .app_data(web::Data::new(state.clone()))
+                .configure(configure_public_api)
+        })
+        .listen(listener)
+        .unwrap()
+        .run();
+        let handle = server.handle();
+        actix_web::rt::spawn(server);
+        let token = mint_provider_token().await.unwrap();
+        Some(Self {
+            pool,
+            coordination,
+            client: Client::new(),
+            token,
+            http_base: format!("http://{address}/v1"),
+            ws_base: format!("ws://{address}/v1/events"),
+            server: handle,
+        })
+    }
+
+    async fn submit_job(&self) -> Value {
+        let response = self
+            .client
+            .post(format!("{}/jobs", self.http_base))
+            .bearer_auth(&self.token)
+            .header("Idempotency-Key", idempotency_key())
+            .json(&json!({
+                "label": "live events",
+                "source": {
+                    "kind": "http",
+                    "url": "https://example.invalid/live",
+                    "result_path": "payload.bin"
+                }
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::ACCEPTED);
+        response.json().await.unwrap()
+    }
+
+    async fn get_job(&self, token: &str, job_id: &str) -> reqwest::Response {
+        self.client
+            .get(format!("{}/jobs/{job_id}", self.http_base))
+            .bearer_auth(token)
+            .send()
+            .await
+            .unwrap()
+    }
+
+    async fn list_jobs(&self, token: &str) -> Value {
+        let response = self
+            .client
+            .get(format!("{}/jobs", self.http_base))
+            .bearer_auth(token)
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(response.status(), StatusCode::OK);
+        response.json().await.unwrap()
+    }
+
+    async fn connect(&self, token: &str, cursor: Option<&str>) -> Socket {
+        let url = cursor.map_or_else(
+            || self.ws_base.clone(),
+            |cursor| format!("{}?cursor={cursor}", self.ws_base),
+        );
+        let mut request = url.into_client_request().unwrap();
+        request.headers_mut().insert(
+            AUTHORIZATION,
+            HeaderValue::from_str(&format!("Bearer {token}")).unwrap(),
+        );
+        let (socket, response) = connect_async(request).await.unwrap();
+        assert_eq!(response.status(), 101);
+        socket
+    }
+}
+
+impl Drop for Harness {
+    fn drop(&mut self) {
+        actix_web::rt::spawn(self.server.stop(true));
+    }
+}
+
+#[actix_web::test]
+async fn live_event_replay_authorization_progress_and_resync_converge() {
+    let Some(harness) = Harness::new().await else {
+        return;
+    };
+
+    let mut absent = harness.connect(&harness.token, None).await;
+    assert_resync(next_frame(&mut absent).await, ResyncReasonDto::CursorAbsent);
+
+    let submitted = harness.submit_job().await;
+    let job_id = submitted["id"].as_str().unwrap().to_owned();
+    let initial_cursor = submitted["event_cursor"].as_str().unwrap().to_owned();
+
+    let mut invalid = harness
+        .connect(&harness.token, Some(&format!("{initial_cursor}x")))
+        .await;
+    assert_resync(
+        next_frame(&mut invalid).await,
+        ResyncReasonDto::CursorInvalid,
+    );
+
+    update_job_and_enqueue(&harness.pool, &job_id).await;
+    update_job_and_enqueue(&harness.pool, &job_id).await;
+    let mut first_replay = harness.connect(&harness.token, Some(&initial_cursor)).await;
+    let first = next_job_frame(&mut first_replay, &job_id).await;
+    let second = next_job_frame(&mut first_replay, &job_id).await;
+    let (first_id, first_sequence) = job_event_identity(&first);
+    let (second_id, second_sequence) = job_event_identity(&second);
+    assert_ne!(first_id, second_id);
+    assert_eq!((first_sequence, second_sequence), (1, 2));
+    let replay_cursor = frame_cursor(&second).to_owned();
+
+    let mut duplicate_replay = harness.connect(&harness.token, Some(&initial_cursor)).await;
+    let duplicate_first = next_job_frame(&mut duplicate_replay, &job_id).await;
+    let duplicate_second = next_job_frame(&mut duplicate_replay, &job_id).await;
+    assert_eq!(job_event_identity(&duplicate_first).0, first_id);
+    assert_eq!(job_event_identity(&duplicate_second).0, second_id);
+
+    let mut caught_up = harness.connect(&harness.token, Some(&replay_cursor)).await;
+    assert!(matches!(
+        next_frame(&mut caught_up).await,
+        EventFrameDto::Heartbeat(_)
+    ));
+    first_replay.close(None).await.unwrap();
+    duplicate_replay.close(None).await.unwrap();
+    caught_up.close(None).await.unwrap();
+
+    let fresh = harness
+        .get_job(&harness.token, &job_id)
+        .await
+        .json::<Value>()
+        .await
+        .unwrap();
+    let expiring_cursor = fresh["event_cursor"].as_str().unwrap().to_owned();
+    tokio::time::sleep(Duration::from_millis(600)).await;
+    let mut expired = harness
+        .connect(&harness.token, Some(&expiring_cursor))
+        .await;
+    assert_resync(
+        next_frame(&mut expired).await,
+        ResyncReasonDto::CursorExpired,
+    );
+
+    let before_gap = harness
+        .get_job(&harness.token, &job_id)
+        .await
+        .json::<Value>()
+        .await
+        .unwrap()["event_cursor"]
+        .as_str()
+        .unwrap()
+        .to_owned();
+    update_job_and_enqueue(&harness.pool, &job_id).await;
+    assert!(
+        prune_live_event_history(
+            &harness.pool,
+            OffsetDateTime::now_utc() + time::Duration::seconds(1)
+        )
+        .await
+        .unwrap()
+            > 0
+    );
+    let mut unavailable = harness.connect(&harness.token, Some(&before_gap)).await;
+    assert_resync(
+        next_frame(&mut unavailable).await,
+        ResyncReasonDto::HistoryUnavailable,
+    );
+    let after_gap = harness
+        .get_job(&harness.token, &job_id)
+        .await
+        .json::<Value>()
+        .await
+        .unwrap()["event_cursor"]
+        .as_str()
+        .unwrap()
+        .to_owned();
+    let mut recovered = harness.connect(&harness.token, Some(&after_gap)).await;
+    assert!(matches!(
+        next_frame(&mut recovered).await,
+        EventFrameDto::Heartbeat(_)
+    ));
+    recovered.close(None).await.unwrap();
+
+    let active_gap_cursor = harness
+        .get_job(&harness.token, &job_id)
+        .await
+        .json::<Value>()
+        .await
+        .unwrap()["event_cursor"]
+        .as_str()
+        .unwrap()
+        .to_owned();
+    let mut active_gap = harness
+        .connect(&harness.token, Some(&active_gap_cursor))
+        .await;
+    assert!(matches!(
+        next_frame(&mut active_gap).await,
+        EventFrameDto::Heartbeat(_)
+    ));
+    update_job_and_enqueue(&harness.pool, &job_id).await;
+    prune_live_event_history(
+        &harness.pool,
+        OffsetDateTime::now_utc() + time::Duration::seconds(1),
+    )
+    .await
+    .unwrap();
+    assert_resync_reason(
+        next_resync(&mut active_gap).await,
+        ResyncReasonDto::HistoryUnavailable,
+    );
+
+    let first_attempt = insert_running_attempt(&harness.pool, &job_id, 1).await;
+    let first_owner = LeaseOwner::new(first_attempt.as_str(), 1, "worker_live_01").unwrap();
+    assert!(
+        harness
+            .coordination
+            .activate(&job_id, &first_owner, ActivateOptions::default())
+            .await
+            .unwrap()
+            .ok()
+    );
+    let progress_cursor = harness
+        .get_job(&harness.token, &job_id)
+        .await
+        .json::<Value>()
+        .await
+        .unwrap()["event_cursor"]
+        .as_str()
+        .unwrap()
+        .to_owned();
+    let mut progress_socket = harness
+        .connect(&harness.token, Some(&progress_cursor))
+        .await;
+    assert!(
+        harness
+            .coordination
+            .report_progress_value(
+                &job_id,
+                &first_owner,
+                ProgressValue {
+                    completed_bytes: 42,
+                    total_bytes: Some(100),
+                    completed_members: 1,
+                    total_members: Some(2),
+                },
+            )
+            .await
+            .unwrap()
+            .ok()
+    );
+    let progress = next_job_progress(&mut progress_socket, &job_id, Some(42)).await;
+    assert_matching_progress(&progress, &first_attempt, 1, 42);
+    let mut progress_replay = harness
+        .connect(&harness.token, Some(&progress_cursor))
+        .await;
+    let replayed_progress = next_job_progress(&mut progress_replay, &job_id, Some(42)).await;
+    assert_eq!(
+        job_event_identity(&replayed_progress).0,
+        job_event_identity(&progress).0
+    );
+
+    let second_attempt = insert_running_attempt(&harness.pool, &job_id, 2).await;
+    let stale_suppressed = next_job_progress(&mut progress_socket, &job_id, None).await;
+    assert_job_authority(&stale_suppressed, &second_attempt, 2);
+
+    assert!(
+        harness
+            .coordination
+            .release(&job_id, &first_owner)
+            .await
+            .unwrap()
+            .ok()
+    );
+    harness.coordination.clean_terminal(&job_id).await.unwrap();
+    let second_owner = LeaseOwner::new(second_attempt.as_str(), 2, "worker_live_02").unwrap();
+    assert!(
+        harness
+            .coordination
+            .activate(&job_id, &second_owner, ActivateOptions::default())
+            .await
+            .unwrap()
+            .ok()
+    );
+    assert!(
+        harness
+            .coordination
+            .report_progress(&job_id, &second_owner, 77)
+            .await
+            .unwrap()
+            .ok()
+    );
+    let restored_progress = next_job_progress(&mut progress_socket, &job_id, Some(77)).await;
+    assert_matching_progress(&restored_progress, &second_attempt, 2, 77);
+    assert!(
+        harness
+            .coordination
+            .release(&job_id, &second_owner)
+            .await
+            .unwrap()
+            .ok()
+    );
+    harness.coordination.clean_terminal(&job_id).await.unwrap();
+    let coordination_lost = next_job_progress(&mut progress_socket, &job_id, None).await;
+    assert_job_authority(&coordination_lost, &second_attempt, 2);
+    let snapshot_after_loss: Value = harness
+        .get_job(&harness.token, &job_id)
+        .await
+        .json()
+        .await
+        .unwrap();
+    assert!(snapshot_after_loss["progress"].is_null());
+
+    transition_cancelled(&harness.pool, &job_id).await;
+    let deletion_cursor = harness
+        .get_job(&harness.token, &job_id)
+        .await
+        .json::<Value>()
+        .await
+        .unwrap()["event_cursor"]
+        .as_str()
+        .unwrap()
+        .to_owned();
+    let deletion_response = harness
+        .client
+        .post(format!("{}/jobs/{job_id}/hard-delete", harness.http_base))
+        .bearer_auth(&harness.token)
+        .header("Idempotency-Key", idempotency_key())
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(deletion_response.status(), StatusCode::ACCEPTED);
+    let operation: Value = deletion_response.json().await.unwrap();
+    let operation_id = operation["id"].as_str().unwrap();
+    let mut deletion_socket = harness
+        .connect(&harness.token, Some(&deletion_cursor))
+        .await;
+    let deletion = next_deletion_frame(&mut deletion_socket, operation_id).await;
+    let EventFrameDto::DeletionChanged(deletion) = deletion else {
+        unreachable!()
+    };
+    assert_eq!(deletion.job_id.as_str(), job_id);
+    assert_eq!(
+        serde_json::to_value(deletion.state).unwrap(),
+        json!("completed")
+    );
+
+    let authorized_cursor = harness.list_jobs(&harness.token).await["event_cursor"]
+        .as_str()
+        .unwrap()
+        .to_owned();
+    revoke_remora_grants().await.unwrap();
+    let reduced_token = mint_provider_token().await.unwrap();
+    let mut changed = harness
+        .connect(&reduced_token, Some(&authorized_cursor))
+        .await;
+    assert_resync(
+        next_frame(&mut changed).await,
+        ResyncReasonDto::AuthorizationChanged,
+    );
+    assert_eq!(
+        harness.get_job(&reduced_token, &job_id).await.status(),
+        StatusCode::NOT_FOUND
+    );
+    let reduced_snapshot = harness.list_jobs(&reduced_token).await;
+    assert!(reduced_snapshot["items"].as_array().unwrap().is_empty());
+    let reduced_cursor = reduced_snapshot["event_cursor"].as_str().unwrap();
+    let mut filtered = harness.connect(&reduced_token, Some(reduced_cursor)).await;
+    enqueue_job_event(&harness.pool, &job_id).await;
+    for _ in 0..2 {
+        assert!(matches!(
+            next_frame(&mut filtered).await,
+            EventFrameDto::Heartbeat(_)
+        ));
+    }
+
+    progress_socket.close(None).await.unwrap();
+    progress_replay.close(None).await.unwrap();
+    deletion_socket.close(None).await.unwrap();
+    filtered.close(None).await.unwrap();
+}
+
+async fn update_job_and_enqueue(pool: &PgPool, job_id: &str) {
+    let mut tx = pool.begin().await.unwrap();
+    sqlx::query("UPDATE jobs SET revision = revision + 1, updated_at = now() WHERE job_id = $1")
+        .bind(job_id)
+        .execute(tx.as_mut())
+        .await
+        .unwrap();
+    enqueue(&mut tx, "job.updated", json!({ "job_id": job_id }))
+        .await
+        .unwrap();
+    tx.commit().await.unwrap();
+}
+
+async fn enqueue_job_event(pool: &PgPool, job_id: &str) {
+    let mut tx = pool.begin().await.unwrap();
+    enqueue(&mut tx, "job.updated", json!({ "job_id": job_id }))
+        .await
+        .unwrap();
+    tx.commit().await.unwrap();
+}
+
+async fn insert_running_attempt(pool: &PgPool, job_id: &str, fence: i64) -> OpaqueId {
+    let attempt_id = opaque_id("attempt");
+    sqlx::query(
+        "INSERT INTO attempts (
+            attempt_id, job_id, fence, worker_incarnation, state, initial_progress,
+            authorized_at, executing_at, execution_envelope, staging_origin_attempt_id
+         ) VALUES ($1, $2, $3, $4, 'executing', 0, now(), now(), '{}'::jsonb, $1)",
+    )
+    .bind(attempt_id.as_str())
+    .bind(job_id)
+    .bind(fence)
+    .bind(format!("worker_live_{fence:02}"))
+    .execute(pool)
+    .await
+    .unwrap();
+    let mut tx = pool.begin().await.unwrap();
+    sqlx::query(
+        "UPDATE jobs
+         SET phase = 'running', current_fence = $2, authority_attempt_id = $3,
+             revision = revision + 1, updated_at = now()
+         WHERE job_id = $1",
+    )
+    .bind(job_id)
+    .bind(fence)
+    .bind(attempt_id.as_str())
+    .execute(tx.as_mut())
+    .await
+    .unwrap();
+    enqueue(&mut tx, "job.updated", json!({ "job_id": job_id }))
+        .await
+        .unwrap();
+    tx.commit().await.unwrap();
+    attempt_id
+}
+
+async fn transition_cancelled(pool: &PgPool, job_id: &str) {
+    let mut tx = pool.begin().await.unwrap();
+    sqlx::query(
+        "UPDATE jobs
+         SET phase = 'cancelled', cancel_requested = TRUE, authority_attempt_id = NULL,
+             dispatch_needed = FALSE, terminal_at = now(), revision = revision + 1,
+             updated_at = now()
+         WHERE job_id = $1",
+    )
+    .bind(job_id)
+    .execute(tx.as_mut())
+    .await
+    .unwrap();
+    enqueue(&mut tx, "job.updated", json!({ "job_id": job_id }))
+        .await
+        .unwrap();
+    tx.commit().await.unwrap();
+}
+
+async fn next_frame(socket: &mut Socket) -> EventFrameDto {
+    tokio::time::timeout(Duration::from_secs(3), async {
+        loop {
+            match socket.next().await {
+                Some(Ok(Message::Text(text))) => {
+                    return serde_json::from_str(text.as_ref()).unwrap();
+                }
+                Some(Ok(Message::Ping(bytes))) => {
+                    socket.send(Message::Pong(bytes)).await.unwrap();
+                }
+                Some(Ok(Message::Close(reason))) => {
+                    panic!("websocket closed before a frame: {reason:?}");
+                }
+                Some(Ok(_)) => {}
+                Some(Err(error)) => panic!("websocket error: {error}"),
+                None => panic!("websocket ended before a frame"),
+            }
+        }
+    })
+    .await
+    .expect("event frame timeout")
+}
+
+async fn next_job_frame(socket: &mut Socket, job_id: &str) -> EventFrameDto {
+    loop {
+        let frame = next_frame(socket).await;
+        if matches!(&frame, EventFrameDto::JobChanged(event) if event.job_id.as_str() == job_id) {
+            return frame;
+        }
+    }
+}
+
+async fn next_job_progress(
+    socket: &mut Socket,
+    job_id: &str,
+    completed_bytes: Option<u64>,
+) -> EventFrameDto {
+    loop {
+        let frame = next_job_frame(socket, job_id).await;
+        let EventFrameDto::JobChanged(event) = &frame else {
+            unreachable!()
+        };
+        let observed = event
+            .progress
+            .0
+            .as_ref()
+            .map(|progress| progress.completed_bytes.get());
+        if observed == completed_bytes {
+            return frame;
+        }
+    }
+}
+
+async fn next_deletion_frame(socket: &mut Socket, operation_id: &str) -> EventFrameDto {
+    loop {
+        let frame = next_frame(socket).await;
+        if matches!(
+            &frame,
+            EventFrameDto::DeletionChanged(event)
+                if event.operation_id.as_str() == operation_id
+        ) {
+            return frame;
+        }
+    }
+}
+
+async fn next_resync(socket: &mut Socket) -> EventFrameDto {
+    loop {
+        let frame = next_frame(socket).await;
+        if matches!(frame, EventFrameDto::ResyncRequired(_)) {
+            return frame;
+        }
+    }
+}
+
+fn assert_resync(frame: EventFrameDto, expected: ResyncReasonDto) {
+    let sequence = assert_resync_reason(frame, expected);
+    assert_eq!(sequence, 1);
+}
+
+fn assert_resync_reason(frame: EventFrameDto, expected: ResyncReasonDto) -> u64 {
+    let EventFrameDto::ResyncRequired(frame) = frame else {
+        panic!("expected resync_required");
+    };
+    assert_eq!(frame.reason, expected);
+    frame.sequence.get()
+}
+
+fn job_event_identity(frame: &EventFrameDto) -> (&str, u64) {
+    let EventFrameDto::JobChanged(frame) = frame else {
+        panic!("expected job_changed");
+    };
+    (frame.event_id.as_str(), frame.sequence.get())
+}
+
+fn frame_cursor(frame: &EventFrameDto) -> &str {
+    match frame {
+        EventFrameDto::JobChanged(frame) => frame.cursor.as_str(),
+        EventFrameDto::DeletionChanged(frame) => frame.cursor.as_str(),
+        EventFrameDto::ResyncRequired(frame) => frame.cursor.as_str(),
+        EventFrameDto::Heartbeat(frame) => frame.cursor.as_str(),
+    }
+}
+
+fn assert_matching_progress(
+    frame: &EventFrameDto,
+    attempt_id: &OpaqueId,
+    fence: u64,
+    completed_bytes: u64,
+) {
+    let EventFrameDto::JobChanged(frame) = frame else {
+        panic!("expected job_changed");
+    };
+    let progress = frame.progress.0.as_ref().expect("matching progress");
+    assert_eq!(progress.attempt_id, *attempt_id);
+    assert_eq!(progress.fence.get(), fence);
+    assert_eq!(progress.completed_bytes.get(), completed_bytes);
+}
+
+fn assert_job_authority(frame: &EventFrameDto, attempt_id: &OpaqueId, fence: u64) {
+    let EventFrameDto::JobChanged(frame) = frame else {
+        panic!("expected job_changed");
+    };
+    assert!(frame.progress.0.is_none());
+    let authority = frame
+        .authoritative_attempt
+        .0
+        .as_ref()
+        .expect("authoritative attempt");
+    assert_eq!(authority.attempt_id, *attempt_id);
+    assert_eq!(authority.fence.get(), fence);
+}
+
+fn opaque_id(prefix: &str) -> OpaqueId {
+    OpaqueId::parse(format!("{prefix}_{}", nonce())).unwrap()
+}
+
+fn idempotency_key() -> String {
+    format!("key-{}-replay-safe", nonce())
+}
+
+fn nonce() -> String {
+    let mut bytes = [0_u8; 8];
+    getrandom::fill(&mut bytes).unwrap();
+    bytes.iter().map(|byte| format!("{byte:02x}")).collect()
+}

crates/remora-api/tests/public_api.rs

@@ -1,13 +1,11 @@
 use std::{net::TcpListener, time::Duration};
 
 use actix_web::{App, HttpServer, web};
-use base64::{
-    Engine as _,
-    engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD},
-};
-use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
 use remora_admission::{AdmissionConfig, AdmissionService, migrate as admission_migrate};
-use remora_api::{PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api};
+use remora_api::{
+    PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,
+    migrate as api_migrate,
+};
 use remora_attempt::{AttemptConfig, AttemptService, migrate as attempt_migrate};
 use remora_contracts::{
     AttemptPageDto, DeletionOperationDto, JobPageDto, JobResourceDto, ResultDescriptorDto,
@@ -24,9 +22,7 @@ use remora_deletion::{
 };
 use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
 use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
-use remora_identity::{
-    DefaultTarget, OidcConfig, OidcIdentityProvider, migrate as identity_migrate,
-};
+use remora_identity::{DefaultTarget, migrate as identity_migrate};
 use remora_result::migrate as result_migrate;
 use remora_target::{
     CreateObjectRequest, TargetStore, TargetStoreConfig, attempt_manifest_key_in,
@@ -37,6 +33,10 @@ use serde_json::{Value, json};
 use sqlx::PgPool;
 use time::OffsetDateTime;
 
+mod support;
+
+use support::{mint_provider_token, oidc_provider, required_env, revoke_remora_grants};
+
 struct Harness {
     pool: PgPool,
     coordination: CoordinationStore,
@@ -68,6 +68,7 @@ impl Harness {
         attempt_migrate(&pool).await.unwrap();
         result_migrate(&pool).await.unwrap();
         deletion_migrate(&pool).await.unwrap();
+        api_migrate(&pool).await.unwrap();
         sqlx::query("TRUNCATE TABLE jobs CASCADE")
             .execute(&pool)
             .await
@@ -140,6 +141,9 @@ impl Harness {
             PublicApiConfig {
                 cursor_secret: b"public-api-fixture-cursor-secret".to_vec(),
                 member_access_ttl: Duration::from_secs(60),
+                event_retention: Duration::from_secs(60),
+                event_poll_interval: Duration::from_millis(50),
+                event_heartbeat_interval: Duration::from_secs(1),
             },
         )
         .unwrap();
@@ -737,6 +741,14 @@ async fn public_routes_are_authenticated_replayable_authorized_and_redacted() {
         .await
         .unwrap();
     assert_eq!(lost_scope.status(), StatusCode::NOT_FOUND);
+    let hidden_submit_replay = harness
+        .request_with_token(reqwest::Method::POST, "/jobs", &unprivileged_token)
+        .header("Idempotency-Key", &submit_key)
+        .json(&submission)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(hidden_submit_replay.status(), StatusCode::NOT_FOUND);
     let hidden_peer_job = harness
         .request_with_token(
             reqwest::Method::GET,
@@ -957,316 +969,6 @@ async fn remove_mutation_record_failure(pool: &PgPool) {
         .unwrap();
 }
 
-async fn revoke_remora_grants() -> Result<(), String> {
-    match required_env("REMORA_OIDC_PROVIDER").as_str() {
-        "keycloak" => revoke_keycloak_remora_roles().await,
-        "zitadel" => revoke_zitadel_remora_roles().await,
-        provider => Err(format!("unsupported OIDC provider {provider}")),
-    }
-}
-
-async fn revoke_keycloak_remora_roles() -> Result<(), String> {
-    let realm = required_env("REMORA_OIDC_TENANT");
-    let client_id = required_env("REMORA_OIDC_CLIENT_ID");
-    let username = required_env("REMORA_OIDC_USERNAME");
-    let client = Client::new();
-    let admin_response = client
-        .post("http://keycloak.remora.test:8080/realms/master/protocol/openid-connect/token")
-        .form(&[
-            ("grant_type", "password"),
-            ("client_id", "admin-cli"),
-            ("username", "remora-admin"),
-            ("password", "synthetic-keycloak"),
-        ])
-        .send()
-        .await
-        .map_err(|error| error.to_string())?;
-    let admin_body: Value = admin_response
-        .json()
-        .await
-        .map_err(|error| error.to_string())?;
-    let admin_token = admin_body["access_token"]
-        .as_str()
-        .ok_or_else(|| "keycloak admin token missing".to_owned())?;
-    let users: Value = client
-        .get(format!(
-            "http://keycloak.remora.test:8080/admin/realms/{realm}/users?username={username}&exact=true"
-        ))
-        .bearer_auth(admin_token)
-        .send()
-        .await
-        .map_err(|error| error.to_string())?
-        .json()
-        .await
-        .map_err(|error| error.to_string())?;
-    let user_id = users
-        .as_array()
-        .and_then(|rows| rows.first())
-        .and_then(|row| row["id"].as_str())
-        .ok_or_else(|| "keycloak user missing".to_owned())?;
-    let clients: Value = client
-        .get(format!(
-            "http://keycloak.remora.test:8080/admin/realms/{realm}/clients?clientId={client_id}"
-        ))
-        .bearer_auth(admin_token)
-        .send()
-        .await
-        .map_err(|error| error.to_string())?
-        .json()
-        .await
-        .map_err(|error| error.to_string())?;
-    let client_uuid = clients
-        .as_array()
-        .and_then(|rows| rows.first())
-        .and_then(|row| row["id"].as_str())
-        .ok_or_else(|| "keycloak client missing".to_owned())?;
-    let roles_url = format!(
-        "http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}"
-    );
-    let mapped: Value = client
-        .get(&roles_url)
-        .bearer_auth(admin_token)
-        .send()
-        .await
-        .map_err(|error| error.to_string())?
-        .json()
-        .await
-        .map_err(|error| error.to_string())?;
-    let roles = mapped
-        .as_array()
-        .cloned()
-        .unwrap_or_default()
-        .into_iter()
-        .filter(|role| {
-            role["name"]
-                .as_str()
-                .is_some_and(|name| name.starts_with("remora:v1:"))
-        })
-        .collect::<Vec<_>>();
-    let response = client
-        .delete(roles_url)
-        .bearer_auth(admin_token)
-        .json(&roles)
-        .send()
-        .await
-        .map_err(|error| error.to_string())?;
-    if response.status().is_success() {
-        Ok(())
-    } else {
-        Err(format!(
-            "keycloak role revocation failed: {}",
-            response.status()
-        ))
-    }
-}
-
-async fn revoke_zitadel_remora_roles() -> Result<(), String> {
-    let issuer = required_env("REMORA_OIDC_ISSUER");
-    let organization = required_env("REMORA_OIDC_TENANT");
-    let user_id = required_env("REMORA_OIDC_USER_ID");
-    let project_id = required_env("REMORA_OIDC_PROJECT_ID");
-    let token = required_env("REMORA_OIDC_MGMT_TOKEN");
-    let host = issuer
-        .strip_prefix("http://")
-        .or_else(|| issuer.strip_prefix("https://"))
-        .unwrap_or(&issuer);
-    let client = Client::new();
-    let grants: Value = client
-        .post(format!("{issuer}/management/v1/users/grants/_search"))
-        .header("Host", host)
-        .bearer_auth(&token)
-        .header("x-zitadel-orgid", &organization)
-        .json(&json!({ "queries": [{ "userIdQuery": { "userId": user_id } }] }))
-        .send()
-        .await
-        .map_err(|error| error.to_string())?
-        .json()
-        .await
-        .map_err(|error| error.to_string())?;
-    let grant_id = grants["result"]
-        .as_array()
-        .and_then(|rows| {
-            rows.iter().find(|row| {
-                row["projectId"]
-                    .as_str()
-                    .is_some_and(|value| value == project_id)
-            })
-        })
-        .and_then(|row| row["id"].as_str())
-        .ok_or_else(|| "zitadel project grant missing".to_owned())?;
-    let response = client
-        .put(format!(
-            "{issuer}/management/v1/users/{user_id}/grants/{grant_id}"
-        ))
-        .header("Host", host)
-        .bearer_auth(token)
-        .header("x-zitadel-orgid", organization)
-        .json(&json!({ "roleKeys": ["test-user"] }))
-        .send()
-        .await
-        .map_err(|error| error.to_string())?;
-    if response.status().is_success() {
-        Ok(())
-    } else {
-        Err(format!(
-            "zitadel role revocation failed: {}",
-            response.status()
-        ))
-    }
-}
-
-fn oidc_provider() -> OidcIdentityProvider {
-    let provider = required_env("REMORA_OIDC_PROVIDER");
-    let audience = if provider == "zitadel" {
-        required_env("REMORA_OIDC_PROJECT_ID")
-    } else {
-        required_env("REMORA_OIDC_CLIENT_ID")
-    };
-    let mut config = OidcConfig::new(required_env("REMORA_OIDC_ISSUER"), audience).unwrap();
-    if provider == "zitadel" {
-        let project_id = required_env("REMORA_OIDC_PROJECT_ID");
-        config = config
-            .with_zitadel_project_roles()
-            .with_project_role_claim(format!("urn:zitadel:iam:org:project:{project_id}:roles"))
-            .with_resource_access_client(required_env("REMORA_OIDC_CLIENT_ID"));
-    } else {
-        config = config.with_resource_access_client(required_env("REMORA_OIDC_CLIENT_ID"));
-    }
-    OidcIdentityProvider::new(config).unwrap()
-}
-
-async fn mint_provider_token() -> Result<String, String> {
-    match required_env("REMORA_OIDC_PROVIDER").as_str() {
-        "keycloak" => mint_keycloak_token().await,
-        "zitadel" => mint_zitadel_token().await,
-        provider => Err(format!("unsupported OIDC provider {provider}")),
-    }
-}
-
-async fn mint_keycloak_token() -> Result<String, String> {
-    let response = Client::new()
-        .post(format!(
-            "{}/protocol/openid-connect/token",
-            required_env("REMORA_OIDC_ISSUER").trim_end_matches('/')
-        ))
-        .form(&[
-            ("grant_type", "password"),
-            ("client_id", required_env("REMORA_OIDC_CLIENT_ID").as_str()),
-            ("username", required_env("REMORA_OIDC_USERNAME").as_str()),
-            ("password", required_env("REMORA_OIDC_PASSWORD").as_str()),
-            ("scope", "openid"),
-        ])
-        .send()
-        .await
-        .map_err(|error| error.to_string())?;
-    token_from_response(response).await
-}
-
-async fn mint_zitadel_token() -> Result<String, String> {
-    let issuer = required_env("REMORA_OIDC_ISSUER")
-        .trim_end_matches('/')
-        .to_owned();
-    let token_url = format!("{issuer}/oauth/v2/token");
-    let encoded_key = required_env("REMORA_OIDC_USER_KEY_B64");
-    let key_json = URL_SAFE_NO_PAD
-        .decode(encoded_key.as_bytes())
-        .or_else(|_| STANDARD.decode(encoded_key.as_bytes()))
-        .map_err(|error| format!("decode user key: {error}"))?;
-    let key: Value =
-        serde_json::from_slice(&key_json).map_err(|error| format!("parse user key: {error}"))?;
-    let key_id = key
-        .get("keyId")
-        .or_else(|| key.get("key_id"))
-        .and_then(Value::as_str)
-        .ok_or_else(|| "user key missing keyId".to_owned())?
-        .to_owned();
-    let subject = key
-        .get("userId")
-        .or_else(|| key.get("userID"))
-        .and_then(Value::as_str)
-        .map(str::to_owned)
-        .unwrap_or_else(|| required_env("REMORA_OIDC_USER_ID"));
-    let private_key = key["key"]
-        .as_str()
-        .ok_or_else(|| "user key missing private key".to_owned())?
-        .replace("\\n", "\n");
-    let now = std::time::SystemTime::now()
-        .duration_since(std::time::UNIX_EPOCH)
-        .map_err(|error| error.to_string())?
-        .as_secs() as i64;
-    let encoding =
-        EncodingKey::from_rsa_pem(private_key.as_bytes()).map_err(|error| error.to_string())?;
-    let project_id = required_env("REMORA_OIDC_PROJECT_ID");
-    let host = issuer
-        .strip_prefix("http://")
-        .or_else(|| issuer.strip_prefix("https://"))
-        .unwrap_or(&issuer);
-    let scope = format!(
-        "openid profile \
-         urn:zitadel:iam:org:project:id:{project_id}:aud \
-         urn:zitadel:iam:org:project:roles \
-         urn:zitadel:iam:org:projects:roles \
-         urn:zitadel:iam:org:project:{project_id}:roles"
-    );
-    let client = Client::new();
-    let mut last_error = String::new();
-    for audience in [&token_url, &issuer] {
-        let mut header = Header::new(Algorithm::RS256);
-        header.kid = Some(key_id.clone());
-        let assertion = encode(
-            &header,
-            &json!({
-                "iss": subject,
-                "sub": subject,
-                "aud": audience,
-                "iat": now,
-                "exp": now + 3600
-            }),
-            &encoding,
-        )
-        .map_err(|error| error.to_string())?;
-        let response = client
-            .post(&token_url)
-            .header("Host", host)
-            .form(&[
-                ("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
-                ("assertion", assertion.as_str()),
-                ("scope", scope.as_str()),
-            ])
-            .send()
-            .await
-            .map_err(|error| error.to_string())?;
-        if response.status().is_success() {
-            return token_from_response(response).await;
-        }
-        last_error = format!(
-            "token endpoint: {} {}",
-            response.status(),
-            response.text().await.unwrap_or_default()
-        );
-    }
-    Err(last_error)
-}
-
-async fn token_from_response(response: reqwest::Response) -> Result<String, String> {
-    if !response.status().is_success() {
-        return Err(format!(
-            "token endpoint: {} {}",
-            response.status(),
-            response.text().await.unwrap_or_default()
-        ));
-    }
-    let body: Value = response.json().await.map_err(|error| error.to_string())?;
-    body.get("access_token")
-        .and_then(Value::as_str)
-        .map(str::to_owned)
-        .ok_or_else(|| "token response missing access_token".into())
-}
-
-fn required_env(name: &str) -> String {
-    std::env::var(name).unwrap_or_else(|_| panic!("{name} is required"))
-}
-
 fn target_store() -> TargetStore {
     let mut config = TargetStoreConfig::from_env().unwrap();
     config.target_store_id = "target-main".into();

crates/remora-api/tests/support/mod.rs

@@ -0,0 +1,318 @@
+use base64::{
+    Engine as _,
+    engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD},
+};
+use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
+use remora_identity::{OidcConfig, OidcIdentityProvider};
+use reqwest::Client;
+use serde_json::{Value, json};
+
+pub fn required_env(name: &str) -> String {
+    std::env::var(name).unwrap_or_else(|_| panic!("{name} is required"))
+}
+
+pub fn oidc_provider() -> OidcIdentityProvider {
+    let provider = required_env("REMORA_OIDC_PROVIDER");
+    let audience = if provider == "zitadel" {
+        required_env("REMORA_OIDC_PROJECT_ID")
+    } else {
+        required_env("REMORA_OIDC_CLIENT_ID")
+    };
+    let mut config = OidcConfig::new(required_env("REMORA_OIDC_ISSUER"), audience).unwrap();
+    if provider == "zitadel" {
+        let project_id = required_env("REMORA_OIDC_PROJECT_ID");
+        config = config
+            .with_zitadel_project_roles()
+            .with_project_role_claim(format!("urn:zitadel:iam:org:project:{project_id}:roles"))
+            .with_resource_access_client(required_env("REMORA_OIDC_CLIENT_ID"));
+    } else {
+        config = config.with_resource_access_client(required_env("REMORA_OIDC_CLIENT_ID"));
+    }
+    OidcIdentityProvider::new(config).unwrap()
+}
+
+pub async fn mint_provider_token() -> Result<String, String> {
+    match required_env("REMORA_OIDC_PROVIDER").as_str() {
+        "keycloak" => mint_keycloak_token().await,
+        "zitadel" => mint_zitadel_token().await,
+        provider => Err(format!("unsupported OIDC provider {provider}")),
+    }
+}
+
+pub async fn revoke_remora_grants() -> Result<(), String> {
+    match required_env("REMORA_OIDC_PROVIDER").as_str() {
+        "keycloak" => revoke_keycloak_remora_roles().await,
+        "zitadel" => revoke_zitadel_remora_roles().await,
+        provider => Err(format!("unsupported OIDC provider {provider}")),
+    }
+}
+
+async fn mint_keycloak_token() -> Result<String, String> {
+    let response = Client::new()
+        .post(format!(
+            "{}/protocol/openid-connect/token",
+            required_env("REMORA_OIDC_ISSUER").trim_end_matches('/')
+        ))
+        .form(&[
+            ("grant_type", "password"),
+            ("client_id", required_env("REMORA_OIDC_CLIENT_ID").as_str()),
+            ("username", required_env("REMORA_OIDC_USERNAME").as_str()),
+            ("password", required_env("REMORA_OIDC_PASSWORD").as_str()),
+            ("scope", "openid"),
+        ])
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    token_from_response(response).await
+}
+
+async fn mint_zitadel_token() -> Result<String, String> {
+    let issuer = required_env("REMORA_OIDC_ISSUER")
+        .trim_end_matches('/')
+        .to_owned();
+    let token_url = format!("{issuer}/oauth/v2/token");
+    let encoded_key = required_env("REMORA_OIDC_USER_KEY_B64");
+    let key_json = URL_SAFE_NO_PAD
+        .decode(encoded_key.as_bytes())
+        .or_else(|_| STANDARD.decode(encoded_key.as_bytes()))
+        .map_err(|error| format!("decode user key: {error}"))?;
+    let key: Value =
+        serde_json::from_slice(&key_json).map_err(|error| format!("parse user key: {error}"))?;
+    let key_id = key
+        .get("keyId")
+        .or_else(|| key.get("key_id"))
+        .and_then(Value::as_str)
+        .ok_or_else(|| "user key missing keyId".to_owned())?
+        .to_owned();
+    let subject = key
+        .get("userId")
+        .or_else(|| key.get("userID"))
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .unwrap_or_else(|| required_env("REMORA_OIDC_USER_ID"));
+    let private_key = key["key"]
+        .as_str()
+        .ok_or_else(|| "user key missing private key".to_owned())?
+        .replace("\\n", "\n");
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map_err(|error| error.to_string())?
+        .as_secs() as i64;
+    let encoding =
+        EncodingKey::from_rsa_pem(private_key.as_bytes()).map_err(|error| error.to_string())?;
+    let project_id = required_env("REMORA_OIDC_PROJECT_ID");
+    let host = issuer
+        .strip_prefix("http://")
+        .or_else(|| issuer.strip_prefix("https://"))
+        .unwrap_or(&issuer);
+    let scope = format!(
+        "openid profile \
+         urn:zitadel:iam:org:project:id:{project_id}:aud \
+         urn:zitadel:iam:org:project:roles \
+         urn:zitadel:iam:org:projects:roles \
+         urn:zitadel:iam:org:project:{project_id}:roles"
+    );
+    let client = Client::new();
+    let mut last_error = String::new();
+    for audience in [&token_url, &issuer] {
+        let mut header = Header::new(Algorithm::RS256);
+        header.kid = Some(key_id.clone());
+        let assertion = encode(
+            &header,
+            &json!({
+                "iss": subject,
+                "sub": subject,
+                "aud": audience,
+                "iat": now,
+                "exp": now + 3600
+            }),
+            &encoding,
+        )
+        .map_err(|error| error.to_string())?;
+        let response = client
+            .post(&token_url)
+            .header("Host", host)
+            .form(&[
+                ("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
+                ("assertion", assertion.as_str()),
+                ("scope", scope.as_str()),
+            ])
+            .send()
+            .await
+            .map_err(|error| error.to_string())?;
+        if response.status().is_success() {
+            return token_from_response(response).await;
+        }
+        last_error = format!(
+            "token endpoint: {} {}",
+            response.status(),
+            response.text().await.unwrap_or_default()
+        );
+    }
+    Err(last_error)
+}
+
+async fn token_from_response(response: reqwest::Response) -> Result<String, String> {
+    if !response.status().is_success() {
+        return Err(format!(
+            "token endpoint: {} {}",
+            response.status(),
+            response.text().await.unwrap_or_default()
+        ));
+    }
+    let body: Value = response.json().await.map_err(|error| error.to_string())?;
+    body.get("access_token")
+        .and_then(Value::as_str)
+        .map(str::to_owned)
+        .ok_or_else(|| "token response missing access_token".into())
+}
+
+async fn revoke_keycloak_remora_roles() -> Result<(), String> {
+    let realm = required_env("REMORA_OIDC_TENANT");
+    let client_id = required_env("REMORA_OIDC_CLIENT_ID");
+    let username = required_env("REMORA_OIDC_USERNAME");
+    let client = Client::new();
+    let admin_response = client
+        .post("http://keycloak.remora.test:8080/realms/master/protocol/openid-connect/token")
+        .form(&[
+            ("grant_type", "password"),
+            ("client_id", "admin-cli"),
+            ("username", "remora-admin"),
+            ("password", "synthetic-keycloak"),
+        ])
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    let admin_body: Value = admin_response
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let admin_token = admin_body["access_token"]
+        .as_str()
+        .ok_or_else(|| "keycloak admin token missing".to_owned())?;
+    let users: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/users?username={username}&exact=true"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let user_id = users
+        .as_array()
+        .and_then(|rows| rows.first())
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "keycloak user missing".to_owned())?;
+    let clients: Value = client
+        .get(format!(
+            "http://keycloak.remora.test:8080/admin/realms/{realm}/clients?clientId={client_id}"
+        ))
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let client_uuid = clients
+        .as_array()
+        .and_then(|rows| rows.first())
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "keycloak client missing".to_owned())?;
+    let roles_url = format!(
+        "http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}"
+    );
+    let mapped: Value = client
+        .get(&roles_url)
+        .bearer_auth(admin_token)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let roles = mapped
+        .as_array()
+        .cloned()
+        .unwrap_or_default()
+        .into_iter()
+        .filter(|role| {
+            role["name"]
+                .as_str()
+                .is_some_and(|name| name.starts_with("remora:v1:"))
+        })
+        .collect::<Vec<_>>();
+    let response = client
+        .delete(roles_url)
+        .bearer_auth(admin_token)
+        .json(&roles)
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    if response.status().is_success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "keycloak role revocation failed: {}",
+            response.status()
+        ))
+    }
+}
+
+async fn revoke_zitadel_remora_roles() -> Result<(), String> {
+    let issuer = required_env("REMORA_OIDC_ISSUER");
+    let organization = required_env("REMORA_OIDC_TENANT");
+    let user_id = required_env("REMORA_OIDC_USER_ID");
+    let project_id = required_env("REMORA_OIDC_PROJECT_ID");
+    let token = required_env("REMORA_OIDC_MGMT_TOKEN");
+    let host = issuer
+        .strip_prefix("http://")
+        .or_else(|| issuer.strip_prefix("https://"))
+        .unwrap_or(&issuer);
+    let client = Client::new();
+    let grants: Value = client
+        .post(format!("{issuer}/management/v1/users/grants/_search"))
+        .header("Host", host)
+        .bearer_auth(&token)
+        .header("x-zitadel-orgid", &organization)
+        .json(&json!({ "queries": [{ "userIdQuery": { "userId": user_id } }] }))
+        .send()
+        .await
+        .map_err(|error| error.to_string())?
+        .json()
+        .await
+        .map_err(|error| error.to_string())?;
+    let grant_id = grants["result"]
+        .as_array()
+        .and_then(|rows| {
+            rows.iter().find(|row| {
+                row["projectId"]
+                    .as_str()
+                    .is_some_and(|value| value == project_id)
+            })
+        })
+        .and_then(|row| row["id"].as_str())
+        .ok_or_else(|| "zitadel project grant missing".to_owned())?;
+    let response = client
+        .put(format!(
+            "{issuer}/management/v1/users/{user_id}/grants/{grant_id}"
+        ))
+        .header("Host", host)
+        .bearer_auth(token)
+        .header("x-zitadel-orgid", organization)
+        .json(&json!({ "roleKeys": ["test-user"] }))
+        .send()
+        .await
+        .map_err(|error| error.to_string())?;
+    if response.status().is_success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "zitadel role revocation failed: {}",
+            response.status()
+        ))
+    }
+}