remora/crates/remora-api/tests/public_api.rs

use std::{net::TcpListener, time::Duration};

use actix_web::{App, HttpServer, web};
use remora_admission::{AdmissionConfig, AdmissionService};
use remora_api::{PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api};
use remora_attempt::{AttemptConfig, AttemptService};
use remora_contracts::{
    AttemptPageDto, DeletionOperationDto, JobPageDto, JobResourceDto, ResultDescriptorDto,
    ResultMemberPageDto, SourceCredentialDto,
};
use remora_coordination::{
    ActivateOptions, CoordinationStore, LeaseOwner, ProgressValue, StoreConfig,
};
use remora_credentials::{CredentialRegistry, VaultClient, VaultConfig};
use remora_deletion::{DeletionConfig, DeletionService, TargetRegistry};
use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
use remora_durable::{PoolConfig, connect};
use remora_identity::DefaultTarget;
use remora_schema::migrate as migrate_schema;
use remora_target::{
    CreateObjectRequest, TargetStore, TargetStoreConfig, attempt_manifest_key_in,
    attempt_object_key_in,
};
use reqwest::{Client, StatusCode};
use serde_json::{Value, json};
use sqlx::PgPool;
use time::OffsetDateTime;

mod support;

use support::{
    mint_provider_token, oidc_provider, required_env, restore_remora_grants, revoke_remora_grants,
};

struct Harness {
    pool: PgPool,
    coordination: CoordinationStore,
    base: String,
    client: Client,
    token: String,
    server: actix_web::dev::ServerHandle,
}

impl Harness {
    async fn new() -> Option<Self> {
        if std::env::var("REMORA_PUBLIC_API_REQUIRE").is_err() {
            return None;
        }
        restore_remora_grants().await.unwrap();
        let pool = connect(
            &required_env("DATABASE_URL"),
            PoolConfig {
                max_connections: 24,
                acquire_timeout: Duration::from_secs(10),
                idle_timeout: Some(Duration::from_secs(30)),
            },
        )
        .await
        .unwrap();
        migrate_schema(&pool).await.unwrap();
        sqlx::query("TRUNCATE TABLE jobs CASCADE")
            .execute(&pool)
            .await
            .unwrap();
        sqlx::query("TRUNCATE TABLE source_credentials CASCADE")
            .execute(&pool)
            .await
            .unwrap();

        let target = target_store();
        let target_store_id =
            TargetStoreId::parse(target.config().target_store_id.clone()).unwrap();
        let bucket = BucketName::parse(target.config().bucket.clone()).unwrap();
        let prefix =
            TargetPrefix::parse(&required_env("REMORA_TARGET_PREFIX"), ["_remora"]).unwrap();
        let admission = AdmissionService::new(
            pool.clone(),
            AdmissionConfig {
                default_target: DefaultTarget {
                    store: target_store_id,
                    bucket,
                    prefix,
                },
                reserved_target_segments: vec!["_remora".into()],
                policy_version: "public-api-v1".into(),
            },
        )
        .unwrap();
        let coordination = CoordinationStore::connect(
            &required_env("REMORA_COORDINATION_URL"),
            format!(
                "{}public-api:{}:",
                required_env("REMORA_COORDINATION_NAMESPACE"),
                nonce()
            ),
            StoreConfig {
                global_limit: 16,
                lease_ttl: Duration::from_secs(10),
                reserve_ttl: Duration::from_secs(10),
                progress_ttl: Duration::from_secs(10),
                cancel_ttl: None,
            },
        )
        .await
        .unwrap();
        let attempts = AttemptService::new(
            pool.clone(),
            coordination.clone(),
            AttemptConfig { max_attempts: 3 },
        )
        .unwrap();
        let vault = VaultClient::new(VaultConfig::from_env().unwrap()).unwrap();
        let credentials = CredentialRegistry::new(pool.clone(), vault);
        let targets = TargetRegistry::new([target]);
        let deletion =
            DeletionService::new(pool.clone(), targets.clone(), DeletionConfig::default()).unwrap();
        let state = PublicApiState::new(
            pool.clone(),
            PublicApiServices {
                identity: oidc_provider(),
                admission,
                attempts,
                credentials,
                deletion,
                targets,
            },
            PublicApiConfig {
                cursor_secret: b"public-api-fixture-cursor-secret".to_vec(),
                member_access_ttl: Duration::from_secs(60),
                event_retention: Duration::from_secs(60),
                event_poll_interval: Duration::from_millis(50),
                event_heartbeat_interval: Duration::from_secs(1),
                event_ticket_ttl: Duration::from_secs(30),
                event_allowed_origins: vec!["https://remora.test".into()],
            },
        )
        .unwrap();
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let address = listener.local_addr().unwrap();
        let server = HttpServer::new(move || {
            App::new()
                .app_data(web::Data::new(state.clone()))
                .configure(configure_public_api)
        })
        .listen(listener)
        .unwrap()
        .run();
        let handle = server.handle();
        tokio::spawn(server);
        let token = mint_provider_token().await.unwrap();
        Some(Self {
            pool,
            coordination,
            base: format!("http://{address}/v1"),
            client: Client::builder()
                .redirect(reqwest::redirect::Policy::none())
                .build()
                .unwrap(),
            token,
            server: handle,
        })
    }

    fn request(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder {
        self.request_with_token(method, path, &self.token)
    }

    fn request_with_token(
        &self,
        method: reqwest::Method,
        path: &str,
        token: &str,
    ) -> reqwest::RequestBuilder {
        self.client
            .request(method, format!("{}{}", self.base, path))
            .bearer_auth(token)
    }
}

impl Drop for Harness {
    fn drop(&mut self) {
        tokio::spawn(self.server.stop(true));
    }
}

#[tokio::test]
async fn public_routes_are_authenticated_replayable_authorized_and_redacted() {
    let Some(harness) = Harness::new().await else {
        return;
    };
    let unauthenticated = harness
        .client
        .get(format!("{}/jobs", harness.base))
        .send()
        .await
        .unwrap();
    assert_eq!(unauthenticated.status(), StatusCode::UNAUTHORIZED);
    assert_eq!(
        unauthenticated.headers()["content-type"],
        "application/problem+json"
    );

    let submission = json!({
        "source": {
            "kind": "http",
            "url": "https://example.invalid/private?token=never-public",
            "result_path": "artifact.bin"
        },
        "label": "public-api-test"
    });
    let submit_key = idempotency_key();
    let submitted = harness
        .request(reqwest::Method::POST, "/jobs")
        .header("Idempotency-Key", &submit_key)
        .json(&submission)
        .send()
        .await
        .unwrap();
    assert_eq!(submitted.status(), StatusCode::ACCEPTED);
    assert!(submitted.headers().contains_key("idempotency-expires-at"));
    let job: Value = submitted.json().await.unwrap();
    serde_json::from_value::<JobResourceDto>(job.clone()).unwrap();
    assert_eq!(job["resource_kind"], "job");
    assert_eq!(job["phase"], "Queued");
    let transcript = serde_json::to_string(&job).unwrap();
    assert!(!transcript.contains("token=never-public"));
    let job_id = job["id"].as_str().unwrap();

    let replay = harness
        .request(reqwest::Method::POST, "/jobs")
        .header("Idempotency-Key", &submit_key)
        .json(&submission)
        .send()
        .await
        .unwrap();
    assert_eq!(replay.status(), StatusCode::ACCEPTED);
    assert_eq!(replay.json::<Value>().await.unwrap(), job);
    let conflict = harness
        .request(reqwest::Method::POST, "/jobs")
        .header("Idempotency-Key", &submit_key)
        .json(&json!({
            "source": {
                "kind": "http",
                "url": "https://example.invalid/changed",
                "result_path": "artifact.bin"
            }
        }))
        .send()
        .await
        .unwrap();
    assert_eq!(conflict.status(), StatusCode::CONFLICT);

    let second = harness
        .request(reqwest::Method::POST, "/jobs")
        .header("Idempotency-Key", idempotency_key())
        .json(&json!({
            "source": {
                "kind": "http",
                "url": "https://example.invalid/second",
                "result_path": "artifact.bin"
            },
            "label": "second-public-api-test"
        }))
        .send()
        .await
        .unwrap();
    assert_eq!(second.status(), StatusCode::ACCEPTED);
    let second_job: Value = second.json().await.unwrap();
    let second_job_id = second_job["id"].as_str().unwrap();

    let listed = harness
        .request(reqwest::Method::GET, "/jobs?limit=1")
        .send()
        .await
        .unwrap();
    assert_eq!(listed.status(), StatusCode::OK);
    let page: Value = listed.json().await.unwrap();
    serde_json::from_value::<JobPageDto>(page.clone()).unwrap();
    assert_eq!(
        page["management_capabilities"]["revoke_execution_grants"],
        true
    );
    let first_page_id = page["items"][0]["id"].as_str().unwrap();
    assert!(page["event_cursor"].as_str().unwrap().len() >= 16);
    let next_cursor = page["next_cursor"].as_str().unwrap();
    let next_page = harness
        .request(
            reqwest::Method::GET,
            &format!("/jobs?limit=1&cursor={next_cursor}"),
        )
        .send()
        .await
        .unwrap();
    assert_eq!(next_page.status(), StatusCode::OK);
    let next_page: Value = next_page.json().await.unwrap();
    serde_json::from_value::<JobPageDto>(next_page.clone()).unwrap();
    let second_page_id = next_page["items"][0]["id"].as_str().unwrap();
    assert_ne!(first_page_id, second_page_id);
    assert_eq!(
        [first_page_id, second_page_id]
            .into_iter()
            .collect::<std::collections::BTreeSet<_>>(),
        [job_id, second_job_id]
            .into_iter()
            .collect::<std::collections::BTreeSet<_>>()
    );
    let cross_filter = harness
        .request(
            reqwest::Method::GET,
            &format!("/jobs?include_deleted=true&cursor={next_cursor}"),
        )
        .send()
        .await
        .unwrap();
    assert_eq!(cross_filter.status(), StatusCode::BAD_REQUEST);
    let altered_cursor = format!("{}x", page["event_cursor"].as_str().unwrap());
    let invalid = harness
        .request(
            reqwest::Method::GET,
            &format!("/jobs?cursor={altered_cursor}"),
        )
        .send()
        .await
        .unwrap();
    assert_eq!(invalid.status(), StatusCode::BAD_REQUEST);

    let fetched = harness
        .request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
        .send()
        .await
        .unwrap();
    assert_eq!(fetched.status(), StatusCode::OK);
    assert_eq!(fetched.json::<Value>().await.unwrap()["id"], job_id);

    install_mutation_record_failure(&harness.pool).await;
    let rolled_back = harness
        .request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
        .header("Idempotency-Key", idempotency_key())
        .send()
        .await
        .unwrap();
    assert_eq!(rolled_back.status(), StatusCode::SERVICE_UNAVAILABLE);
    let phase: String = sqlx::query_scalar("SELECT phase FROM jobs WHERE job_id = $1")
        .bind(job_id)
        .fetch_one(&harness.pool)
        .await
        .unwrap();
    assert_eq!(phase, "queued");
    remove_mutation_record_failure(&harness.pool).await;

    let cancel_key = idempotency_key();
    let cancelled = harness
        .request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
        .header("Idempotency-Key", &cancel_key)
        .send()
        .await
        .unwrap();
    assert_eq!(cancelled.status(), StatusCode::ACCEPTED);
    let cancel_projection: Value = cancelled.json().await.unwrap();
    let cancel_replay = harness
        .request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
        .header("Idempotency-Key", &cancel_key)
        .send()
        .await
        .unwrap();
    assert_eq!(
        cancel_replay.json::<Value>().await.unwrap(),
        cancel_projection
    );

    let credential_key = idempotency_key();
    let secret = "synthetic-secret-never-return";
    let created = harness
        .request(reqwest::Method::POST, "/source-credentials")
        .header("Idempotency-Key", &credential_key)
        .json(&json!({
            "kind": "bearer_token",
            "label": "fixture credential",
            "material": { "token": secret }
        }))
        .send()
        .await
        .unwrap();
    assert_eq!(created.status(), StatusCode::CREATED);
    let credential: Value = created.json().await.unwrap();
    serde_json::from_value::<SourceCredentialDto>(credential.clone()).unwrap();
    assert!(!serde_json::to_string(&credential).unwrap().contains(secret));
    let credential_replay = harness
        .request(reqwest::Method::POST, "/source-credentials")
        .header("Idempotency-Key", &credential_key)
        .json(&json!({
            "kind": "bearer_token",
            "label": "fixture credential",
            "material": { "token": secret }
        }))
        .send()
        .await
        .unwrap();
    assert_eq!(credential_replay.json::<Value>().await.unwrap(), credential);
    let credential_conflict = harness
        .request(reqwest::Method::POST, "/source-credentials")
        .header("Idempotency-Key", &credential_key)
        .json(&json!({
            "kind": "bearer_token",
            "label": "changed fixture credential",
            "material": { "token": secret }
        }))
        .send()
        .await
        .unwrap();
    assert_eq!(credential_conflict.status(), StatusCode::CONFLICT);
    let credential_id = credential["id"].as_str().unwrap();

    let credential_page = harness
        .request(reqwest::Method::GET, "/source-credentials?limit=1")
        .send()
        .await
        .unwrap();
    assert_eq!(credential_page.status(), StatusCode::OK);
    assert_eq!(
        credential_page.json::<Value>().await.unwrap()["items"][0]["id"],
        credential_id
    );
    let fetched_credential = harness
        .request(
            reqwest::Method::GET,
            &format!("/source-credentials/{credential_id}"),
        )
        .send()
        .await
        .unwrap();
    assert_eq!(fetched_credential.status(), StatusCode::OK);
    let rotated = harness
        .request(
            reqwest::Method::POST,
            &format!("/source-credentials/{credential_id}/versions"),
        )
        .header("Idempotency-Key", idempotency_key())
        .json(&json!({ "material": { "token": "rotated-secret" } }))
        .send()
        .await
        .unwrap();
    assert_eq!(rotated.status(), StatusCode::CREATED);

    let hugging_face_key = idempotency_key();
    let hugging_face_request = json!({
        "kind": "hugging_face",
        "label": "fixture Hugging Face credential",
        "material": { "token": "synthetic-hugging-face-token" }
    });
    let lost_create_response = harness
        .request(reqwest::Method::POST, "/source-credentials")
        .header("Idempotency-Key", &hugging_face_key)
        .json(&hugging_face_request)
        .send()
        .await
        .unwrap();
    assert_eq!(lost_create_response.status(), StatusCode::CREATED);
    drop(lost_create_response);
    let hugging_face = harness
        .request(reqwest::Method::POST, "/source-credentials")
        .header("Idempotency-Key", &hugging_face_key)
        .json(&hugging_face_request)
        .send()
        .await
        .unwrap()
        .json::<Value>()
        .await
        .unwrap();
    serde_json::from_value::<SourceCredentialDto>(hugging_face.clone()).unwrap();
    assert_eq!(hugging_face["kind"], "hugging_face");
    let hugging_face_id = hugging_face["id"].as_str().unwrap();
    let rotate_key = idempotency_key();
    let rotate_request = json!({ "material": { "token": "rotated-hugging-face-token" } });
    let lost_rotate_response = harness
        .request(
            reqwest::Method::POST,
            &format!("/source-credentials/{hugging_face_id}/versions"),
        )
        .header("Idempotency-Key", &rotate_key)
        .json(&rotate_request)
        .send()
        .await
        .unwrap();
    assert_eq!(lost_rotate_response.status(), StatusCode::CREATED);
    drop(lost_rotate_response);
    let hugging_face_rotated = harness
        .request(
            reqwest::Method::POST,
            &format!("/source-credentials/{hugging_face_id}/versions"),
        )
        .header("Idempotency-Key", &rotate_key)
        .json(&rotate_request)
        .send()
        .await
        .unwrap();
    assert_eq!(hugging_face_rotated.status(), StatusCode::CREATED);
    let hugging_face_rotated: Value = hugging_face_rotated.json().await.unwrap();
    serde_json::from_value::<SourceCredentialDto>(hugging_face_rotated.clone()).unwrap();
    assert_eq!(hugging_face_rotated["kind"], "hugging_face");
    let rotate_conflict = harness
        .request(
            reqwest::Method::POST,
            &format!("/source-credentials/{hugging_face_id}/versions"),
        )
        .header("Idempotency-Key", &rotate_key)
        .json(&json!({ "material": { "token": "changed-token" } }))
        .send()
        .await
        .unwrap();
    assert_eq!(rotate_conflict.status(), StatusCode::CONFLICT);

    let revoked = harness
        .request(
            reqwest::Method::POST,
            &format!("/source-credentials/{credential_id}/revoke"),
        )
        .header("Idempotency-Key", idempotency_key())
        .send()
        .await
        .unwrap();
    assert_eq!(revoked.status(), StatusCode::OK);
    assert_eq!(revoked.json::<Value>().await.unwrap()["status"], "revoked");

    let (attempt_id, worker_id) = seed_committed_result(&harness.pool, job_id).await;
    seed_failed_attempt(&harness.pool, job_id).await;
    let owner = LeaseOwner::new(attempt_id.as_str(), 1, worker_id.as_str()).unwrap();
    harness.coordination.enqueue(job_id).await.unwrap();
    harness
        .coordination
        .activate(job_id, &owner, ActivateOptions::default())
        .await
        .unwrap();
    harness
        .coordination
        .report_progress_value(
            job_id,
            &owner,
            ProgressValue {
                completed_bytes: 7,
                total_bytes: Some(7),
                completed_members: 1,
                total_members: Some(1),
            },
        )
        .await
        .unwrap();
    let with_progress = harness
        .request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
        .send()
        .await
        .unwrap()
        .json::<Value>()
        .await
        .unwrap();
    assert_eq!(with_progress["progress"]["completed_bytes"], 7);
    assert_eq!(with_progress["progress"]["attempt_id"], attempt_id.as_str());
    harness.coordination.release(job_id, &owner).await.unwrap();
    let foreign = LeaseOwner::new(
        opaque_id("attempt").as_str(),
        2,
        opaque_id("worker").as_str(),
    )
    .unwrap();
    harness
        .coordination
        .activate(job_id, &foreign, ActivateOptions::default())
        .await
        .unwrap();
    let stale_progress = harness
        .request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
        .send()
        .await
        .unwrap()
        .json::<Value>()
        .await
        .unwrap();
    assert!(stale_progress["progress"].is_null());
    let result = harness
        .request(reqwest::Method::GET, &format!("/jobs/{job_id}/result"))
        .send()
        .await
        .unwrap();
    assert_eq!(result.status(), StatusCode::OK);
    let result = result.json::<Value>().await.unwrap();
    serde_json::from_value::<ResultDescriptorDto>(result.clone()).unwrap();
    assert_eq!(result["kind"], "manifest");

    let members = harness
        .request(
            reqwest::Method::GET,
            &format!("/jobs/{job_id}/result/members"),
        )
        .send()
        .await
        .unwrap();
    assert_eq!(members.status(), StatusCode::OK);
    let members: Value = members.json().await.unwrap();
    serde_json::from_value::<ResultMemberPageDto>(members.clone()).unwrap();
    let member_id = members["items"][0]["id"].as_str().unwrap();
    let access = harness
        .request(
            reqwest::Method::GET,
            &format!("/jobs/{job_id}/result/members/{member_id}/content"),
        )
        .send()
        .await
        .unwrap();
    assert_eq!(access.status(), StatusCode::OK);
    assert_eq!(access.headers()["cache-control"], "no-store");
    assert_eq!(access.headers()["pragma"], "no-cache");
    let access: Value = access.json().await.unwrap();
    let access_url = reqwest::Url::parse(access["url"].as_str().unwrap()).unwrap();
    assert!(matches!(access_url.scheme(), "http" | "https"));
    assert!(access["expires_at"].is_string());

    let attempts = harness
        .request(reqwest::Method::GET, &format!("/jobs/{job_id}/attempts"))
        .send()
        .await
        .unwrap();
    assert_eq!(attempts.status(), StatusCode::OK);
    let attempts = attempts.json::<Value>().await.unwrap();
    serde_json::from_value::<AttemptPageDto>(attempts.clone()).unwrap();
    let failed_attempt = attempts["items"]
        .as_array()
        .unwrap()
        .iter()
        .find(|attempt| !attempt["failure"].is_null())
        .unwrap();
    assert_eq!(failed_attempt["failure"]["class"], "internal");
    assert_eq!(failed_attempt["failure"]["message"], "job failed");
    assert!(
        !serde_json::to_string(failed_attempt)
            .unwrap()
            .contains("database password")
    );

    let revoked_job = harness
        .request(reqwest::Method::POST, &format!("/jobs/{job_id}/revoke"))
        .header("Idempotency-Key", idempotency_key())
        .send()
        .await
        .unwrap();
    assert_eq!(revoked_job.status(), StatusCode::ACCEPTED);

    let soft_deleted = harness
        .request(
            reqwest::Method::POST,
            &format!("/jobs/{job_id}/soft-delete"),
        )
        .header("Idempotency-Key", idempotency_key())
        .send()
        .await
        .unwrap();
    assert_eq!(soft_deleted.status(), StatusCode::OK);
    let hidden = harness
        .request(reqwest::Method::GET, &format!("/jobs/{job_id}/result"))
        .send()
        .await
        .unwrap();
    assert_eq!(hidden.status(), StatusCode::NOT_FOUND);
    let restored = harness
        .request(reqwest::Method::POST, &format!("/jobs/{job_id}/restore"))
        .header("Idempotency-Key", idempotency_key())
        .send()
        .await
        .unwrap();
    assert_eq!(restored.status(), StatusCode::OK);

    let hard_deleted = harness
        .request(
            reqwest::Method::POST,
            &format!("/jobs/{job_id}/hard-delete"),
        )
        .header("Idempotency-Key", idempotency_key())
        .send()
        .await
        .unwrap();
    assert_eq!(hard_deleted.status(), StatusCode::ACCEPTED);
    let operation: Value = hard_deleted.json().await.unwrap();
    serde_json::from_value::<DeletionOperationDto>(operation.clone()).unwrap();
    let operation_id = operation["id"].as_str().unwrap();
    let fetched_operation = harness
        .request(reqwest::Method::GET, &format!("/deletions/{operation_id}"))
        .send()
        .await
        .unwrap();
    assert_eq!(fetched_operation.status(), StatusCode::OK);
    let tombstone = harness
        .request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
        .send()
        .await
        .unwrap();
    assert_eq!(tombstone.status(), StatusCode::OK);
    let tombstone = tombstone.json::<Value>().await.unwrap();
    serde_json::from_value::<JobResourceDto>(tombstone.clone()).unwrap();
    assert_eq!(tombstone["resource_kind"], "tombstone");

    let peer_owner = opaque_id("peer-principal");
    sqlx::query("UPDATE jobs SET owner_id = $2 WHERE job_id = $1")
        .bind(second_job_id)
        .bind(peer_owner.as_str())
        .execute(&harness.pool)
        .await
        .unwrap();
    sqlx::query("UPDATE source_credentials SET owner_id = $2 WHERE credential_id = $1")
        .bind(hugging_face_id)
        .bind(peer_owner.as_str())
        .execute(&harness.pool)
        .await
        .unwrap();
    let admin_peer_job = harness
        .request(reqwest::Method::GET, &format!("/jobs/{second_job_id}"))
        .send()
        .await
        .unwrap();
    assert_eq!(admin_peer_job.status(), StatusCode::OK);
    let admin_peer_credential = harness
        .request(
            reqwest::Method::GET,
            &format!("/source-credentials/{hugging_face_id}"),
        )
        .send()
        .await
        .unwrap();
    assert_eq!(admin_peer_credential.status(), StatusCode::OK);

    revoke_remora_grants().await.unwrap();
    let unprivileged_token = mint_provider_token().await.unwrap();
    let lost_scope = harness
        .request_with_token(
            reqwest::Method::GET,
            &format!("/jobs/{job_id}"),
            &unprivileged_token,
        )
        .send()
        .await
        .unwrap();
    assert_eq!(lost_scope.status(), StatusCode::NOT_FOUND);
    let unprivileged_page = harness
        .request_with_token(reqwest::Method::GET, "/jobs", &unprivileged_token)
        .send()
        .await
        .unwrap();
    assert_eq!(unprivileged_page.status(), StatusCode::OK);
    assert_eq!(
        unprivileged_page.json::<Value>().await.unwrap()["management_capabilities"]["revoke_execution_grants"],
        false
    );
    let hidden_submit_replay = harness
        .request_with_token(reqwest::Method::POST, "/jobs", &unprivileged_token)
        .header("Idempotency-Key", &submit_key)
        .json(&submission)
        .send()
        .await
        .unwrap();
    assert_eq!(hidden_submit_replay.status(), StatusCode::NOT_FOUND);
    let hidden_peer_job = harness
        .request_with_token(
            reqwest::Method::GET,
            &format!("/jobs/{second_job_id}"),
            &unprivileged_token,
        )
        .send()
        .await
        .unwrap();
    assert_eq!(hidden_peer_job.status(), StatusCode::NOT_FOUND);
    let hidden_peer_credential = harness
        .request_with_token(
            reqwest::Method::GET,
            &format!("/source-credentials/{hugging_face_id}"),
            &unprivileged_token,
        )
        .send()
        .await
        .unwrap();
    assert_eq!(hidden_peer_credential.status(), StatusCode::NOT_FOUND);
    let invalidated_cursor = harness
        .request_with_token(
            reqwest::Method::GET,
            &format!("/jobs?limit=1&cursor={next_cursor}"),
            &unprivileged_token,
        )
        .send()
        .await
        .unwrap();
    assert_eq!(invalidated_cursor.status(), StatusCode::BAD_REQUEST);
    let forbidden_revoke = harness
        .request_with_token(
            reqwest::Method::POST,
            &format!("/jobs/{second_job_id}/revoke"),
            &unprivileged_token,
        )
        .header("Idempotency-Key", idempotency_key())
        .send()
        .await
        .unwrap();
    assert_eq!(forbidden_revoke.status(), StatusCode::FORBIDDEN);
}

async fn seed_committed_result(pool: &PgPool, job_id: &str) -> (OpaqueId, OpaqueId) {
    let attempt_id = opaque_id("attempt");
    let worker_id = opaque_id("worker");
    let manifest_id = opaque_id("manifest");
    let member_id = opaque_id("member");
    let object_id = opaque_id("object");
    let prefix = TargetPrefix::parse(
        &required_env("REMORA_TARGET_PREFIX"),
        std::iter::empty::<&str>(),
    )
    .unwrap()
    .canonical();
    let member_key = attempt_object_key_in(&prefix, &attempt_id, &object_id, Some("artifact.bin"));
    let manifest_key = attempt_manifest_key_in(&prefix, &attempt_id, &manifest_id);
    let target = target_store();
    let member = target
        .put_create_only(
            &CreateObjectRequest::for_target_attempt_object(
                &prefix,
                &attempt_id,
                &object_id,
                Some("artifact.bin"),
            ),
            b"payload".to_vec(),
        )
        .await
        .unwrap();
    let manifest = target
        .put_create_only(
            &CreateObjectRequest::for_target_attempt_manifest(&prefix, &attempt_id, &manifest_id),
            b"manifest".to_vec(),
        )
        .await
        .unwrap();
    let now = OffsetDateTime::now_utc();
    sqlx::query(
        "INSERT INTO attempts (
            attempt_id, job_id, fence, worker_incarnation, state, initial_progress,
            authorized_at, executing_at, committed_at, staging_origin_attempt_id
         ) VALUES ($1, $2, 1, $3, 'committed', 0, $4, $4, $4, $1)",
    )
    .bind(attempt_id.as_str())
    .bind(job_id)
    .bind(worker_id.as_str())
    .bind(now)
    .execute(pool)
    .await
    .unwrap();
    sqlx::query(
        "INSERT INTO result_inventories (
            attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
            manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
            manifest_size, member_count, total_bytes, staging_origin_attempt_id, target_prefix
         ) VALUES ($1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 1, 7, $1, $9)",
    )
    .bind(attempt_id.as_str())
    .bind(manifest_id.as_str())
    .bind(&manifest.identity.target_store_id)
    .bind(&manifest.identity.bucket)
    .bind(&manifest_key)
    .bind(&manifest.identity.version_id)
    .bind(manifest.content_sha256.as_str())
    .bind(i64::try_from(manifest.size).unwrap())
    .bind(&prefix)
    .execute(pool)
    .await
    .unwrap();
    sqlx::query(
        "INSERT INTO result_inventory_members (
            attempt_id, member_id, path, size, object_store, object_bucket,
            object_key, object_version_id, object_sha256
         ) VALUES ($1, $2, 'artifact.bin', 7, $3, $4, $5, $6, $7)",
    )
    .bind(attempt_id.as_str())
    .bind(member_id.as_str())
    .bind(&member.identity.target_store_id)
    .bind(&member.identity.bucket)
    .bind(&member_key)
    .bind(&member.identity.version_id)
    .bind(member.content_sha256.as_str())
    .execute(pool)
    .await
    .unwrap();
    let committed = json!({
        "attempt_id": attempt_id,
        "manifest_id": manifest_id,
        "schema_version": 1,
        "manifest": {
            "store": manifest.identity.target_store_id,
            "bucket": manifest.identity.bucket,
            "key": manifest.identity.key,
            "version_id": manifest.identity.version_id,
            "sha256": manifest.content_sha256.as_str(),
            "size": manifest.size
        },
        "member_count": 1,
        "total_bytes": 7
    });
    sqlx::query(
        "UPDATE jobs
         SET phase = 'succeeded', lifecycle = 'visible', revision = revision + 1,
             current_fence = 1, authority_attempt_id = $2, committed_result = $3,
             terminal_at = $4, updated_at = $4
         WHERE job_id = $1",
    )
    .bind(job_id)
    .bind(attempt_id.as_str())
    .bind(committed)
    .bind(now)
    .execute(pool)
    .await
    .unwrap();
    (attempt_id, worker_id)
}

async fn seed_failed_attempt(pool: &PgPool, job_id: &str) {
    let attempt_id = opaque_id("failed-attempt");
    let worker_id = opaque_id("failed-worker");
    let now = OffsetDateTime::now_utc();
    sqlx::query(
        "INSERT INTO attempts (
            attempt_id, job_id, fence, worker_incarnation, state, initial_progress,
            failure, authorized_at, executing_at, stopped_at, staging_origin_attempt_id
         ) VALUES ($1, $2, 2, $3, 'stopped', 0, $4, $5, $5, $5, $1)",
    )
    .bind(attempt_id.as_str())
    .bind(job_id)
    .bind(worker_id.as_str())
    .bind(json!({
        "class": "internal",
        "disposition": "retryable",
        "details": {
            "exception": "database password synthetic-secret-never-return"
        }
    }))
    .bind(now)
    .execute(pool)
    .await
    .unwrap();
}

async fn install_mutation_record_failure(pool: &PgPool) {
    sqlx::query(
        "CREATE OR REPLACE FUNCTION remora_test_fail_cancel_record()
         RETURNS trigger LANGUAGE plpgsql AS $$
         BEGIN
           IF NEW.scope_operation = 'job.cancel' THEN
             RAISE EXCEPTION 'synthetic mutation record failure';
           END IF;
           RETURN NEW;
         END
         $$",
    )
    .execute(pool)
    .await
    .unwrap();
    sqlx::query(
        "CREATE TRIGGER remora_test_fail_cancel_record
         BEFORE INSERT ON mutation_records
         FOR EACH ROW EXECUTE FUNCTION remora_test_fail_cancel_record()",
    )
    .execute(pool)
    .await
    .unwrap();
}

async fn remove_mutation_record_failure(pool: &PgPool) {
    sqlx::query("DROP TRIGGER remora_test_fail_cancel_record ON mutation_records")
        .execute(pool)
        .await
        .unwrap();
    sqlx::query("DROP FUNCTION remora_test_fail_cancel_record()")
        .execute(pool)
        .await
        .unwrap();
}

fn target_store() -> TargetStore {
    let mut config = TargetStoreConfig::from_env().unwrap();
    config.target_store_id = "target-main".into();
    TargetStore::connect(config).unwrap()
}

fn opaque_id(prefix: &str) -> OpaqueId {
    OpaqueId::parse(format!("{prefix}{}", nonce())).unwrap()
}

fn idempotency_key() -> String {
    format!("key-{}-replay-safe", nonce())
}

fn nonce() -> String {
    let mut bytes = [0_u8; 8];
    getrandom::fill(&mut bytes).unwrap();
    bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}