use std::{collections::HashSet, time::Duration}; use actix_web::{ HttpRequest, HttpResponse, ResponseError, http::{StatusCode, header}, web, }; use remora_admission::{AdmissionDecision, AdmissionService}; use remora_attempt::{AttemptService, CommittedResultRecord, ProgressSnapshot}; use remora_contracts::{ ApiErrorCodeDto, ApiErrorDto, CredentialMaterialDto, DateTimeDto, JobSubmissionDto, NullableDto, ResultMemberAccessDto, SourceCredentialCreateDto, SourceCredentialDto, SourceCredentialKindDto, SourceCredentialPageDto, SourceCredentialRotateDto, SourceCredentialStatusDto, SourceCredentialVersionDto, }; use remora_credentials::{ CreateCredential, CredentialCaller, CredentialError, CredentialRegistry, RotateTxOutcome, SecretMaterial, SourceCredential, SourceCredentialKind, SourceCredentialStatus, }; use remora_deletion::{ DeletionError, DeletionOperation, DeletionProblemClass, DeletionService, DeletionState, TargetRegistry, }; use remora_domain::{ BucketName, OpaqueId, Target, TargetPrefix, TargetStoreId, fingerprint, ports::MutationScope, }; use remora_durable::{ CursorCodec, DurableError, KeysetCursor, KeysetPosition, MutationDisposition, MutationStatus, PgTx, execute_mutation, first_page_cursor, next_page_cursor, observe_mutation, }; use remora_identity::{ AuthorizedResource, ManagementOperation, OidcIdentityProvider, Principal, may_manage, resolve_principal, }; use remora_result::{CommittedResult, PageSize, PublicationInput, ResultInventory}; use remora_target::{ContentSha256, ObjectIdentity, StoredObject}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use sqlx::{PgPool, types::Json}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; mod events; mod routes; const DEFAULT_LIMIT: u32 = 50; const MAX_LIMIT: u32 = 200; #[derive(Clone)] pub struct PublicApiConfig { pub cursor_secret: Vec, pub member_access_ttl: Duration, pub event_retention: Duration, pub event_poll_interval: Duration, pub event_heartbeat_interval: Duration, pub event_ticket_ttl: Duration, pub event_allowed_origins: Vec, } #[derive(Clone)] pub struct PublicApiState { pool: PgPool, identity: OidcIdentityProvider, admission: AdmissionService, attempts: AttemptService, credentials: CredentialRegistry, deletion: DeletionService, targets: TargetRegistry, results: ResultInventory, cursors: CursorCodec, member_access_ttl: Duration, event_retention: Duration, event_poll_interval: Duration, event_heartbeat_interval: Duration, event_ticket_ttl: Duration, event_allowed_origins: HashSet, } pub struct PublicApiServices { pub identity: OidcIdentityProvider, pub admission: AdmissionService, pub attempts: AttemptService, pub credentials: CredentialRegistry, pub deletion: DeletionService, pub targets: TargetRegistry, } impl PublicApiState { pub fn new( pool: PgPool, services: PublicApiServices, config: PublicApiConfig, ) -> Result { if config.member_access_ttl.is_zero() { return Err(DurableError::Config( "member access expiry must be positive".into(), )); } if config.event_retention.is_zero() || config.event_poll_interval.is_zero() || config.event_heartbeat_interval.is_zero() || config.event_ticket_ttl.is_zero() { return Err(DurableError::Config( "event retention, intervals, and ticket expiry must be positive".into(), )); } if config.event_ticket_ttl > Duration::from_secs(60) { return Err(DurableError::Config( "event ticket expiry must not exceed 60 seconds".into(), )); } let event_allowed_origins = config .event_allowed_origins .into_iter() .collect::>(); if event_allowed_origins.is_empty() || event_allowed_origins .iter() .any(|origin| origin.is_empty() || origin.trim() != origin) { return Err(DurableError::Config( "at least one exact event origin is required".into(), )); } Ok(Self { results: ResultInventory::new(pool.clone()), cursors: CursorCodec::new(config.cursor_secret)?, pool, identity: services.identity, admission: services.admission, attempts: services.attempts, credentials: services.credentials, deletion: services.deletion, targets: services.targets, member_access_ttl: config.member_access_ttl, event_retention: config.event_retention, event_poll_interval: config.event_poll_interval, event_heartbeat_interval: config.event_heartbeat_interval, event_ticket_ttl: config.event_ticket_ttl, event_allowed_origins, }) } async fn principal(&self, request: &HttpRequest) -> Result { let authorization = request .headers() .get(header::AUTHORIZATION) .and_then(|value| value.to_str().ok()) .ok_or_else(PublicApiError::unauthenticated)?; let external = self .identity .authenticate_token(authorization) .await .map_err(|error| match error { remora_identity::IdentityError::ProviderUnavailable(_) => { PublicApiError::unavailable() } _ => PublicApiError::unauthenticated(), })?; resolve_principal(&self.pool, external) .await .map_err(|_| PublicApiError::unavailable()) } } pub fn configure_public_api(config: &mut web::ServiceConfig) { config.service( web::scope("/v1") .app_data( web::JsonConfig::default() .limit(1024 * 1024) .error_handler(|_, _| PublicApiError::invalid_request().into()), ) .configure(routes::jobs) .configure(routes::results) .configure(routes::credentials) .configure(routes::deletions) .configure(routes::events), ); } pub async fn prune_live_event_history( pool: &PgPool, before: OffsetDateTime, ) -> Result { events::prune_history(pool, before).await } #[derive(Deserialize)] struct PageQuery { cursor: Option, limit: Option, } #[derive(Deserialize, Serialize)] struct JobPageQuery { cursor: Option, limit: Option, #[serde(default)] include_deleted: bool, } async fn submit_job( state: web::Data, http: HttpRequest, request: web::Json, ) -> Result { validate_request("JobSubmission", &*request)?; let principal = state.principal(&http).await?; let key = idempotency_key(&http)?; let request = request.into_inner(); let request_fingerprint = fingerprint(&request).map_err(|_| PublicApiError::invalid_request())?; let admission = state.admission.clone(); let state_for_projection = state.get_ref().clone(); let principal_for_effect = principal.clone(); let outcome = execute_mutation( &state.pool, MutationScope { subject: &principal.id, operation: "job.submit", key, }, "submit_job", &[], request_fingerprint, move |tx| { Box::pin(async move { match admission .submit_in_tx(tx, &principal_for_effect, request) .await? { AdmissionDecision::Accepted(accepted) => { let projection = load_authorized_job_in_tx( tx, &state_for_projection, &principal_for_effect, &accepted.job_id, false, ) .await .map_err(|error| DurableError::Effect(error.to_string()))?; Ok((MutationStatus::Accepted, projection)) } AdmissionDecision::Rejected { .. } => Ok(( MutationStatus::Rejected, public_error_body(ApiErrorCodeDto::InvalidRequest, "invalid request"), )), } }) }, ) .await?; if outcome.disposition == MutationDisposition::Conflict { return Err(PublicApiError::idempotency_conflict()); } if outcome.status == MutationStatus::Accepted { let job_id = outcome .response .get("id") .and_then(Value::as_str) .ok_or_else(PublicApiError::internal)?; let job_id = parse_id(job_id.to_owned())?; authorize_job(&state, &principal, &job_id, ManagementOperation::Read).await?; } let response = if outcome.status == MutationStatus::Rejected { HttpResponse::BadRequest() .content_type("application/problem+json") .json(outcome.response) } else { HttpResponse::Accepted().json(outcome.response) }; Ok(with_idempotency_expiry(response)) } async fn get_job( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; let job_id = parse_id(path.into_inner())?; let job = load_authorized_job(&state, &principal, &job_id, true).await?; Ok(HttpResponse::Ok().json(job)) } async fn list_jobs( state: web::Data, http: HttpRequest, query: web::Query, ) -> Result { let principal = state.principal(&http).await?; let limit = page_limit(query.limit)?; let filters = json!({ "resource": "jobs", "include_deleted": query.include_deleted, "authorization": authorization_snapshot(&principal) }); let filter = CursorCodec::filter_fingerprint(&filters)?; let cursor = query .cursor .as_deref() .map(|token| state.cursors.bind(token, principal.id.as_str(), filter)) .transpose() .map_err(PublicApiError::from_cursor)?; let event_cursor = event_cursor(&state, &principal).await?; let mut rows = load_job_page_rows( &state.pool, &principal, query.include_deleted, cursor.as_ref(), limit + 1, ) .await?; let has_more = rows.len() > limit as usize; rows.truncate(limit as usize); let mut items = Vec::with_capacity(rows.len()); for row in &rows { items.push(job_row_projection( row, current_progress(&state, row).await?, &event_cursor, )?); } let next_cursor = if has_more { let last = rows.last().ok_or_else(PublicApiError::internal)?; let last = position(last.created_at, &last.job_id); let cursor = match cursor.as_ref() { Some(current) => next_page_cursor(current, last), None => first_page_cursor( principal.id.as_str(), filter, position(rows[0].created_at, &rows[0].job_id), last, ), }; Some(state.cursors.encode(&cursor)?) } else { None }; Ok(HttpResponse::Ok().json(json!({ "items": items, "next_cursor": next_cursor, "event_cursor": event_cursor, "management_capabilities": { "revoke_execution_grants": principal.administrator } }))) } async fn get_job_result( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; let job_id = parse_id(path.into_inner())?; let row = load_visible_job_row(&state, &principal, &job_id).await?; let result = committed_result(&row)?.ok_or_else(PublicApiError::precondition_failed)?; state.results.resolve_committed(&result).await?; Ok(HttpResponse::Ok().json(result_projection(&result))) } async fn list_result_members( state: web::Data, http: HttpRequest, path: web::Path, query: web::Query, ) -> Result { let principal = state.principal(&http).await?; let job_id = parse_id(path.into_inner())?; let row = load_visible_job_row(&state, &principal, &job_id).await?; let committed = committed_result(&row)?.ok_or_else(PublicApiError::precondition_failed)?; let filter = CursorCodec::filter_fingerprint(&json!({ "resource": "result_members", "job_id": job_id, "authorization": authorization_snapshot(&principal) }))?; let cursor = query .cursor .as_deref() .map(|token| state.cursors.bind(token, principal.id.as_str(), filter)) .transpose() .map_err(PublicApiError::from_cursor)?; let after = cursor .as_ref() .and_then(|value| value.after.as_ref()) .map(|value| value.id.as_str()); let page_size = PageSize::new(page_limit(query.limit)?)?; let page = state .results .page_members(&committed, after, page_size) .await?; let next_cursor = if let Some(after) = page.next_after.as_deref() { let next = KeysetPosition { created_at: row.created_at, id: after.to_owned(), }; let cursor = match cursor.as_ref() { Some(current) => next_page_cursor(current, next), None => first_page_cursor( principal.id.as_str(), filter, KeysetPosition { created_at: row.created_at, id: String::new(), }, next, ), }; Some(state.cursors.encode(&cursor)?) } else { None }; let items = page .items .iter() .map(|member| { json!({ "id": member.id(), "path": member.path(), "size": member.size(), "object": object_identity_projection( member.object().store(), member.object().bucket(), member.object().key(), member.object().version_id(), member.content_sha256().as_str(), member.size() ) }) }) .collect::>(); Ok(HttpResponse::Ok().json(json!({ "items": items, "next_cursor": next_cursor }))) } async fn access_result_member( state: web::Data, http: HttpRequest, path: web::Path<(String, String)>, ) -> Result { let principal = state.principal(&http).await?; let (job_id, member_id) = path.into_inner(); let job_id = parse_id(job_id)?; let member_id = parse_id(member_id)?; let row = load_visible_job_row(&state, &principal, &job_id).await?; let committed = committed_result(&row)?.ok_or_else(PublicApiError::precondition_failed)?; let member = state.results.lookup_member(&committed, &member_id).await?; let target = state .targets .resolve(member.object().store(), member.object().bucket()) .ok_or_else(PublicApiError::unavailable)?; let issued_at = OffsetDateTime::now_utc(); let access = target .presign_read(&member.exact_identity(), state.member_access_ttl) .await .map_err(|_| PublicApiError::unavailable())?; let duration = time::Duration::try_from(access.expires_in).map_err(|_| PublicApiError::internal())?; Ok(HttpResponse::Ok() .insert_header((header::CACHE_CONTROL, "no-store")) .insert_header((header::PRAGMA, "no-cache")) .json(ResultMemberAccessDto { url: access.url, expires_at: datetime(issued_at + duration)?, })) } async fn cancel_job( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; let job_id = parse_id(path.into_inner())?; authorize_job(&state, &principal, &job_id, ManagementOperation::Cancel).await?; let key = idempotency_key(&http)?.to_owned(); let job = idempotent_job_mutation( &state, &principal, &job_id, &key, "job.cancel", JobMutationAction::Cancel, ) .await?; Ok(with_idempotency_expiry(HttpResponse::Accepted().json(job))) } async fn soft_delete_job( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { lifecycle_mutation(state, http, path, LifecycleAction::SoftDelete).await } async fn restore_job( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { lifecycle_mutation(state, http, path, LifecycleAction::Restore).await } enum LifecycleAction { SoftDelete, Restore, } async fn lifecycle_mutation( state: web::Data, http: HttpRequest, path: web::Path, action: LifecycleAction, ) -> Result { let principal = state.principal(&http).await?; let job_id = parse_id(path.into_inner())?; let key = idempotency_key(&http)?.to_owned(); let (operation, management) = match action { LifecycleAction::SoftDelete => ("job.soft_delete", ManagementOperation::SoftDelete), LifecycleAction::Restore => ("job.restore", ManagementOperation::Restore), }; authorize_job(&state, &principal, &job_id, management).await?; let action = match action { LifecycleAction::SoftDelete => JobMutationAction::SoftDelete, LifecycleAction::Restore => JobMutationAction::Restore, }; let job = idempotent_job_mutation(&state, &principal, &job_id, &key, operation, action).await?; Ok(with_idempotency_expiry(HttpResponse::Ok().json(job))) } async fn hard_delete_job( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; let job_id = parse_id(path.into_inner())?; authorize_job(&state, &principal, &job_id, ManagementOperation::HardDelete).await?; let key = idempotency_key(&http)?.to_owned(); let request_fingerprint = fingerprint(&json!({ "job_id": job_id })).map_err(|_| PublicApiError::internal())?; let principal_for_effect = principal.clone(); let job_for_effect = job_id.clone(); let outcome = execute_mutation( &state.pool, MutationScope { subject: &principal.id, operation: "job.hard_delete", key: &key, }, "hard_delete_job", &[], request_fingerprint, move |tx| { Box::pin(async move { let outcome = DeletionService::hard_delete_in_tx(tx, &principal_for_effect, &job_for_effect) .await .map_err(DurableError::effect)?; Ok(( MutationStatus::Accepted, deletion_projection(&outcome.operation) .map_err(|error| DurableError::Effect(error.to_string()))?, )) }) }, ) .await?; if outcome.disposition == MutationDisposition::Conflict { return Err(PublicApiError::idempotency_conflict()); } Ok(with_idempotency_expiry( HttpResponse::Accepted().json(outcome.response), )) } async fn revoke_job( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; if !principal.administrator { return Err(PublicApiError::forbidden()); } let job_id = parse_id(path.into_inner())?; let row = load_job_row(&state.pool, &job_id) .await? .ok_or_else(PublicApiError::not_found)?; let key = idempotency_key(&http)?.to_owned(); let grant_id = parse_id(row.execution_grant_id.clone())?; let job = idempotent_job_mutation( &state, &principal, &job_id, &key, "job.revoke", JobMutationAction::Revoke { grant_id }, ) .await?; Ok(with_idempotency_expiry(HttpResponse::Accepted().json(job))) } async fn list_job_attempts( state: web::Data, http: HttpRequest, path: web::Path, query: web::Query, ) -> Result { let principal = state.principal(&http).await?; let job_id = parse_id(path.into_inner())?; authorize_job(&state, &principal, &job_id, ManagementOperation::Read).await?; let limit = page_limit(query.limit)?; let filter = CursorCodec::filter_fingerprint(&json!({ "resource": "attempts", "job_id": job_id, "authorization": authorization_snapshot(&principal) }))?; let cursor = query .cursor .as_deref() .map(|token| state.cursors.bind(token, principal.id.as_str(), filter)) .transpose() .map_err(PublicApiError::from_cursor)?; let after = cursor .as_ref() .and_then(|value| value.after.as_ref()) .map(|value| value.id.parse::()) .transpose() .map_err(|_| PublicApiError::cursor_invalid())?; let rows = sqlx::query_as::<_, AttemptRow>( "SELECT a.attempt_id, a.fence, a.worker_incarnation, a.state, a.failure, CASE WHEN j.committed_result->>'attempt_id' = a.attempt_id THEN j.committed_result END AS result, a.authorized_at, COALESCE(a.committed_at, a.stopped_at, a.reaped_at, a.executing_at, a.authorized_at) AS updated_at FROM attempts a JOIN jobs j ON j.job_id = a.job_id WHERE a.job_id = $1 AND ($2::bigint IS NULL OR a.fence < $2) ORDER BY a.fence DESC LIMIT $3", ) .bind(job_id.as_str()) .bind(after) .bind(i64::from(limit) + 1) .fetch_all(&state.pool) .await?; let has_more = rows.len() > limit as usize; let visible = rows.iter().take(limit as usize).collect::>(); let progress = state .attempts .progress_snapshot(&job_id) .await .unwrap_or(None); let items = visible .iter() .map(|row| attempt_projection(row, progress.as_ref())) .collect::, _>>()?; let next_cursor = if has_more { let last = visible.last().ok_or_else(PublicApiError::internal)?; let next = KeysetPosition { created_at: last.authorized_at, id: last.fence.to_string(), }; let cursor = match cursor.as_ref() { Some(current) => next_page_cursor(current, next), None => first_page_cursor( principal.id.as_str(), filter, KeysetPosition { created_at: rows[0].authorized_at, id: rows[0].fence.to_string(), }, next, ), }; Some(state.cursors.encode(&cursor)?) } else { None }; Ok(HttpResponse::Ok().json(json!({ "items": items, "next_cursor": next_cursor }))) } async fn create_source_credential( state: web::Data, http: HttpRequest, request: web::Json, ) -> Result { validate_request("SourceCredentialCreate", &*request)?; let principal = state.principal(&http).await?; let key = idempotency_key(&http)?; let request = request.into_inner(); let request_fingerprint = fingerprint(&request).map_err(|_| PublicApiError::invalid_request())?; let create = create_credential(request)?; let credentials = state.credentials.clone(); let owner = principal.id.clone(); let scope = MutationScope { subject: &principal.id, operation: "credential.create", key, }; if let Some(outcome) = observe_mutation( &state.pool, scope, "create_source_credential", &[], request_fingerprint, ) .await? { return credential_mutation_response(outcome, StatusCode::CREATED); } let prepared = credentials .prepare_create(&owner, create) .await .map_err(PublicApiError::from_credential)?; let outcome = execute_mutation( &state.pool, scope, "create_source_credential", &[], request_fingerprint, move |tx| { Box::pin(async move { let credential = CredentialRegistry::commit_prepared_create_in_tx(tx, prepared) .await .map_err(|error| DurableError::Effect(error.to_string()))?; let response = serde_json::to_value( credential_dto(credential) .map_err(|error| DurableError::Effect(error.to_string()))?, ) .map_err(|error| DurableError::Effect(error.to_string()))?; Ok((MutationStatus::Accepted, response)) }) }, ) .await?; credential_mutation_response(outcome, StatusCode::CREATED) } async fn list_source_credentials( state: web::Data, http: HttpRequest, query: web::Query, ) -> Result { let principal = state.principal(&http).await?; let caller = credential_caller(&principal); let filter = CursorCodec::filter_fingerprint(&json!({ "resource": "credentials", "authorization": authorization_snapshot(&principal) }))?; let cursor = query .cursor .as_deref() .map(|token| state.cursors.bind(token, principal.id.as_str(), filter)) .transpose() .map_err(PublicApiError::from_cursor)?; let registry_cursor = cursor .as_ref() .and_then(|value| value.after.as_ref()) .map(|value| value.id.as_str()); let page = state .credentials .list(&caller, page_limit(query.limit)?, registry_cursor) .await .map_err(PublicApiError::from_credential)?; let next_cursor = page .next_cursor .as_deref() .map(|raw| { let next = KeysetPosition { created_at: OffsetDateTime::UNIX_EPOCH, id: raw.to_owned(), }; let bound = match cursor.as_ref() { Some(current) => next_page_cursor(current, next), None => first_page_cursor( principal.id.as_str(), filter, KeysetPosition { created_at: OffsetDateTime::UNIX_EPOCH, id: String::new(), }, next, ), }; state.cursors.encode(&bound) }) .transpose()?; let dto = SourceCredentialPageDto { items: page .items .into_iter() .map(credential_dto) .collect::>()?, next_cursor: NullableDto(next_cursor), }; Ok(HttpResponse::Ok().json(dto)) } async fn get_source_credential( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; let credential_id = parse_id(path.into_inner())?; let credential = state .credentials .get(&credential_caller(&principal), &credential_id) .await .map_err(PublicApiError::from_credential)? .ok_or_else(PublicApiError::not_found)?; Ok(HttpResponse::Ok().json(credential_dto(credential)?)) } async fn rotate_source_credential( state: web::Data, http: HttpRequest, path: web::Path, request: web::Json, ) -> Result { validate_request("SourceCredentialRotate", &*request)?; let principal = state.principal(&http).await?; let credential_id = parse_id(path.into_inner())?; let key = idempotency_key(&http)?; let request = request.into_inner(); let request_fingerprint = fingerprint(&json!({ "credential_id": credential_id, "request": request })) .map_err(|_| PublicApiError::invalid_request())?; let credentials = state.credentials.clone(); let caller = credential_caller(&principal); let scope = MutationScope { subject: &principal.id, operation: "credential.rotate", key, }; if let Some(outcome) = observe_mutation( &state.pool, scope, "rotate_source_credential", &[], request_fingerprint, ) .await? { return credential_mutation_response(outcome, StatusCode::CREATED); } let existing = credentials .get(&caller, &credential_id) .await .map_err(PublicApiError::from_credential)? .ok_or_else(PublicApiError::not_found)?; let material = credential_material_for_kind(request.material, existing.kind); let prepared = credentials .prepare_rotate(&caller, &credential_id, material) .await .map_err(PublicApiError::from_credential)?; let outcome = execute_mutation( &state.pool, scope, "rotate_source_credential", &[], request_fingerprint, move |tx| { Box::pin(async move { match CredentialRegistry::commit_prepared_rotate_in_tx(tx, prepared).await? { RotateTxOutcome::Applied(credential) => { let response = serde_json::to_value( credential_dto(credential) .map_err(|error| DurableError::Effect(error.to_string()))?, ) .map_err(|error| DurableError::Effect(error.to_string()))?; Ok((MutationStatus::Accepted, response)) } RotateTxOutcome::Revoked => Ok(( MutationStatus::Rejected, public_error_body( ApiErrorCodeDto::PreconditionFailed, "precondition failed", ), )), RotateTxOutcome::IntentClaimed => Err(DurableError::Effect( "credential rotation intent is unavailable".into(), )), } }) }, ) .await?; credential_mutation_response(outcome, StatusCode::CREATED) } async fn revoke_source_credential( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; let credential_id = parse_id(path.into_inner())?; let key = idempotency_key(&http)?; let request_fingerprint = fingerprint(&json!({ "credential_id": credential_id })) .map_err(|_| PublicApiError::internal())?; let caller = credential_caller(&principal); let scope = MutationScope { subject: &principal.id, operation: "credential.revoke", key, }; if let Some(outcome) = observe_mutation( &state.pool, scope, "revoke_source_credential", &[], request_fingerprint, ) .await? { return credential_mutation_response(outcome, StatusCode::OK); } state .credentials .get(&caller, &credential_id) .await .map_err(PublicApiError::from_credential)? .ok_or_else(PublicApiError::not_found)?; let credential_for_effect = credential_id.clone(); let outcome = execute_mutation( &state.pool, scope, "revoke_source_credential", &[], request_fingerprint, move |tx| { Box::pin(async move { let credential = CredentialRegistry::revoke_in_tx(tx, &caller, &credential_for_effect) .await .map_err(|error| DurableError::Effect(error.to_string()))?; let response = serde_json::to_value( credential_dto(credential) .map_err(|error| DurableError::Effect(error.to_string()))?, ) .map_err(|error| DurableError::Effect(error.to_string()))?; Ok((MutationStatus::Accepted, response)) }) }, ) .await?; credential_mutation_response(outcome, StatusCode::OK) } async fn get_deletion_operation( state: web::Data, http: HttpRequest, path: web::Path, ) -> Result { let principal = state.principal(&http).await?; let operation_id = parse_id(path.into_inner())?; let operation = state .deletion .operation(&principal, &operation_id) .await .map_err(PublicApiError::from_deletion)?; Ok(HttpResponse::Ok().json(deletion_projection(&operation)?)) } enum JobMutationAction { Cancel, SoftDelete, Restore, Revoke { grant_id: OpaqueId }, } async fn idempotent_job_mutation( state: &PublicApiState, principal: &Principal, job_id: &OpaqueId, key: &str, operation: &'static str, action: JobMutationAction, ) -> Result { let request_fingerprint = fingerprint(&json!({ "job_id": job_id })).map_err(|_| PublicApiError::internal())?; let state_for_projection = state.clone(); let principal_for_projection = principal.clone(); let job_for_projection = job_id.clone(); let outcome = execute_mutation( &state.pool, MutationScope { subject: &principal.id, operation, key, }, operation, &[], request_fingerprint, move |tx| { Box::pin(async move { match action { JobMutationAction::Cancel => { AttemptService::request_cancel_in_tx(tx, &job_for_projection).await?; } JobMutationAction::SoftDelete => { DeletionService::soft_delete_in_tx( tx, &principal_for_projection, &job_for_projection, ) .await .map_err(DurableError::effect)?; } JobMutationAction::Restore => { DeletionService::restore_in_tx( tx, &principal_for_projection, &job_for_projection, ) .await .map_err(DurableError::effect)?; } JobMutationAction::Revoke { grant_id } => { remora_admission::AdmissionService::revoke_execution_grant_in_tx( tx, &principal_for_projection, &grant_id, ) .await?; AttemptService::request_cancel_in_tx(tx, &job_for_projection).await?; } } let projection = load_authorized_job_in_tx( tx, &state_for_projection, &principal_for_projection, &job_for_projection, true, ) .await .map_err(|error| DurableError::Effect(error.to_string()))?; Ok((MutationStatus::Accepted, projection)) }) }, ) .await?; if outcome.disposition == MutationDisposition::Conflict { Err(PublicApiError::idempotency_conflict()) } else { Ok(outcome.response) } } #[derive(sqlx::FromRow)] struct JobRow { job_id: String, owner_id: String, label: Option, revision: i64, phase: String, lifecycle: String, source_projection: Json, target_store: String, target_bucket: String, target_prefix: Json>, execution_grant_id: String, authority_attempt_id: Option, authority_worker_incarnation: Option, current_fence: i64, checkpoint: Option>, committed_result: Option>, failure: Option>, deletion_operation_id: Option, hard_delete_started_at: Option, created_at: OffsetDateTime, updated_at: OffsetDateTime, } async fn load_job_row(pool: &PgPool, job_id: &OpaqueId) -> Result, PublicApiError> { Ok(sqlx::query_as::<_, JobRow>( "SELECT job_id, owner_id, label, revision, phase, lifecycle, source_projection, target_store, target_bucket, target_prefix, execution_grant_id, authority_attempt_id, (SELECT worker_incarnation FROM attempts WHERE attempt_id = jobs.authority_attempt_id) AS authority_worker_incarnation, current_fence, checkpoint, committed_result, failure, deletion_operation_id, hard_delete_started_at, created_at, updated_at FROM jobs WHERE job_id = $1", ) .bind(job_id.as_str()) .fetch_optional(pool) .await?) } async fn load_job_row_in_tx( tx: &mut PgTx<'_>, job_id: &OpaqueId, ) -> Result, PublicApiError> { Ok(sqlx::query_as::<_, JobRow>( "SELECT job_id, owner_id, label, revision, phase, lifecycle, source_projection, target_store, target_bucket, target_prefix, execution_grant_id, authority_attempt_id, (SELECT worker_incarnation FROM attempts WHERE attempt_id = jobs.authority_attempt_id) AS authority_worker_incarnation, current_fence, checkpoint, committed_result, failure, deletion_operation_id, hard_delete_started_at, created_at, updated_at FROM jobs WHERE job_id = $1", ) .bind(job_id.as_str()) .fetch_optional(tx.as_mut()) .await?) } async fn load_authorized_job_in_tx( tx: &mut PgTx<'_>, state: &PublicApiState, principal: &Principal, job_id: &OpaqueId, include_deleted: bool, ) -> Result { let row = load_job_row_in_tx(tx, job_id) .await? .ok_or_else(PublicApiError::not_found)?; if !include_deleted && row.lifecycle != "visible" { return Err(PublicApiError::not_found()); } authorize_row(principal, &row, ManagementOperation::Read)?; let cursor = event_cursor_in_tx(tx, state, principal).await?; job_row_projection(&row, None, &cursor) } async fn load_authorized_job( state: &PublicApiState, principal: &Principal, job_id: &OpaqueId, include_deleted: bool, ) -> Result { let cursor = event_cursor(state, principal).await?; let row = load_job_row(&state.pool, job_id) .await? .ok_or_else(PublicApiError::not_found)?; if !include_deleted && row.lifecycle != "visible" { return Err(PublicApiError::not_found()); } authorize_row(principal, &row, ManagementOperation::Read)?; let progress = current_progress(state, &row).await?; job_row_projection(&row, progress, &cursor) } async fn load_visible_job_row( state: &PublicApiState, principal: &Principal, job_id: &OpaqueId, ) -> Result { let row = load_job_row(&state.pool, job_id) .await? .ok_or_else(PublicApiError::not_found)?; if row.lifecycle != "visible" { return Err(PublicApiError::not_found()); } authorize_row(principal, &row, ManagementOperation::Read)?; Ok(row) } async fn authorize_job( state: &PublicApiState, principal: &Principal, job_id: &OpaqueId, operation: ManagementOperation, ) -> Result { let row = load_job_row(&state.pool, job_id) .await? .ok_or_else(PublicApiError::not_found)?; authorize_row(principal, &row, operation)?; Ok(row) } fn authorize_row( principal: &Principal, row: &JobRow, operation: ManagementOperation, ) -> Result<(), PublicApiError> { let resource = AuthorizedResource { owner: parse_id(row.owner_id.clone())?, target: row_target(row)?, }; if may_manage(principal, &resource, operation) { Ok(()) } else { Err(PublicApiError::not_found()) } } async fn load_job_page_rows( pool: &PgPool, principal: &Principal, include_deleted: bool, cursor: Option<&KeysetCursor>, wanted: u32, ) -> Result, PublicApiError> { let mut after = cursor.and_then(|value| value.after.clone()); let high_water = cursor.map(|value| value.high_water.clone()); let mut visible = Vec::new(); loop { let rows = sqlx::query_as::<_, JobRow>( "SELECT job_id, owner_id, label, revision, phase, lifecycle, source_projection, target_store, target_bucket, target_prefix, execution_grant_id, authority_attempt_id, (SELECT worker_incarnation FROM attempts WHERE attempt_id = jobs.authority_attempt_id) AS authority_worker_incarnation, current_fence, checkpoint, committed_result, failure, deletion_operation_id, hard_delete_started_at, created_at, updated_at FROM jobs WHERE ($1::text IS NULL OR owner_id = $1) AND ($2 OR lifecycle = 'visible') AND ($3::timestamptz IS NULL OR (created_at, job_id) <= ($3, $4)) AND ($5::timestamptz IS NULL OR (created_at, job_id) < ($5, $6)) ORDER BY created_at DESC, job_id DESC LIMIT 200", ) .bind((!principal.administrator).then(|| principal.id.as_str())) .bind(include_deleted) .bind(high_water.as_ref().map(|value| value.created_at)) .bind(high_water.as_ref().map(|value| value.id.as_str())) .bind(after.as_ref().map(|value| value.created_at)) .bind(after.as_ref().map(|value| value.id.as_str())) .fetch_all(pool) .await?; if rows.is_empty() { break; } let count = rows.len(); for row in rows { after = Some(position(row.created_at, &row.job_id)); if authorize_row(principal, &row, ManagementOperation::List).is_ok() { visible.push(row); if visible.len() >= wanted as usize { return Ok(visible); } } } if count < 200 { break; } } Ok(visible) } async fn current_progress( state: &PublicApiState, row: &JobRow, ) -> Result, PublicApiError> { let Some(authority_attempt_id) = row.authority_attempt_id.as_deref() else { return Ok(None); }; let Ok(progress) = state .attempts .progress_snapshot(&parse_id(row.job_id.clone())?) .await else { return Ok(None); }; Ok(progress.filter(|progress| { progress.attempt_id.as_str() == authority_attempt_id && i64::try_from(progress.fence).ok() == Some(row.current_fence) })) } fn progress_projection(progress: &ProgressSnapshot) -> Result { Ok(json!({ "attempt_id": progress.attempt_id, "fence": progress.fence, "completed_bytes": progress.progress.completed_bytes, "total_bytes": progress.progress.total_bytes, "completed_members": progress.progress.completed_members, "total_members": progress.progress.total_members, "observed_at": format_time(progress.observed_at)? })) } fn job_row_projection( row: &JobRow, progress: Option, event_cursor: &str, ) -> Result { let target = json!({ "store": row.target_store, "bucket": row.target_bucket, "prefix": row.target_prefix.0.join("/") }); if row.lifecycle == "hard_deleted" || row.lifecycle == "hard_deleting" { return Ok(json!({ "resource_kind": "tombstone", "id": row.job_id, "owner": row.owner_id, "target": target, "terminal_phase": phase_projection(&row.phase)?, "deletion_operation_id": row.deletion_operation_id, "created_at": format_time(row.created_at)?, "hard_delete_started_at": row.hard_delete_started_at.map(format_time).transpose()?, "updated_at": format_time(row.updated_at)?, "event_cursor": event_cursor })); } let authority = row .authority_attempt_id .as_ref() .zip(row.authority_worker_incarnation.as_ref()) .map(|(attempt_id, worker_incarnation)| { json!({ "attempt_id": attempt_id, "fence": row.current_fence, "worker_incarnation": worker_incarnation }) }); let checkpoint = row .checkpoint .as_ref() .map(|value| checkpoint_projection(&value.0)) .transpose()?; let result = committed_result(row)?.as_ref().map(result_projection); let failure = row .failure .as_ref() .map(|value| failure_projection(&value.0)) .transpose()?; Ok(json!({ "resource_kind": "job", "id": row.job_id, "owner": row.owner_id, "label": row.label, "revision": row.revision, "phase": phase_projection(&row.phase)?, "lifecycle": row.lifecycle, "source": row.source_projection.0, "target": target, "authoritative_attempt": authority, "progress": progress.as_ref().map(progress_projection).transpose()?, "checkpoint": checkpoint, "result": result, "failure": failure, "created_at": format_time(row.created_at)?, "updated_at": format_time(row.updated_at)?, "event_cursor": event_cursor })) } fn phase_projection(value: &str) -> Result<&'static str, PublicApiError> { match value { "queued" => Ok("Queued"), "running" => Ok("Running"), "retry_wait" => Ok("RetryWait"), "cancelling" => Ok("Cancelling"), "succeeded" => Ok("Succeeded"), "failed" => Ok("Failed"), "cancelled" => Ok("Cancelled"), _ => Err(PublicApiError::internal()), } } fn checkpoint_projection(value: &Value) -> Result { let state = value .get("state") .and_then(Value::as_str) .ok_or_else(PublicApiError::internal)?; match state { "reusable" => { let checkpoint = value .get("checkpoint") .ok_or_else(PublicApiError::internal)?; Ok(json!({ "reusable": true, "validated_bytes": checkpoint.get("validated_bytes"), "validated_members": checkpoint.get("validated_members") })) } "complete" => Ok(json!({ "reusable": false, "validated_bytes": value.get("validated_bytes"), "validated_members": value.get("validated_members") })), _ => Err(PublicApiError::internal()), } } fn failure_projection(value: &Value) -> Result { let class = value .get("class") .and_then(Value::as_str) .ok_or_else(PublicApiError::internal)?; let disposition = value .get("disposition") .and_then(Value::as_str) .ok_or_else(PublicApiError::internal)?; Ok(json!({ "class": class, "message": "job failed", "retryable": disposition == "retryable" })) } fn row_target(row: &JobRow) -> Result { Ok(Target { store: TargetStoreId::parse(row.target_store.clone()) .map_err(|_| PublicApiError::internal())?, bucket: BucketName::parse(row.target_bucket.clone()) .map_err(|_| PublicApiError::internal())?, prefix: TargetPrefix::parse(&row.target_prefix.0.join("/"), std::iter::empty()) .map_err(|_| PublicApiError::internal())?, }) } fn committed_result(row: &JobRow) -> Result, PublicApiError> { let Some(value) = row.committed_result.as_ref() else { return Ok(None); }; let record: CommittedResultRecord = serde_json::from_value(value.0.clone()).map_err(|_| PublicApiError::internal())?; Ok(Some(PublicationInput { attempt_id: record.attempt_id, manifest_id: record.manifest_id, schema_version: record.schema_version, manifest: StoredObject { identity: ObjectIdentity { target_store_id: record.manifest.store, bucket: record.manifest.bucket, key: record.manifest.key, version_id: record.manifest.version_id, }, content_sha256: ContentSha256::parse(record.manifest.sha256) .map_err(|_| PublicApiError::internal())?, size: record.manifest.size, }, member_count: record.member_count, total_bytes: record.total_bytes, })) } fn result_projection(result: &CommittedResult) -> Value { json!({ "kind": "manifest", "manifest_id": result.manifest_id, "schema_version": result.schema_version, "manifest": object_identity_projection( &result.manifest.identity.target_store_id, &result.manifest.identity.bucket, &result.manifest.identity.key, result.manifest.identity.version_id.as_deref(), result.manifest.content_sha256.as_str(), result.manifest.size ), "member_count": result.member_count, "total_bytes": result.total_bytes }) } fn object_identity_projection( store: &str, bucket: &str, key: &str, version_id: Option<&str>, checksum: &str, size: u64, ) -> Value { let mut object = json!({ "store": store, "bucket": bucket, "key": key, "checksum": checksum, "size": size }); if let Some(version_id) = version_id { object["version_id"] = json!(version_id); } object } #[derive(sqlx::FromRow)] struct AttemptRow { attempt_id: String, fence: i64, worker_incarnation: String, state: String, failure: Option>, result: Option>, authorized_at: OffsetDateTime, updated_at: OffsetDateTime, } fn attempt_projection( row: &AttemptRow, progress: Option<&ProgressSnapshot>, ) -> Result { let state = match row.state.as_str() { "authorized" => "AttemptAuthorized", "executing" => "AttemptExecuting", "reaped" | "stopped" => "AttemptStopped", "committed" => "AttemptCommitted", _ => return Err(PublicApiError::internal()), }; let result = row .result .as_ref() .map(|value| result_record_projection(&value.0)) .transpose()?; let progress = progress .filter(|progress| { progress.attempt_id.as_str() == row.attempt_id && i64::try_from(progress.fence).ok() == Some(row.fence) }) .map(progress_projection) .transpose()?; Ok(json!({ "id": row.attempt_id, "fence": row.fence, "worker_incarnation": row.worker_incarnation, "state": state, "progress": progress, "result": result, "failure": row .failure .as_ref() .map(|value| failure_projection(&value.0)) .transpose()?, "created_at": format_time(row.authorized_at)?, "updated_at": format_time(row.updated_at)? })) } fn result_record_projection(value: &Value) -> Result { let record: CommittedResultRecord = serde_json::from_value(value.clone()).map_err(|_| PublicApiError::internal())?; Ok(json!({ "kind": "manifest", "manifest_id": record.manifest_id, "schema_version": record.schema_version, "manifest": object_identity_projection( &record.manifest.store, &record.manifest.bucket, &record.manifest.key, record.manifest.version_id.as_deref(), &record.manifest.sha256, record.manifest.size ), "member_count": record.member_count, "total_bytes": record.total_bytes })) } fn create_credential( request: SourceCredentialCreateDto, ) -> Result { let (label, material) = match request { SourceCredentialCreateDto::BasicAuth(value) => ( value.label, SecretMaterial::BasicAuth { username: value.material.username, password: value.material.password, }, ), SourceCredentialCreateDto::BearerToken(value) => ( value.label, SecretMaterial::BearerToken { token: value.material.token, }, ), SourceCredentialCreateDto::HttpHeaders(value) => ( value.label, SecretMaterial::HttpHeaders { headers: value.material.headers, }, ), SourceCredentialCreateDto::ObjectStore(value) => ( value.label, SecretMaterial::ObjectStore { access_key_id: value.material.access_key_id, secret_access_key: value.material.secret_access_key, session_token: value.material.session_token, }, ), SourceCredentialCreateDto::HuggingFace(value) => ( value.label, SecretMaterial::HuggingFace { token: value.material.token, }, ), }; Ok(CreateCredential { label, material }) } fn credential_material(material: CredentialMaterialDto) -> SecretMaterial { match material { CredentialMaterialDto::BasicAuth(value) => SecretMaterial::BasicAuth { username: value.username, password: value.password, }, CredentialMaterialDto::Token(value) => SecretMaterial::BearerToken { token: value.token }, CredentialMaterialDto::HttpHeaders(value) => SecretMaterial::HttpHeaders { headers: value.headers, }, CredentialMaterialDto::ObjectStore(value) => SecretMaterial::ObjectStore { access_key_id: value.access_key_id, secret_access_key: value.secret_access_key, session_token: value.session_token, }, } } fn credential_material_for_kind( material: CredentialMaterialDto, kind: SourceCredentialKind, ) -> SecretMaterial { match (material, kind) { (CredentialMaterialDto::Token(value), SourceCredentialKind::HuggingFace) => { SecretMaterial::HuggingFace { token: value.token } } (material, _) => credential_material(material), } } fn credential_mutation_response( outcome: remora_durable::MutationOutcome, accepted_status: StatusCode, ) -> Result { if outcome.disposition == MutationDisposition::Conflict { return Err(PublicApiError::idempotency_conflict()); } let response = if outcome.status == MutationStatus::Rejected { HttpResponse::PreconditionFailed() .content_type("application/problem+json") .json(outcome.response) } else { HttpResponse::build(accepted_status).json(outcome.response) }; Ok(with_idempotency_expiry(response)) } fn public_error_body(code: ApiErrorCodeDto, message: &'static str) -> Value { json!({ "code": code, "message": message, "request_id": request_id() }) } fn credential_dto(value: SourceCredential) -> Result { let kind = match value.kind { SourceCredentialKind::BasicAuth => SourceCredentialKindDto::BasicAuth, SourceCredentialKind::BearerToken => SourceCredentialKindDto::BearerToken, SourceCredentialKind::HttpHeaders => SourceCredentialKindDto::HttpHeaders, SourceCredentialKind::ObjectStore => SourceCredentialKindDto::ObjectStore, SourceCredentialKind::HuggingFace => SourceCredentialKindDto::HuggingFace, }; let status = match value.status { SourceCredentialStatus::Active => SourceCredentialStatusDto::Active, SourceCredentialStatus::Revoked => SourceCredentialStatusDto::Revoked, }; Ok(SourceCredentialDto { id: value.id, owner: value.owner, label: value.label, kind, status, active_version: NullableDto( value .active_version .map(|version| { Ok::<_, PublicApiError>(SourceCredentialVersionDto { id: version.id, created_at: datetime(version.created_at)?, }) }) .transpose()?, ), created_at: datetime(value.created_at)?, updated_at: datetime(value.updated_at)?, }) } fn credential_caller(principal: &Principal) -> CredentialCaller { CredentialCaller { principal_id: principal.id.clone(), administrator: principal.administrator, } } fn deletion_projection(operation: &DeletionOperation) -> Result { let state = match operation.state { DeletionState::Queued => "queued", DeletionState::Running => "running", DeletionState::RetryWait => "retry_wait", DeletionState::Blocked => "blocked", DeletionState::Completed => "completed", }; let problem = operation.problem.as_ref().map(|problem| { let class = match problem.class { DeletionProblemClass::Authorization => "authorization", DeletionProblemClass::TargetUnavailable => "target_unavailable", DeletionProblemClass::Configuration => "configuration", DeletionProblemClass::Internal => "internal", }; json!({ "class": class, "message": problem.message }) }); Ok(json!({ "id": operation.id, "job_id": operation.job_id, "state": state, "problem": problem, "created_at": format_time(operation.created_at)?, "updated_at": format_time(operation.updated_at)? })) } async fn event_cursor( state: &PublicApiState, principal: &Principal, ) -> Result { let high_water = sqlx::query_as::<_, (i64, OffsetDateTime)>( "SELECT GREATEST( COALESCE((SELECT MAX(id) FROM outbox_messages), 0), history.floor_outbox_id ), now() FROM live_event_history_state history WHERE history.singleton", ) .fetch_one(&state.pool) .await?; encode_event_cursor(state, principal, high_water) } async fn event_cursor_in_tx( tx: &mut PgTx<'_>, state: &PublicApiState, principal: &Principal, ) -> Result { let high_water = sqlx::query_as::<_, (i64, OffsetDateTime)>( "SELECT GREATEST( COALESCE((SELECT MAX(id) FROM outbox_messages), 0), history.floor_outbox_id ), now() FROM live_event_history_state history WHERE history.singleton", ) .fetch_one(tx.as_mut()) .await?; encode_event_cursor(state, principal, high_water) } fn encode_event_cursor( state: &PublicApiState, principal: &Principal, high_water: (i64, OffsetDateTime), ) -> Result { let created_at = high_water.1; let id = format!("event-{}", high_water.0); let filter = CursorCodec::filter_fingerprint(&json!({ "resource": "events", "authorization": authorization_snapshot(principal) }))?; state .cursors .encode(&first_page_cursor( principal.id.as_str(), filter, KeysetPosition { created_at, id: id.clone(), }, KeysetPosition { created_at, id }, )) .map_err(Into::into) } fn position(created_at: OffsetDateTime, id: &str) -> KeysetPosition { KeysetPosition { created_at, id: id.to_owned(), } } fn authorization_snapshot(principal: &Principal) -> Value { let mut scopes = principal .scopes .iter() .map(|scope| { ( scope.store.as_str(), scope.bucket.as_str(), scope.prefix.canonical(), ) }) .collect::>(); scopes.sort_unstable(); scopes.dedup(); json!({ "administrator": principal.administrator, "scopes": scopes }) } fn page_limit(limit: Option) -> Result { let limit = limit.unwrap_or(DEFAULT_LIMIT); if !(1..=MAX_LIMIT).contains(&limit) { return Err(PublicApiError::invalid_request()); } Ok(limit) } fn parse_id(value: impl Into) -> Result { OpaqueId::parse(value).map_err(|_| PublicApiError::invalid_request()) } fn idempotency_key(request: &HttpRequest) -> Result<&str, PublicApiError> { let value = request .headers() .get("Idempotency-Key") .and_then(|value| value.to_str().ok()) .ok_or_else(PublicApiError::invalid_request)?; if !(16..=128).contains(&value.len()) || !value .bytes() .all(|byte| byte.is_ascii_alphanumeric() || b"._~-".contains(&byte)) { return Err(PublicApiError::invalid_request()); } Ok(value) } fn with_idempotency_expiry(mut response: HttpResponse) -> HttpResponse { let expiry = OffsetDateTime::now_utc() + time::Duration::hours(24); if let Ok(value) = expiry.format(&Rfc3339) && let Ok(value) = header::HeaderValue::from_str(&value) { response.headers_mut().insert( header::HeaderName::from_static("idempotency-expires-at"), value, ); } response } fn format_time(value: OffsetDateTime) -> Result { value .format(&Rfc3339) .map_err(|_| PublicApiError::internal()) } fn datetime(value: OffsetDateTime) -> Result { DateTimeDto::parse(format_time(value)?).map_err(|_| PublicApiError::internal()) } #[derive(Debug)] struct PublicApiError { status: StatusCode, code: ApiErrorCodeDto, message: &'static str, } impl PublicApiError { fn invalid_request() -> Self { Self::new( StatusCode::BAD_REQUEST, ApiErrorCodeDto::InvalidRequest, "invalid request", ) } fn unauthenticated() -> Self { Self::new( StatusCode::UNAUTHORIZED, ApiErrorCodeDto::Unauthenticated, "authentication required", ) } fn forbidden() -> Self { Self::new( StatusCode::FORBIDDEN, ApiErrorCodeDto::Forbidden, "operation forbidden", ) } fn not_found() -> Self { Self::new( StatusCode::NOT_FOUND, ApiErrorCodeDto::NotFound, "resource not found", ) } fn idempotency_conflict() -> Self { Self::new( StatusCode::CONFLICT, ApiErrorCodeDto::IdempotencyConflict, "idempotency key conflict", ) } fn precondition_failed() -> Self { Self::new( StatusCode::CONFLICT, ApiErrorCodeDto::PreconditionFailed, "resource precondition failed", ) } fn cursor_invalid() -> Self { Self::new( StatusCode::BAD_REQUEST, ApiErrorCodeDto::CursorInvalid, "invalid cursor", ) } fn unavailable() -> Self { Self::new( StatusCode::SERVICE_UNAVAILABLE, ApiErrorCodeDto::Unavailable, "service unavailable", ) } fn internal() -> Self { Self::new( StatusCode::INTERNAL_SERVER_ERROR, ApiErrorCodeDto::Internal, "internal error", ) } fn new(status: StatusCode, code: ApiErrorCodeDto, message: &'static str) -> Self { Self { status, code, message, } } fn from_cursor(error: DurableError) -> Self { match error { DurableError::InvalidCursor | DurableError::CursorBindingMismatch => { Self::cursor_invalid() } _ => Self::unavailable(), } } fn from_credential(error: CredentialError) -> Self { match error { CredentialError::NotFound => Self::not_found(), CredentialError::InvalidInput(_) | CredentialError::KindMismatch => { Self::invalid_request() } CredentialError::Revoked | CredentialError::NoActiveVersion => { Self::precondition_failed() } CredentialError::SecretStoreUnavailable => Self::unavailable(), _ => Self::unavailable(), } } fn from_deletion(error: DeletionError) -> Self { match error { DeletionError::NotFound => Self::not_found(), DeletionError::NotTerminal | DeletionError::HardDeleteStarted => { Self::precondition_failed() } DeletionError::CorruptInventory => Self::precondition_failed(), _ => Self::unavailable(), } } } impl std::fmt::Display for PublicApiError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter.write_str(self.message) } } impl ResponseError for PublicApiError { fn status_code(&self) -> StatusCode { self.status } fn error_response(&self) -> HttpResponse { HttpResponse::build(self.status) .content_type("application/problem+json") .json(ApiErrorDto { code: self.code, message: self.message.to_owned(), request_id: request_id(), fields: None, }) } } impl From for PublicApiError { fn from(_: sqlx::Error) -> Self { Self::unavailable() } } impl From for PublicApiError { fn from(error: DurableError) -> Self { Self::from_cursor(error) } } impl From for PublicApiError { fn from(error: remora_result::ResultError) -> Self { match error { remora_result::ResultError::MemberNotFound => Self::not_found(), remora_result::ResultError::NotSealed => Self::precondition_failed(), _ => Self::unavailable(), } } } fn request_id() -> OpaqueId { let mut bytes = [0_u8; 16]; getrandom::fill(&mut bytes).expect("operating system randomness is required"); let mut value = String::from("r"); for byte in bytes { use std::fmt::Write as _; write!(value, "{byte:02x}").expect("writing to String cannot fail"); } OpaqueId::parse(value).expect("random request id is valid") } fn validate_request(schema: &'static str, request: &impl Serialize) -> Result<(), PublicApiError> { remora_contracts::request_is_valid(schema, request) .then_some(()) .ok_or_else(PublicApiError::invalid_request) } #[cfg(test)] mod tests { use remora_domain::StorageScope; use super::*; #[test] fn cursor_authorization_snapshot_is_order_independent_and_change_sensitive() { let scope = |prefix: &str| StorageScope { store: TargetStoreId::parse("target").unwrap(), bucket: BucketName::parse("bucket").unwrap(), prefix: TargetPrefix::parse(prefix, std::iter::empty()).unwrap(), }; let principal = |administrator, scopes| Principal { id: OpaqueId::parse("principal").unwrap(), issuer: "https://issuer.example".into(), subject: "subject".into(), administrator, scopes, }; let first = principal(false, vec![scope("b"), scope("a")]); let reordered = principal(false, vec![scope("a"), scope("b")]); let duplicated = principal(false, vec![scope("a"), scope("b"), scope("a")]); let expanded = principal(false, vec![scope("a"), scope("b"), scope("c")]); let administrator = principal(true, vec![scope("a"), scope("b")]); assert_eq!( authorization_snapshot(&first), authorization_snapshot(&reordered) ); assert_eq!( authorization_snapshot(&first), authorization_snapshot(&duplicated) ); assert_ne!( authorization_snapshot(&first), authorization_snapshot(&expanded) ); assert_ne!( authorization_snapshot(&reordered), authorization_snapshot(&administrator) ); let codec = CursorCodec::new(b"public-api-test-cursor-secret".to_vec()).unwrap(); let filter = CursorCodec::filter_fingerprint(&authorization_snapshot(&first)).unwrap(); let position = KeysetPosition { created_at: OffsetDateTime::UNIX_EPOCH, id: "job".into(), }; let token = codec .encode(&first_page_cursor( first.id.as_str(), filter, position.clone(), position, )) .unwrap(); assert!(codec.bind(&token, "different-subject", filter).is_err()); let expanded_filter = CursorCodec::filter_fingerprint(&authorization_snapshot(&expanded)).unwrap(); assert!( codec .bind(&token, first.id.as_str(), expanded_filter) .is_err() ); } }