ms2k/remora
SHA256
Watch
0
Code Releases Activity

Implement authorized live event replay

Browse source
This commit is contained in:
ms2k 2026-07-17 23:08:28 +09:00
commit 06a7ac56b7
1 changed files with 2370 additions and 364 deletions
Download patch file Download diff file Expand all files Collapse all files

148
crates/remora-api/src/public.rs
View file

@ -5,7 +5,7 @@ use actix_web::{
http::{StatusCode, header},
web,
};
use remora_admission::{AdmissionDecision, AdmissionError, AdmissionService};
use remora_admission::{AdmissionDecision, AdmissionService};
use remora_attempt::{AttemptService, CommittedResultRecord, ProgressSnapshot};
use remora_contracts::{
ApiErrorCodeDto, ApiErrorDto, CredentialMaterialDto, DateTimeDto, JobSubmissionDto,
@ -39,6 +39,8 @@ use serde_json::{Value, json};
use sqlx::{PgPool, types::Json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
mod events;
const DEFAULT_LIMIT: u32 = 50;
const MAX_LIMIT: u32 = 200;
@ -46,6 +48,9 @@ const MAX_LIMIT: u32 = 200;
pub struct PublicApiConfig {
pub cursor_secret: Vec<u8>,
pub member_access_ttl: Duration,
pub event_retention: Duration,
pub event_poll_interval: Duration,
pub event_heartbeat_interval: Duration,
}
#[derive(Clone)]
@ -60,6 +65,9 @@ pub struct PublicApiState {
results: ResultInventory,
cursors: CursorCodec,
member_access_ttl: Duration,
event_retention: Duration,
event_poll_interval: Duration,
event_heartbeat_interval: Duration,
}
pub struct PublicApiServices {
@ -82,6 +90,14 @@ impl PublicApiState {
"member access expiry must be positive".into(),
));
}
if config.event_retention.is_zero()
|| config.event_poll_interval.is_zero()
|| config.event_heartbeat_interval.is_zero()
{
return Err(DurableError::Config(
"event retention and intervals must be positive".into(),
));
}
Ok(Self {
results: ResultInventory::new(pool.clone()),
cursors: CursorCodec::new(config.cursor_secret)?,
@ -93,6 +109,9 @@ impl PublicApiState {
deletion: services.deletion,
targets: services.targets,
member_access_ttl: config.member_access_ttl,
event_retention: config.event_retention,
event_poll_interval: config.event_poll_interval,
event_heartbeat_interval: config.event_heartbeat_interval,
})
}
@ -173,10 +192,18 @@ pub fn configure_public_api(config: &mut web::ServiceConfig) {
.route(
"/deletions/{operation_id}",
web::get().to(get_deletion_operation),
),
)
.route("/events", web::get().to(events::connect)),
);
}
pub async fn prune_live_event_history(
pool: &PgPool,
before: OffsetDateTime,
) -> Result<u64, DurableError> {
events::prune_history(pool, before).await
}
#[derive(Deserialize)]
struct PageQuery {
cursor: Option<String>,
@ -198,17 +225,69 @@ async fn submit_job(
) -> Result<HttpResponse, PublicApiError> {
let principal = state.principal(&http).await?;
let key = idempotency_key(&http)?;
let outcome = state
.admission
.submit(&principal, key, request.into_inner())
.await
.map_err(PublicApiError::from_admission)?;
let accepted = match outcome.decision {
AdmissionDecision::Accepted(accepted) => accepted,
AdmissionDecision::Rejected { .. } => return Err(PublicApiError::invalid_request()),
let request = request.into_inner();
let request_fingerprint =
fingerprint(&request).map_err(|_| PublicApiError::invalid_request())?;
let admission = state.admission.clone();
let state_for_projection = state.get_ref().clone();
let principal_for_effect = principal.clone();
let outcome = execute_mutation(
&state.pool,
MutationScope {
subject: &principal.id,
operation: "job.submit",
key,
},
"submit_job",
&[],
request_fingerprint,
move |tx| {
Box::pin(async move {
match admission
.submit_in_tx(tx, &principal_for_effect, request)
.await?
{
AdmissionDecision::Accepted(accepted) => {
let projection = load_authorized_job_in_tx(
tx,
&state_for_projection,
&principal_for_effect,
&accepted.job_id,
false,
)
.await
.map_err(|error| DurableError::Effect(error.to_string()))?;
Ok((MutationStatus::Accepted, projection))
}
AdmissionDecision::Rejected { .. } => Ok((
MutationStatus::Rejected,
public_error_body(ApiErrorCodeDto::InvalidRequest, "invalid request"),
)),
}
})
},
)
.await?;
if outcome.disposition == MutationDisposition::Conflict {
return Err(PublicApiError::idempotency_conflict());
}
if outcome.status == MutationStatus::Accepted {
let job_id = outcome
.response
.get("id")
.and_then(Value::as_str)
.ok_or_else(PublicApiError::internal)?;
let job_id = parse_id(job_id.to_owned())?;
authorize_job(&state, &principal, &job_id, ManagementOperation::Read).await?;
}
let response = if outcome.status == MutationStatus::Rejected {
HttpResponse::BadRequest()
.content_type("application/problem+json")
.json(outcome.response)
} else {
HttpResponse::Accepted().json(outcome.response)
};
let job = load_authorized_job(&state, &principal, &accepted.job_id, false).await?;
Ok(with_idempotency_expiry(HttpResponse::Accepted().json(job)))
Ok(with_idempotency_expiry(response))
}
async fn get_job(
@ -241,6 +320,7 @@ async fn list_jobs(
.map(|token| state.cursors.bind(token, principal.id.as_str(), filter))
.transpose()
.map_err(PublicApiError::from_cursor)?;
let event_cursor = event_cursor(&state, &principal).await?;
let mut rows = load_job_page_rows(
&state.pool,
&principal,
@ -251,7 +331,6 @@ async fn list_jobs(
.await?;
let has_more = rows.len() > limit as usize;
rows.truncate(limit as usize);
let event_cursor = event_cursor(&state, &principal).await?;
let mut items = Vec::with_capacity(rows.len());
for row in &rows {
items.push(job_row_projection(
@ -1073,6 +1152,7 @@ async fn load_authorized_job(
job_id: &OpaqueId,
include_deleted: bool,
) -> Result<Value, PublicApiError> {
let cursor = event_cursor(state, principal).await?;
let row = load_job_row(&state.pool, job_id)
.await?
.ok_or_else(PublicApiError::not_found)?;
@ -1080,7 +1160,6 @@ async fn load_authorized_job(
return Err(PublicApiError::not_found());
}
authorize_row(principal, &row, ManagementOperation::Read)?;
let cursor = event_cursor(state, principal).await?;
let progress = current_progress(state, &row).await?;
job_row_projection(&row, progress, &cursor)
}
@ -1654,11 +1733,16 @@ async fn event_cursor(
principal: &Principal,
) -> Result<String, PublicApiError> {
let high_water = sqlx::query_as::<_, (i64, OffsetDateTime)>(
"SELECT id, created_at FROM outbox_messages ORDER BY id DESC LIMIT 1",
"SELECT GREATEST(
COALESCE((SELECT MAX(id) FROM outbox_messages), 0),
history.floor_outbox_id
),
now()
FROM live_event_history_state history
WHERE history.singleton",
)
.fetch_optional(&state.pool)
.await?
.unwrap_or((0, OffsetDateTime::UNIX_EPOCH));
.fetch_one(&state.pool)
.await?;
encode_event_cursor(state, principal, high_water)
}
@ -1668,11 +1752,16 @@ async fn event_cursor_in_tx(
principal: &Principal,
) -> Result<String, PublicApiError> {
let high_water = sqlx::query_as::<_, (i64, OffsetDateTime)>(
"SELECT id, created_at FROM outbox_messages ORDER BY id DESC LIMIT 1",
"SELECT GREATEST(
COALESCE((SELECT MAX(id) FROM outbox_messages), 0),
history.floor_outbox_id
),
now()
FROM live_event_history_state history
WHERE history.singleton",
)
.fetch_optional(tx.as_mut())
.await?
.unwrap_or((0, OffsetDateTime::UNIX_EPOCH));
.fetch_one(tx.as_mut())
.await?;
encode_event_cursor(state, principal, high_water)
}
@ -1721,6 +1810,7 @@ fn authorization_snapshot(principal: &Principal) -> Value {
})
.collect::<Vec<_>>();
scopes.sort_unstable();
scopes.dedup();
json!({
"administrator": principal.administrator,
"scopes": scopes
@ -1875,15 +1965,6 @@ impl PublicApiError {
}
}
fn from_admission(error: AdmissionError) -> Self {
match error {
AdmissionError::IdempotencyConflict => Self::idempotency_conflict(),
AdmissionError::Unauthorized => Self::forbidden(),
AdmissionError::Invalid(_) => Self::invalid_request(),
_ => Self::unavailable(),
}
}
fn from_credential(error: CredentialError) -> Self {
match error {
CredentialError::NotFound => Self::not_found(),
@ -1988,6 +2069,7 @@ mod tests {
};
let first = principal(false, vec![scope("b"), scope("a")]);
let reordered = principal(false, vec![scope("a"), scope("b")]);
let duplicated = principal(false, vec![scope("a"), scope("b"), scope("a")]);
let expanded = principal(false, vec![scope("a"), scope("b"), scope("c")]);
let administrator = principal(true, vec![scope("a"), scope("b")]);
@ -1995,6 +2077,10 @@ mod tests {
authorization_snapshot(&first),
authorization_snapshot(&reordered)
);
assert_eq!(
authorization_snapshot(&first),
authorization_snapshot(&duplicated)
);
assert_ne!(
authorization_snapshot(&first),
authorization_snapshot(&expanded)