ms2k/remora
SHA256
Watch
0
Code Releases Activity

Implement replay-safe authorized public HTTP API

Browse source
This commit is contained in:
ms2k 2026-07-17 21:23:18 +09:00
commit 4c93c25692
4 changed files with 3940 additions and 345 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/remora-attempt/src/lib.rs
View file

@ -12,7 +12,7 @@ pub use migrate::migrate;
pub use mutation_types::{
AttemptMutationOutcome, CheckpointMultipart, CheckpointProjection, CheckpointRecord,
CommittedManifestRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
JobMutationProjection, MutationRejection, ProgressRecord,
JobMutationProjection, MutationRejection, ProgressRecord, ProgressSnapshot,
};
pub use recovery_types::{
CancelOutcome, CleanupClaim, CleanupCompletion, CleanupIdentity, CooperativeStopOutcome,

8
crates/remora-attempt/src/mutation_types.rs
View file

@ -59,6 +59,14 @@ pub struct ProgressRecord {
pub total_members: Option<u64>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ProgressSnapshot {
pub attempt_id: OpaqueId,
pub fence: u64,
pub progress: ProgressRecord,
pub observed_at: time::OffsetDateTime,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MutationRejection {

98
crates/remora-attempt/src/recovery.rs
View file

@ -5,7 +5,8 @@ use remora_domain::{
FailureClass, FailureDisposition, JobPhase, OpaqueId, fingerprint, ports::MutationScope,
};
use remora_durable::{
DurableError, MutationDisposition, MutationStatus, enqueue, execute_mutation, with_transaction,
DurableError, MutationDisposition, MutationStatus, PgTx, enqueue, execute_mutation,
with_transaction,
};
use remora_target::{attempt_key_attempt_id, attempt_staging_prefix, validate_attempt_key};
use serde::Serialize;
@ -24,56 +25,61 @@ impl AttemptService {
pub async fn request_cancel(&self, job_id: &OpaqueId) -> Result<CancelOutcome, AttemptError> {
with_transaction(&self.pool, |tx| {
let job_id = job_id.clone();
Box::pin(async move {
let Some(row) = sqlx::query_as::<_, CancelRow>(
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
)
.bind(job_id.as_str())
.fetch_optional(tx.as_mut())
.await?
else {
return Ok(CancelOutcome::NotFound);
};
let phase = parse_phase(&row.phase)?;
if phase.is_terminal() {
return Ok(CancelOutcome::AlreadyTerminal);
}
if row.cancel_requested {
return Ok(CancelOutcome::AlreadyRequested);
}
let now = OffsetDateTime::now_utc();
if phase == JobPhase::Running {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
} else {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
authority_attempt_id = NULL, dispatch_needed = FALSE,
terminal_at = $2, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
}
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(CancelOutcome::Requested)
})
Box::pin(async move { Self::request_cancel_in_tx(tx, &job_id).await })
})
.await
.map_err(Into::into)
}
pub async fn request_cancel_in_tx(
tx: &mut PgTx<'_>,
job_id: &OpaqueId,
) -> Result<CancelOutcome, DurableError> {
let Some(row) = sqlx::query_as::<_, CancelRow>(
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
)
.bind(job_id.as_str())
.fetch_optional(tx.as_mut())
.await?
else {
return Ok(CancelOutcome::NotFound);
};
let phase = parse_phase(&row.phase)?;
if phase.is_terminal() {
return Ok(CancelOutcome::AlreadyTerminal);
}
if row.cancel_requested {
return Ok(CancelOutcome::AlreadyRequested);
}
let now = OffsetDateTime::now_utc();
if phase == JobPhase::Running {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
} else {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
authority_attempt_id = NULL, dispatch_needed = FALSE,
terminal_at = $2, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
}
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(CancelOutcome::Requested)
}
pub async fn cooperative_stop(
&self,
authority: &AttemptAuthority,

25
crates/remora-attempt/src/service.rs
View file

@ -16,8 +16,8 @@ use time::OffsetDateTime;
use crate::{
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptError,
AttemptFaultInjector, AttemptFaultPoint, AttemptState, CheckpointProjection, ClaimOutcome,
ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, RecoveryOutcome,
RenewalMutationOutcome, RenewalOutcome,
ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, ProgressRecord, ProgressSnapshot,
RecoveryOutcome, RenewalMutationOutcome, RenewalOutcome,
};
struct NoFaults;
@ -87,6 +87,27 @@ impl AttemptService {
})
}
pub async fn progress_snapshot(
&self,
job_id: &OpaqueId,
) -> Result<Option<ProgressSnapshot>, AttemptError> {
let Some(hint) = self.coordination.get_progress(job_id.as_str()).await? else {
return Ok(None);
};
Ok(Some(ProgressSnapshot {
attempt_id: OpaqueId::parse(hint.attempt_id)
.map_err(|error| AttemptError::Invalid(error.to_string()))?,
fence: hint.fence,
progress: ProgressRecord {
completed_bytes: hint.bytes,
total_bytes: hint.total_bytes,
completed_members: hint.completed_members,
total_members: hint.total_members,
},
observed_at: OffsetDateTime::now_utc(),
}))
}
pub async fn claim(
&self,
job_id: &OpaqueId,