Add PostgreSQL DurableStore substrate for WP-03

Introduce remora-durable with pool/migrations, a RemoraApiMutation-shaped
ledger, claim-fenced outbox projection, CAS helpers, and keyset cursors,
plus a kind Job acceptance suite against the hermetic fixture.
This commit is contained in:
ms2k 2026-07-16 03:56:44 +09:00
commit 957d170150
9 changed files with 3203 additions and 18 deletions

View file

@ -0,0 +1,90 @@
use sqlx::{AssertSqlSafe, PgPool};
use crate::error::DurableError;
use crate::tx::{self, PgTx};
/// Optimistic compare-and-swap on a caller-owned table.
///
/// The table must expose `revision BIGINT NOT NULL` and a primary key column
/// bound as `$1`. Feature packages own the surrounding schema; this helper only
/// serializes the revision bump. Identifiers are restricted to lowercase SQL
/// identifiers before interpolation.
pub async fn cas_set_text(
tx: &mut PgTx<'_>,
table: &str,
id_column: &str,
id: &str,
value_column: &str,
value: &str,
expected_revision: i64,
) -> Result<i64, DurableError> {
validate_ident(table)?;
validate_ident(id_column)?;
validate_ident(value_column)?;
let sql = format!(
"UPDATE {table}
SET {value_column} = $2,
revision = revision + 1
WHERE {id_column} = $1
AND revision = $3
RETURNING revision"
);
let next: Option<(i64,)> = sqlx::query_as(AssertSqlSafe(sql))
.bind(id)
.bind(value)
.bind(expected_revision)
.fetch_optional(tx.as_mut())
.await?;
next.map(|(revision,)| revision)
.ok_or(DurableError::CasConflict)
}
pub async fn cas_set_text_pool(
pool: &PgPool,
table: &str,
id_column: &str,
id: &str,
value_column: &str,
value: &str,
expected_revision: i64,
) -> Result<i64, DurableError> {
let table = table.to_owned();
let id_column = id_column.to_owned();
let id = id.to_owned();
let value_column = value_column.to_owned();
let value = value.to_owned();
tx::with_transaction(pool, move |tx| {
Box::pin(async move {
cas_set_text(
tx,
&table,
&id_column,
&id,
&value_column,
&value,
expected_revision,
)
.await
})
})
.await
}
fn validate_ident(value: &str) -> Result<(), DurableError> {
if value.is_empty()
|| value.len() > 63
|| !value.chars().all(|character| {
character.is_ascii_lowercase() || character.is_ascii_digit() || character == '_'
})
|| value
.chars()
.next()
.is_none_or(|character| !(character.is_ascii_lowercase() || character == '_'))
{
return Err(DurableError::Config(format!(
"invalid SQL identifier '{value}'"
)));
}
Ok(())
}

View file

@ -0,0 +1,231 @@
use base64::Engine;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use hmac::{Hmac, KeyInit, Mac};
use remora_domain::fingerprint;
use serde::{Deserialize, Serialize};
use sha2::Sha256;
use time::OffsetDateTime;
use crate::error::DurableError;
type HmacSha256 = Hmac<Sha256>;
/// Stable keyset order for job-style listings: `(created_at DESC, id DESC)`.
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct KeysetPosition {
#[serde(with = "time::serde::rfc3339")]
pub created_at: OffsetDateTime,
pub id: String,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
struct CursorPayload {
subject: String,
filter_fingerprint: String,
#[serde(with = "time::serde::rfc3339")]
high_water_created_at: OffsetDateTime,
high_water_id: String,
after: Option<KeysetPosition>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct KeysetCursor {
pub subject: String,
pub filter_fingerprint: [u8; 32],
pub high_water: KeysetPosition,
pub after: Option<KeysetPosition>,
}
#[derive(Clone, Debug)]
pub struct CursorCodec {
secret: Vec<u8>,
}
impl CursorCodec {
pub fn new(secret: impl Into<Vec<u8>>) -> Result<Self, DurableError> {
let secret = secret.into();
if secret.len() < 16 {
return Err(DurableError::Config(
"cursor integrity secret must be at least 16 bytes".into(),
));
}
Ok(Self { secret })
}
pub fn filter_fingerprint(filters: &impl Serialize) -> Result<[u8; 32], DurableError> {
let digest = fingerprint(filters).map_err(|_| {
DurableError::Config("filters cannot be represented as canonical JSON".into())
})?;
Ok(*digest.as_bytes())
}
pub fn encode(&self, cursor: &KeysetCursor) -> Result<String, DurableError> {
let payload = CursorPayload {
subject: cursor.subject.clone(),
filter_fingerprint: hex::encode(cursor.filter_fingerprint),
high_water_created_at: cursor.high_water.created_at,
high_water_id: cursor.high_water.id.clone(),
after: cursor.after.clone(),
};
let body = serde_json::to_vec(&payload).map_err(|error| {
DurableError::Config(format!("cursor payload is not JSON-serializable: {error}"))
})?;
let mut mac = HmacSha256::new_from_slice(&self.secret)
.map_err(|error| DurableError::Config(format!("invalid cursor secret: {error}")))?;
mac.update(&body);
let tag = mac.finalize().into_bytes();
let mut framed = Vec::with_capacity(body.len() + tag.len());
framed.extend_from_slice(&body);
framed.extend_from_slice(&tag);
Ok(URL_SAFE_NO_PAD.encode(framed))
}
pub fn decode(&self, token: &str) -> Result<KeysetCursor, DurableError> {
let framed = URL_SAFE_NO_PAD
.decode(token.as_bytes())
.map_err(|_| DurableError::InvalidCursor)?;
if framed.len() <= 32 {
return Err(DurableError::InvalidCursor);
}
let (body, tag) = framed.split_at(framed.len() - 32);
let mut mac = HmacSha256::new_from_slice(&self.secret)
.map_err(|error| DurableError::Config(format!("invalid cursor secret: {error}")))?;
mac.update(body);
mac.verify_slice(tag)
.map_err(|_| DurableError::InvalidCursor)?;
let payload: CursorPayload =
serde_json::from_slice(body).map_err(|_| DurableError::InvalidCursor)?;
let filter_fingerprint = parse_hex32(&payload.filter_fingerprint)?;
Ok(KeysetCursor {
subject: payload.subject,
filter_fingerprint,
high_water: KeysetPosition {
created_at: payload.high_water_created_at,
id: payload.high_water_id,
},
after: payload.after,
})
}
pub fn bind(
&self,
token: &str,
subject: &str,
filter_fingerprint: [u8; 32],
) -> Result<KeysetCursor, DurableError> {
let cursor = self.decode(token)?;
if cursor.subject != subject || cursor.filter_fingerprint != filter_fingerprint {
return Err(DurableError::CursorBindingMismatch);
}
Ok(cursor)
}
}
/// Build a first-page cursor after returning one page.
///
/// `high_water` is the newest visible row at snapshot time (membership bound).
/// `last_on_page` is the last row of the returned page and becomes the exclusive
/// keyset lower bound for the next page. For a single-item page the two
/// positions are equal.
pub fn first_page_cursor(
subject: impl Into<String>,
filter_fingerprint: [u8; 32],
high_water: KeysetPosition,
last_on_page: KeysetPosition,
) -> KeysetCursor {
KeysetCursor {
subject: subject.into(),
filter_fingerprint,
high_water,
after: Some(last_on_page),
}
}
/// Advance a keyset cursor after returning `last` on the current page.
pub fn next_page_cursor(current: &KeysetCursor, last: KeysetPosition) -> KeysetCursor {
KeysetCursor {
subject: current.subject.clone(),
filter_fingerprint: current.filter_fingerprint,
high_water: current.high_water.clone(),
after: Some(last),
}
}
fn parse_hex32(value: &str) -> Result<[u8; 32], DurableError> {
if value.len() != 64 {
return Err(DurableError::InvalidCursor);
}
let mut out = [0_u8; 32];
for (index, slot) in out.iter_mut().enumerate() {
*slot = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16)
.map_err(|_| DurableError::InvalidCursor)?;
}
Ok(out)
}
// Minimal hex encode without an extra dependency.
mod hex {
pub fn encode(bytes: [u8; 32]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(64);
for byte in bytes {
output.push(char::from(HEX[usize::from(byte >> 4)]));
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
output
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use time::macros::datetime;
#[test]
fn round_trips_and_binds_subject_and_filters() {
let codec = CursorCodec::new(b"test-cursor-secret").unwrap();
let filters = json!({"bucket": "alpha", "lifecycle": "visible"});
let fingerprint = CursorCodec::filter_fingerprint(&filters).unwrap();
let newest = KeysetPosition {
created_at: datetime!(2026-07-16 12:00:00 UTC),
id: "job-zzzzzzzz".into(),
};
let last = KeysetPosition {
created_at: datetime!(2026-07-16 11:00:00 UTC),
id: "job-aaaaaaaa".into(),
};
let cursor = first_page_cursor("subject-001", fingerprint, newest, last);
let token = codec.encode(&cursor).unwrap();
let decoded = codec.bind(&token, "subject-001", fingerprint).unwrap();
assert_eq!(decoded, cursor);
assert!(matches!(
codec.bind(&token, "other-subject", fingerprint),
Err(DurableError::CursorBindingMismatch)
));
let other = CursorCodec::filter_fingerprint(&json!({"bucket": "beta"})).unwrap();
assert!(matches!(
codec.bind(&token, "subject-001", other),
Err(DurableError::CursorBindingMismatch)
));
}
#[test]
fn rejects_tampered_tokens() {
let codec = CursorCodec::new(b"test-cursor-secret").unwrap();
let fingerprint = [7_u8; 32];
let position = KeysetPosition {
created_at: datetime!(2026-07-16 12:00:00 UTC),
id: "job-aaaaaaaa".into(),
};
let cursor = first_page_cursor("subject-001", fingerprint, position.clone(), position);
let mut token = codec.encode(&cursor).unwrap();
token.replace_range(0..1, if token.starts_with('A') { "B" } else { "A" });
assert!(matches!(
codec.decode(&token),
Err(DurableError::InvalidCursor)
));
}
}

View file

@ -0,0 +1,41 @@
use std::fmt;
#[derive(Debug, thiserror::Error)]
pub enum DurableError {
#[error("database error: {0}")]
Database(#[from] sqlx::Error),
#[error("migration error: {0}")]
Migrate(#[from] sqlx::migrate::MigrateError),
/// Transport-layer mapping of a ledger conflict disposition.
///
/// The substrate ledger returns
/// `MutationOutcome { disposition: Conflict, … }` for RemoraApiMutation
/// conflict observations. Higher layers may convert that outcome into this
/// error when an HTTP or worker adapter wants an error path.
#[error("mutation conflict for subject={subject} operation={operation} key={key}")]
MutationConflict {
subject: String,
operation: String,
key: String,
},
#[error("mutation effect failed: {0}")]
Effect(String),
#[error("outbox claim does not own message {message_id}")]
OutboxClaimMismatch { message_id: i64 },
#[error("outbox message {message_id} is missing")]
OutboxMissing { message_id: i64 },
#[error("invalid keyset cursor")]
InvalidCursor,
#[error("keyset cursor binding mismatch")]
CursorBindingMismatch,
#[error("compare-and-swap precondition failed")]
CasConflict,
#[error("configuration error: {0}")]
Config(String),
}
impl DurableError {
pub fn effect(error: impl fmt::Display) -> Self {
Self::Effect(error.to_string())
}
}

View file

@ -0,0 +1,260 @@
use remora_domain::RequestFingerprint;
use remora_domain::ports::MutationScope;
use serde_json::Value;
use sha2::{Digest, Sha256};
use sqlx::PgPool;
use crate::error::DurableError;
use crate::tx::{self, BoxFuture, PgTx};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MutationDisposition {
First,
Replay,
Conflict,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum MutationStatus {
Accepted,
Rejected,
}
impl MutationStatus {
fn as_str(self) -> &'static str {
match self {
Self::Accepted => "accepted",
Self::Rejected => "rejected",
}
}
fn parse(value: &str) -> Result<Self, DurableError> {
match value {
"accepted" => Ok(Self::Accepted),
"rejected" => Ok(Self::Rejected),
other => Err(DurableError::Config(format!(
"unknown mutation status '{other}'"
))),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct MutationOutcome {
pub disposition: MutationDisposition,
pub status: MutationStatus,
pub response: Value,
}
#[derive(Clone, Debug)]
struct StoredRecord {
kind: String,
eligibility_claim: Vec<u8>,
request_fingerprint: Vec<u8>,
response_status: String,
response_body: Value,
}
/// Execute a mutation refining `RemoraApiMutation`.
///
/// Identity is `(subject, operation, key)`. Exact kind, eligibility claim, and
/// request fingerprint replay the stored response. A change to any of those
/// fields is a conflict and leaves the record unchanged. The eligibility claim
/// binds attempt, fence, worker incarnation, or other authority material
/// outside the semantic payload fingerprint.
///
/// Concurrent first observations for the same identity serialize on a
/// transaction-scoped advisory lock before eligibility or effects run.
pub async fn execute_mutation<D>(
pool: &PgPool,
scope: MutationScope<'_>,
kind: &str,
eligibility_claim: &[u8],
request_fingerprint: RequestFingerprint,
decide: D,
) -> Result<MutationOutcome, DurableError>
where
D: for<'c> FnOnce(
&'c mut PgTx<'_>,
) -> BoxFuture<'c, Result<(MutationStatus, Value), DurableError>>
+ Send
+ 'static,
{
let subject = scope.subject.clone();
let operation = scope.operation.to_owned();
let key = scope.key.to_owned();
let kind = kind.to_owned();
let eligibility_claim = eligibility_claim.to_owned();
tx::with_transaction(pool, move |tx| {
Box::pin(async move {
let scope = MutationScope {
subject: &subject,
operation: &operation,
key: &key,
};
execute_mutation_in_tx(
tx,
scope,
&kind,
&eligibility_claim,
request_fingerprint,
decide,
)
.await
})
})
.await
}
pub async fn execute_mutation_in_tx<D>(
tx: &mut PgTx<'_>,
scope: MutationScope<'_>,
kind: &str,
eligibility_claim: &[u8],
request_fingerprint: RequestFingerprint,
decide: D,
) -> Result<MutationOutcome, DurableError>
where
D: for<'c> FnOnce(
&'c mut PgTx<'_>,
) -> BoxFuture<'c, Result<(MutationStatus, Value), DurableError>>
+ Send,
{
tx::set_lock_timeout(tx).await?;
// Serialize empty-row first observations: FOR UPDATE only locks existing
// rows, so concurrent first writers need an exclusive identity claim first.
tx::advisory_xact_lock(tx, identity_lock_key(scope)).await?;
let fingerprint = request_fingerprint.as_bytes().as_slice();
if let Some(existing) = load_record(tx, scope).await? {
return observe_existing(kind, eligibility_claim, fingerprint, existing);
}
let (status, response) = decide(tx).await?;
let effect_applied = status == MutationStatus::Accepted;
insert_record(
tx,
&MutationInsert {
scope,
kind,
eligibility_claim,
fingerprint,
status,
response_body: &response,
effect_applied,
},
)
.await?;
Ok(MutationOutcome {
disposition: MutationDisposition::First,
status,
response,
})
}
struct MutationInsert<'a> {
scope: MutationScope<'a>,
kind: &'a str,
eligibility_claim: &'a [u8],
fingerprint: &'a [u8],
status: MutationStatus,
response_body: &'a Value,
effect_applied: bool,
}
fn identity_lock_key(scope: MutationScope<'_>) -> i64 {
let mut hasher = Sha256::new();
hasher.update(scope.subject.as_str().as_bytes());
hasher.update([0xff]);
hasher.update(scope.operation.as_bytes());
hasher.update([0xff]);
hasher.update(scope.key.as_bytes());
let digest = hasher.finalize();
i64::from_be_bytes([
digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7],
])
}
fn observe_existing(
kind: &str,
eligibility_claim: &[u8],
fingerprint: &[u8],
existing: StoredRecord,
) -> Result<MutationOutcome, DurableError> {
if existing.kind != kind
|| existing.eligibility_claim != eligibility_claim
|| existing.request_fingerprint != fingerprint
{
return Ok(MutationOutcome {
disposition: MutationDisposition::Conflict,
status: MutationStatus::Rejected,
response: Value::Null,
});
}
let status = MutationStatus::parse(&existing.response_status)?;
Ok(MutationOutcome {
disposition: MutationDisposition::Replay,
status,
response: existing.response_body,
})
}
async fn load_record(
tx: &mut PgTx<'_>,
scope: MutationScope<'_>,
) -> Result<Option<StoredRecord>, DurableError> {
let row = sqlx::query_as::<_, (String, Vec<u8>, Vec<u8>, String, Value)>(
"SELECT kind, eligibility_claim, request_fingerprint, response_status, response_body
FROM mutation_records
WHERE scope_subject = $1
AND scope_operation = $2
AND operation_key = $3
FOR UPDATE",
)
.bind(scope.subject.as_str())
.bind(scope.operation)
.bind(scope.key)
.fetch_optional(tx.as_mut())
.await?;
Ok(row.map(
|(kind, eligibility_claim, request_fingerprint, response_status, response_body)| {
StoredRecord {
kind,
eligibility_claim,
request_fingerprint,
response_status,
response_body,
}
},
))
}
async fn insert_record(tx: &mut PgTx<'_>, record: &MutationInsert<'_>) -> Result<(), DurableError> {
sqlx::query(
"INSERT INTO mutation_records (
scope_subject,
scope_operation,
operation_key,
kind,
eligibility_claim,
request_fingerprint,
response_status,
response_body,
effect_applied
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
)
.bind(record.scope.subject.as_str())
.bind(record.scope.operation)
.bind(record.scope.key)
.bind(record.kind)
.bind(record.eligibility_claim)
.bind(record.fingerprint)
.bind(record.status.as_str())
.bind(record.response_body)
.bind(record.effect_applied)
.execute(tx.as_mut())
.await?;
Ok(())
}

View file

@ -0,0 +1,29 @@
//! PostgreSQL DurableStore substrate.
//!
//! Provides the pool, migration runner, transaction helpers, generic mutation
//! ledger refining `RemoraApiMutation`, durable outbox claim/projection, and
//! integrity-protected keyset cursor primitives. Feature packages own their
//! domain schema additions.
mod cas;
mod cursor;
mod error;
mod ledger;
mod migrate;
mod outbox;
mod pool;
mod tx;
pub use cas::{cas_set_text, cas_set_text_pool};
pub use cursor::{CursorCodec, KeysetCursor, KeysetPosition, first_page_cursor, next_page_cursor};
pub use error::DurableError;
pub use ledger::{
MutationDisposition, MutationOutcome, MutationStatus, execute_mutation, execute_mutation_in_tx,
};
pub use migrate::{migrate, migrator};
pub use outbox::{
OutboxClaim, OutboxClaimFilter, OutboxMessage, claim_batch, claim_batch_in_tx, enqueue,
enqueue_after, mark_projected, mark_projected_pool, release_with_error, unprojected_count,
};
pub use pool::{PoolConfig, connect};
pub use tx::{BoxFuture, PgTx, advisory_xact_lock, set_lock_timeout, with_transaction};

View file

@ -0,0 +1,14 @@
use sqlx::PgPool;
use crate::error::DurableError;
static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations");
pub async fn migrate(pool: &PgPool) -> Result<(), DurableError> {
MIGRATOR.run(pool).await?;
Ok(())
}
pub fn migrator() -> &'static sqlx::migrate::Migrator {
&MIGRATOR
}

View file

@ -0,0 +1,294 @@
use serde_json::Value;
use sqlx::PgPool;
use time::OffsetDateTime;
use crate::error::DurableError;
use crate::tx::{self, PgTx};
/// Ownership token returned by claim; required for projection and release.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct OutboxClaim {
pub message_id: i64,
pub worker_id: String,
pub generation: i64,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct OutboxMessage {
pub id: i64,
pub topic: String,
pub payload: Value,
pub created_at: OffsetDateTime,
pub available_at: OffsetDateTime,
pub attempt_count: i32,
pub claim: OutboxClaim,
}
/// Selects which outbox rows a claim may take.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum OutboxClaimFilter<'a> {
/// Any claimable topic.
Any,
/// Exact topic match.
Topic(&'a str),
}
pub async fn enqueue(tx: &mut PgTx<'_>, topic: &str, payload: Value) -> Result<i64, DurableError> {
let (id,): (i64,) = sqlx::query_as(
"INSERT INTO outbox_messages (topic, payload, available_at)
VALUES ($1, $2, now())
RETURNING id",
)
.bind(topic)
.bind(payload)
.fetch_one(tx.as_mut())
.await?;
Ok(id)
}
/// Enqueue work available after a delay measured by the database clock.
pub async fn enqueue_after(
tx: &mut PgTx<'_>,
topic: &str,
payload: Value,
delay: time::Duration,
) -> Result<i64, DurableError> {
let delay_secs = delay.as_seconds_f64().max(0.0);
let (id,): (i64,) = sqlx::query_as(
"INSERT INTO outbox_messages (topic, payload, available_at)
VALUES ($1, $2, now() + ($3 * interval '1 second'))
RETURNING id",
)
.bind(topic)
.bind(payload)
.bind(delay_secs)
.fetch_one(tx.as_mut())
.await?;
Ok(id)
}
/// Claim claimable outbox rows for exclusive projection using
/// `FOR UPDATE SKIP LOCKED`. Lease eligibility and expiry use the database
/// clock. Each successful claim advances `claim_generation` and returns an
/// ownership token required for projection or release.
pub async fn claim_batch(
pool: &PgPool,
worker_id: &str,
filter: OutboxClaimFilter<'_>,
limit: i64,
lease: time::Duration,
) -> Result<Vec<OutboxMessage>, DurableError> {
let worker_id = worker_id.to_owned();
let topic = match filter {
OutboxClaimFilter::Any => None,
OutboxClaimFilter::Topic(topic) => Some(topic.to_owned()),
};
tx::with_transaction(pool, move |tx| {
Box::pin(async move {
let filter = match topic.as_deref() {
None => OutboxClaimFilter::Any,
Some(topic) => OutboxClaimFilter::Topic(topic),
};
claim_batch_in_tx(tx, &worker_id, filter, limit, lease).await
})
})
.await
}
pub async fn claim_batch_in_tx(
tx: &mut PgTx<'_>,
worker_id: &str,
filter: OutboxClaimFilter<'_>,
limit: i64,
lease: time::Duration,
) -> Result<Vec<OutboxMessage>, DurableError> {
let lease_secs = lease.as_seconds_f64().max(0.0);
let topic = match filter {
OutboxClaimFilter::Any => None,
OutboxClaimFilter::Topic(topic) => Some(topic),
};
let rows = sqlx::query_as::<
_,
(
i64,
String,
Value,
OffsetDateTime,
OffsetDateTime,
i32,
i64,
String,
),
>(
"WITH candidates AS (
SELECT id
FROM outbox_messages
WHERE projected_at IS NULL
AND available_at <= now()
AND (claimed_until IS NULL OR claimed_until < now())
AND ($4::text IS NULL OR topic = $4)
ORDER BY id
FOR UPDATE SKIP LOCKED
LIMIT $1
)
UPDATE outbox_messages AS messages
SET claimed_by = $2,
claimed_until = now() + ($3 * interval '1 second'),
claim_generation = messages.claim_generation + 1,
attempt_count = messages.attempt_count + 1
FROM candidates
WHERE messages.id = candidates.id
RETURNING
messages.id,
messages.topic,
messages.payload,
messages.created_at,
messages.available_at,
messages.attempt_count,
messages.claim_generation,
messages.claimed_by",
)
.bind(limit)
.bind(worker_id)
.bind(lease_secs)
.bind(topic)
.fetch_all(tx.as_mut())
.await?;
Ok(rows
.into_iter()
.map(
|(
id,
topic,
payload,
created_at,
available_at,
attempt_count,
claim_generation,
claimed_by,
)| {
OutboxMessage {
id,
topic,
payload,
created_at,
available_at,
attempt_count,
claim: OutboxClaim {
message_id: id,
worker_id: claimed_by,
generation: claim_generation,
},
}
},
)
.collect())
}
/// Mark a claimed message projected. Idempotent when the row is already
/// projected. Requires the live claim token when still pending.
pub async fn mark_projected(tx: &mut PgTx<'_>, claim: &OutboxClaim) -> Result<(), DurableError> {
let result = sqlx::query(
"UPDATE outbox_messages
SET projected_at = now(),
claimed_by = NULL,
claimed_until = NULL,
last_error = NULL
WHERE id = $1
AND projected_at IS NULL
AND claimed_by = $2
AND claim_generation = $3",
)
.bind(claim.message_id)
.bind(&claim.worker_id)
.bind(claim.generation)
.execute(tx.as_mut())
.await?;
if result.rows_affected() == 1 {
return Ok(());
}
let projected: Option<(bool,)> = sqlx::query_as(
"SELECT projected_at IS NOT NULL
FROM outbox_messages
WHERE id = $1",
)
.bind(claim.message_id)
.fetch_optional(tx.as_mut())
.await?;
match projected {
Some((true,)) => Ok(()),
Some((false,)) => Err(DurableError::OutboxClaimMismatch {
message_id: claim.message_id,
}),
None => Err(DurableError::OutboxMissing {
message_id: claim.message_id,
}),
}
}
pub async fn mark_projected_pool(pool: &PgPool, claim: &OutboxClaim) -> Result<(), DurableError> {
let claim = claim.clone();
tx::with_transaction(pool, move |tx| {
Box::pin(async move { mark_projected(tx, &claim).await })
})
.await
}
/// Release a claimed message for later retry. Fenced to the claim token so a
/// stale claimant cannot clear a reclaimed lease. Retry availability uses the
/// database clock.
pub async fn release_with_error(
tx: &mut PgTx<'_>,
claim: &OutboxClaim,
error: &str,
retry_delay: time::Duration,
) -> Result<(), DurableError> {
let delay_secs = retry_delay.as_seconds_f64().max(0.0);
let result = sqlx::query(
"UPDATE outbox_messages
SET claimed_by = NULL,
claimed_until = NULL,
available_at = now() + ($4 * interval '1 second'),
last_error = $5
WHERE id = $1
AND projected_at IS NULL
AND claimed_by = $2
AND claim_generation = $3",
)
.bind(claim.message_id)
.bind(&claim.worker_id)
.bind(claim.generation)
.bind(delay_secs)
.bind(error)
.execute(tx.as_mut())
.await?;
if result.rows_affected() != 1 {
return Err(DurableError::OutboxClaimMismatch {
message_id: claim.message_id,
});
}
Ok(())
}
pub async fn unprojected_count(
pool: &PgPool,
filter: OutboxClaimFilter<'_>,
) -> Result<i64, DurableError> {
let topic = match filter {
OutboxClaimFilter::Any => None,
OutboxClaimFilter::Topic(topic) => Some(topic),
};
let (count,): (i64,) = sqlx::query_as(
"SELECT COUNT(*)::bigint
FROM outbox_messages
WHERE projected_at IS NULL
AND ($1::text IS NULL OR topic = $1)",
)
.bind(topic)
.fetch_one(pool)
.await?;
Ok(count)
}

View file

@ -0,0 +1,41 @@
use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
use sqlx::{ConnectOptions, PgPool};
use std::str::FromStr;
use std::time::Duration;
use crate::error::DurableError;
#[derive(Clone, Debug)]
pub struct PoolConfig {
pub max_connections: u32,
pub acquire_timeout: Duration,
pub idle_timeout: Option<Duration>,
}
impl Default for PoolConfig {
fn default() -> Self {
Self {
max_connections: 10,
acquire_timeout: Duration::from_secs(5),
idle_timeout: Some(Duration::from_secs(60)),
}
}
}
pub async fn connect(database_url: &str, config: PoolConfig) -> Result<PgPool, DurableError> {
let mut options = PgConnectOptions::from_str(database_url)
.map_err(|error| DurableError::Config(format!("invalid database URL: {error}")))?;
options = options
.application_name("remora-durable")
.statement_cache_capacity(100);
options = options.disable_statement_logging();
let mut builder = PgPoolOptions::new()
.max_connections(config.max_connections)
.acquire_timeout(config.acquire_timeout);
if let Some(idle_timeout) = config.idle_timeout {
builder = builder.idle_timeout(idle_timeout);
}
Ok(builder.connect_with(options).await?)
}

View file

@ -0,0 +1,42 @@
use std::future::Future;
use std::pin::Pin;
use sqlx::{PgPool, Postgres, Transaction};
use crate::error::DurableError;
pub type PgTx<'a> = Transaction<'a, Postgres>;
pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
pub async fn with_transaction<T, F>(pool: &PgPool, work: F) -> Result<T, DurableError>
where
F: for<'c> FnOnce(&'c mut PgTx<'_>) -> BoxFuture<'c, Result<T, DurableError>>,
{
let mut tx = pool.begin().await?;
match work(&mut tx).await {
Ok(value) => {
tx.commit().await?;
Ok(value)
}
Err(error) => {
let _ = tx.rollback().await;
Err(error)
}
}
}
/// Bound how long this transaction waits for row or advisory locks.
pub async fn set_lock_timeout(tx: &mut PgTx<'_>) -> Result<(), DurableError> {
sqlx::query("SET LOCAL lock_timeout = '5s'")
.execute(tx.as_mut())
.await?;
Ok(())
}
pub async fn advisory_xact_lock(tx: &mut PgTx<'_>, key: i64) -> Result<(), DurableError> {
sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(key)
.execute(tx.as_mut())
.await?;
Ok(())
}