Add PostgreSQL DurableStore substrate for WP-03
Introduce remora-durable with pool/migrations, a RemoraApiMutation-shaped ledger, claim-fenced outbox projection, CAS helpers, and keyset cursors, plus a kind Job acceptance suite against the hermetic fixture.
This commit is contained in:
parent
af8bb1a03a
commit
957d170150
9 changed files with 3203 additions and 18 deletions
90
crates/remora-durable/src/cas.rs
Normal file
90
crates/remora-durable/src/cas.rs
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
use sqlx::{AssertSqlSafe, PgPool};
|
||||
|
||||
use crate::error::DurableError;
|
||||
use crate::tx::{self, PgTx};
|
||||
|
||||
/// Optimistic compare-and-swap on a caller-owned table.
|
||||
///
|
||||
/// The table must expose `revision BIGINT NOT NULL` and a primary key column
|
||||
/// bound as `$1`. Feature packages own the surrounding schema; this helper only
|
||||
/// serializes the revision bump. Identifiers are restricted to lowercase SQL
|
||||
/// identifiers before interpolation.
|
||||
pub async fn cas_set_text(
|
||||
tx: &mut PgTx<'_>,
|
||||
table: &str,
|
||||
id_column: &str,
|
||||
id: &str,
|
||||
value_column: &str,
|
||||
value: &str,
|
||||
expected_revision: i64,
|
||||
) -> Result<i64, DurableError> {
|
||||
validate_ident(table)?;
|
||||
validate_ident(id_column)?;
|
||||
validate_ident(value_column)?;
|
||||
|
||||
let sql = format!(
|
||||
"UPDATE {table}
|
||||
SET {value_column} = $2,
|
||||
revision = revision + 1
|
||||
WHERE {id_column} = $1
|
||||
AND revision = $3
|
||||
RETURNING revision"
|
||||
);
|
||||
let next: Option<(i64,)> = sqlx::query_as(AssertSqlSafe(sql))
|
||||
.bind(id)
|
||||
.bind(value)
|
||||
.bind(expected_revision)
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
next.map(|(revision,)| revision)
|
||||
.ok_or(DurableError::CasConflict)
|
||||
}
|
||||
|
||||
pub async fn cas_set_text_pool(
|
||||
pool: &PgPool,
|
||||
table: &str,
|
||||
id_column: &str,
|
||||
id: &str,
|
||||
value_column: &str,
|
||||
value: &str,
|
||||
expected_revision: i64,
|
||||
) -> Result<i64, DurableError> {
|
||||
let table = table.to_owned();
|
||||
let id_column = id_column.to_owned();
|
||||
let id = id.to_owned();
|
||||
let value_column = value_column.to_owned();
|
||||
let value = value.to_owned();
|
||||
tx::with_transaction(pool, move |tx| {
|
||||
Box::pin(async move {
|
||||
cas_set_text(
|
||||
tx,
|
||||
&table,
|
||||
&id_column,
|
||||
&id,
|
||||
&value_column,
|
||||
&value,
|
||||
expected_revision,
|
||||
)
|
||||
.await
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
fn validate_ident(value: &str) -> Result<(), DurableError> {
|
||||
if value.is_empty()
|
||||
|| value.len() > 63
|
||||
|| !value.chars().all(|character| {
|
||||
character.is_ascii_lowercase() || character.is_ascii_digit() || character == '_'
|
||||
})
|
||||
|| value
|
||||
.chars()
|
||||
.next()
|
||||
.is_none_or(|character| !(character.is_ascii_lowercase() || character == '_'))
|
||||
{
|
||||
return Err(DurableError::Config(format!(
|
||||
"invalid SQL identifier '{value}'"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
231
crates/remora-durable/src/cursor.rs
Normal file
231
crates/remora-durable/src/cursor.rs
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
use base64::Engine;
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use remora_domain::fingerprint;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Sha256;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::error::DurableError;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
/// Stable keyset order for job-style listings: `(created_at DESC, id DESC)`.
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
pub struct KeysetPosition {
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub created_at: OffsetDateTime,
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
struct CursorPayload {
|
||||
subject: String,
|
||||
filter_fingerprint: String,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
high_water_created_at: OffsetDateTime,
|
||||
high_water_id: String,
|
||||
after: Option<KeysetPosition>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct KeysetCursor {
|
||||
pub subject: String,
|
||||
pub filter_fingerprint: [u8; 32],
|
||||
pub high_water: KeysetPosition,
|
||||
pub after: Option<KeysetPosition>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CursorCodec {
|
||||
secret: Vec<u8>,
|
||||
}
|
||||
|
||||
impl CursorCodec {
|
||||
pub fn new(secret: impl Into<Vec<u8>>) -> Result<Self, DurableError> {
|
||||
let secret = secret.into();
|
||||
if secret.len() < 16 {
|
||||
return Err(DurableError::Config(
|
||||
"cursor integrity secret must be at least 16 bytes".into(),
|
||||
));
|
||||
}
|
||||
Ok(Self { secret })
|
||||
}
|
||||
|
||||
pub fn filter_fingerprint(filters: &impl Serialize) -> Result<[u8; 32], DurableError> {
|
||||
let digest = fingerprint(filters).map_err(|_| {
|
||||
DurableError::Config("filters cannot be represented as canonical JSON".into())
|
||||
})?;
|
||||
Ok(*digest.as_bytes())
|
||||
}
|
||||
|
||||
pub fn encode(&self, cursor: &KeysetCursor) -> Result<String, DurableError> {
|
||||
let payload = CursorPayload {
|
||||
subject: cursor.subject.clone(),
|
||||
filter_fingerprint: hex::encode(cursor.filter_fingerprint),
|
||||
high_water_created_at: cursor.high_water.created_at,
|
||||
high_water_id: cursor.high_water.id.clone(),
|
||||
after: cursor.after.clone(),
|
||||
};
|
||||
let body = serde_json::to_vec(&payload).map_err(|error| {
|
||||
DurableError::Config(format!("cursor payload is not JSON-serializable: {error}"))
|
||||
})?;
|
||||
let mut mac = HmacSha256::new_from_slice(&self.secret)
|
||||
.map_err(|error| DurableError::Config(format!("invalid cursor secret: {error}")))?;
|
||||
mac.update(&body);
|
||||
let tag = mac.finalize().into_bytes();
|
||||
|
||||
let mut framed = Vec::with_capacity(body.len() + tag.len());
|
||||
framed.extend_from_slice(&body);
|
||||
framed.extend_from_slice(&tag);
|
||||
Ok(URL_SAFE_NO_PAD.encode(framed))
|
||||
}
|
||||
|
||||
pub fn decode(&self, token: &str) -> Result<KeysetCursor, DurableError> {
|
||||
let framed = URL_SAFE_NO_PAD
|
||||
.decode(token.as_bytes())
|
||||
.map_err(|_| DurableError::InvalidCursor)?;
|
||||
if framed.len() <= 32 {
|
||||
return Err(DurableError::InvalidCursor);
|
||||
}
|
||||
let (body, tag) = framed.split_at(framed.len() - 32);
|
||||
let mut mac = HmacSha256::new_from_slice(&self.secret)
|
||||
.map_err(|error| DurableError::Config(format!("invalid cursor secret: {error}")))?;
|
||||
mac.update(body);
|
||||
mac.verify_slice(tag)
|
||||
.map_err(|_| DurableError::InvalidCursor)?;
|
||||
|
||||
let payload: CursorPayload =
|
||||
serde_json::from_slice(body).map_err(|_| DurableError::InvalidCursor)?;
|
||||
let filter_fingerprint = parse_hex32(&payload.filter_fingerprint)?;
|
||||
Ok(KeysetCursor {
|
||||
subject: payload.subject,
|
||||
filter_fingerprint,
|
||||
high_water: KeysetPosition {
|
||||
created_at: payload.high_water_created_at,
|
||||
id: payload.high_water_id,
|
||||
},
|
||||
after: payload.after,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn bind(
|
||||
&self,
|
||||
token: &str,
|
||||
subject: &str,
|
||||
filter_fingerprint: [u8; 32],
|
||||
) -> Result<KeysetCursor, DurableError> {
|
||||
let cursor = self.decode(token)?;
|
||||
if cursor.subject != subject || cursor.filter_fingerprint != filter_fingerprint {
|
||||
return Err(DurableError::CursorBindingMismatch);
|
||||
}
|
||||
Ok(cursor)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a first-page cursor after returning one page.
|
||||
///
|
||||
/// `high_water` is the newest visible row at snapshot time (membership bound).
|
||||
/// `last_on_page` is the last row of the returned page and becomes the exclusive
|
||||
/// keyset lower bound for the next page. For a single-item page the two
|
||||
/// positions are equal.
|
||||
pub fn first_page_cursor(
|
||||
subject: impl Into<String>,
|
||||
filter_fingerprint: [u8; 32],
|
||||
high_water: KeysetPosition,
|
||||
last_on_page: KeysetPosition,
|
||||
) -> KeysetCursor {
|
||||
KeysetCursor {
|
||||
subject: subject.into(),
|
||||
filter_fingerprint,
|
||||
high_water,
|
||||
after: Some(last_on_page),
|
||||
}
|
||||
}
|
||||
|
||||
/// Advance a keyset cursor after returning `last` on the current page.
|
||||
pub fn next_page_cursor(current: &KeysetCursor, last: KeysetPosition) -> KeysetCursor {
|
||||
KeysetCursor {
|
||||
subject: current.subject.clone(),
|
||||
filter_fingerprint: current.filter_fingerprint,
|
||||
high_water: current.high_water.clone(),
|
||||
after: Some(last),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hex32(value: &str) -> Result<[u8; 32], DurableError> {
|
||||
if value.len() != 64 {
|
||||
return Err(DurableError::InvalidCursor);
|
||||
}
|
||||
let mut out = [0_u8; 32];
|
||||
for (index, slot) in out.iter_mut().enumerate() {
|
||||
*slot = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16)
|
||||
.map_err(|_| DurableError::InvalidCursor)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// Minimal hex encode without an extra dependency.
|
||||
mod hex {
|
||||
pub fn encode(bytes: [u8; 32]) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut output = String::with_capacity(64);
|
||||
for byte in bytes {
|
||||
output.push(char::from(HEX[usize::from(byte >> 4)]));
|
||||
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
|
||||
}
|
||||
output
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
use time::macros::datetime;
|
||||
|
||||
#[test]
|
||||
fn round_trips_and_binds_subject_and_filters() {
|
||||
let codec = CursorCodec::new(b"test-cursor-secret").unwrap();
|
||||
let filters = json!({"bucket": "alpha", "lifecycle": "visible"});
|
||||
let fingerprint = CursorCodec::filter_fingerprint(&filters).unwrap();
|
||||
let newest = KeysetPosition {
|
||||
created_at: datetime!(2026-07-16 12:00:00 UTC),
|
||||
id: "job-zzzzzzzz".into(),
|
||||
};
|
||||
let last = KeysetPosition {
|
||||
created_at: datetime!(2026-07-16 11:00:00 UTC),
|
||||
id: "job-aaaaaaaa".into(),
|
||||
};
|
||||
let cursor = first_page_cursor("subject-001", fingerprint, newest, last);
|
||||
let token = codec.encode(&cursor).unwrap();
|
||||
let decoded = codec.bind(&token, "subject-001", fingerprint).unwrap();
|
||||
assert_eq!(decoded, cursor);
|
||||
assert!(matches!(
|
||||
codec.bind(&token, "other-subject", fingerprint),
|
||||
Err(DurableError::CursorBindingMismatch)
|
||||
));
|
||||
let other = CursorCodec::filter_fingerprint(&json!({"bucket": "beta"})).unwrap();
|
||||
assert!(matches!(
|
||||
codec.bind(&token, "subject-001", other),
|
||||
Err(DurableError::CursorBindingMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_tampered_tokens() {
|
||||
let codec = CursorCodec::new(b"test-cursor-secret").unwrap();
|
||||
let fingerprint = [7_u8; 32];
|
||||
let position = KeysetPosition {
|
||||
created_at: datetime!(2026-07-16 12:00:00 UTC),
|
||||
id: "job-aaaaaaaa".into(),
|
||||
};
|
||||
let cursor = first_page_cursor("subject-001", fingerprint, position.clone(), position);
|
||||
let mut token = codec.encode(&cursor).unwrap();
|
||||
token.replace_range(0..1, if token.starts_with('A') { "B" } else { "A" });
|
||||
assert!(matches!(
|
||||
codec.decode(&token),
|
||||
Err(DurableError::InvalidCursor)
|
||||
));
|
||||
}
|
||||
}
|
||||
41
crates/remora-durable/src/error.rs
Normal file
41
crates/remora-durable/src/error.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
use std::fmt;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum DurableError {
|
||||
#[error("database error: {0}")]
|
||||
Database(#[from] sqlx::Error),
|
||||
#[error("migration error: {0}")]
|
||||
Migrate(#[from] sqlx::migrate::MigrateError),
|
||||
/// Transport-layer mapping of a ledger conflict disposition.
|
||||
///
|
||||
/// The substrate ledger returns
|
||||
/// `MutationOutcome { disposition: Conflict, … }` for RemoraApiMutation
|
||||
/// conflict observations. Higher layers may convert that outcome into this
|
||||
/// error when an HTTP or worker adapter wants an error path.
|
||||
#[error("mutation conflict for subject={subject} operation={operation} key={key}")]
|
||||
MutationConflict {
|
||||
subject: String,
|
||||
operation: String,
|
||||
key: String,
|
||||
},
|
||||
#[error("mutation effect failed: {0}")]
|
||||
Effect(String),
|
||||
#[error("outbox claim does not own message {message_id}")]
|
||||
OutboxClaimMismatch { message_id: i64 },
|
||||
#[error("outbox message {message_id} is missing")]
|
||||
OutboxMissing { message_id: i64 },
|
||||
#[error("invalid keyset cursor")]
|
||||
InvalidCursor,
|
||||
#[error("keyset cursor binding mismatch")]
|
||||
CursorBindingMismatch,
|
||||
#[error("compare-and-swap precondition failed")]
|
||||
CasConflict,
|
||||
#[error("configuration error: {0}")]
|
||||
Config(String),
|
||||
}
|
||||
|
||||
impl DurableError {
|
||||
pub fn effect(error: impl fmt::Display) -> Self {
|
||||
Self::Effect(error.to_string())
|
||||
}
|
||||
}
|
||||
260
crates/remora-durable/src/ledger.rs
Normal file
260
crates/remora-durable/src/ledger.rs
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
use remora_domain::RequestFingerprint;
|
||||
use remora_domain::ports::MutationScope;
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::PgPool;
|
||||
|
||||
use crate::error::DurableError;
|
||||
use crate::tx::{self, BoxFuture, PgTx};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MutationDisposition {
|
||||
First,
|
||||
Replay,
|
||||
Conflict,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum MutationStatus {
|
||||
Accepted,
|
||||
Rejected,
|
||||
}
|
||||
|
||||
impl MutationStatus {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Accepted => "accepted",
|
||||
Self::Rejected => "rejected",
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(value: &str) -> Result<Self, DurableError> {
|
||||
match value {
|
||||
"accepted" => Ok(Self::Accepted),
|
||||
"rejected" => Ok(Self::Rejected),
|
||||
other => Err(DurableError::Config(format!(
|
||||
"unknown mutation status '{other}'"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct MutationOutcome {
|
||||
pub disposition: MutationDisposition,
|
||||
pub status: MutationStatus,
|
||||
pub response: Value,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct StoredRecord {
|
||||
kind: String,
|
||||
eligibility_claim: Vec<u8>,
|
||||
request_fingerprint: Vec<u8>,
|
||||
response_status: String,
|
||||
response_body: Value,
|
||||
}
|
||||
|
||||
/// Execute a mutation refining `RemoraApiMutation`.
|
||||
///
|
||||
/// Identity is `(subject, operation, key)`. Exact kind, eligibility claim, and
|
||||
/// request fingerprint replay the stored response. A change to any of those
|
||||
/// fields is a conflict and leaves the record unchanged. The eligibility claim
|
||||
/// binds attempt, fence, worker incarnation, or other authority material
|
||||
/// outside the semantic payload fingerprint.
|
||||
///
|
||||
/// Concurrent first observations for the same identity serialize on a
|
||||
/// transaction-scoped advisory lock before eligibility or effects run.
|
||||
pub async fn execute_mutation<D>(
|
||||
pool: &PgPool,
|
||||
scope: MutationScope<'_>,
|
||||
kind: &str,
|
||||
eligibility_claim: &[u8],
|
||||
request_fingerprint: RequestFingerprint,
|
||||
decide: D,
|
||||
) -> Result<MutationOutcome, DurableError>
|
||||
where
|
||||
D: for<'c> FnOnce(
|
||||
&'c mut PgTx<'_>,
|
||||
) -> BoxFuture<'c, Result<(MutationStatus, Value), DurableError>>
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
let subject = scope.subject.clone();
|
||||
let operation = scope.operation.to_owned();
|
||||
let key = scope.key.to_owned();
|
||||
let kind = kind.to_owned();
|
||||
let eligibility_claim = eligibility_claim.to_owned();
|
||||
|
||||
tx::with_transaction(pool, move |tx| {
|
||||
Box::pin(async move {
|
||||
let scope = MutationScope {
|
||||
subject: &subject,
|
||||
operation: &operation,
|
||||
key: &key,
|
||||
};
|
||||
execute_mutation_in_tx(
|
||||
tx,
|
||||
scope,
|
||||
&kind,
|
||||
&eligibility_claim,
|
||||
request_fingerprint,
|
||||
decide,
|
||||
)
|
||||
.await
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn execute_mutation_in_tx<D>(
|
||||
tx: &mut PgTx<'_>,
|
||||
scope: MutationScope<'_>,
|
||||
kind: &str,
|
||||
eligibility_claim: &[u8],
|
||||
request_fingerprint: RequestFingerprint,
|
||||
decide: D,
|
||||
) -> Result<MutationOutcome, DurableError>
|
||||
where
|
||||
D: for<'c> FnOnce(
|
||||
&'c mut PgTx<'_>,
|
||||
) -> BoxFuture<'c, Result<(MutationStatus, Value), DurableError>>
|
||||
+ Send,
|
||||
{
|
||||
tx::set_lock_timeout(tx).await?;
|
||||
// Serialize empty-row first observations: FOR UPDATE only locks existing
|
||||
// rows, so concurrent first writers need an exclusive identity claim first.
|
||||
tx::advisory_xact_lock(tx, identity_lock_key(scope)).await?;
|
||||
let fingerprint = request_fingerprint.as_bytes().as_slice();
|
||||
|
||||
if let Some(existing) = load_record(tx, scope).await? {
|
||||
return observe_existing(kind, eligibility_claim, fingerprint, existing);
|
||||
}
|
||||
|
||||
let (status, response) = decide(tx).await?;
|
||||
let effect_applied = status == MutationStatus::Accepted;
|
||||
insert_record(
|
||||
tx,
|
||||
&MutationInsert {
|
||||
scope,
|
||||
kind,
|
||||
eligibility_claim,
|
||||
fingerprint,
|
||||
status,
|
||||
response_body: &response,
|
||||
effect_applied,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
Ok(MutationOutcome {
|
||||
disposition: MutationDisposition::First,
|
||||
status,
|
||||
response,
|
||||
})
|
||||
}
|
||||
|
||||
struct MutationInsert<'a> {
|
||||
scope: MutationScope<'a>,
|
||||
kind: &'a str,
|
||||
eligibility_claim: &'a [u8],
|
||||
fingerprint: &'a [u8],
|
||||
status: MutationStatus,
|
||||
response_body: &'a Value,
|
||||
effect_applied: bool,
|
||||
}
|
||||
|
||||
fn identity_lock_key(scope: MutationScope<'_>) -> i64 {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(scope.subject.as_str().as_bytes());
|
||||
hasher.update([0xff]);
|
||||
hasher.update(scope.operation.as_bytes());
|
||||
hasher.update([0xff]);
|
||||
hasher.update(scope.key.as_bytes());
|
||||
let digest = hasher.finalize();
|
||||
i64::from_be_bytes([
|
||||
digest[0], digest[1], digest[2], digest[3], digest[4], digest[5], digest[6], digest[7],
|
||||
])
|
||||
}
|
||||
|
||||
fn observe_existing(
|
||||
kind: &str,
|
||||
eligibility_claim: &[u8],
|
||||
fingerprint: &[u8],
|
||||
existing: StoredRecord,
|
||||
) -> Result<MutationOutcome, DurableError> {
|
||||
if existing.kind != kind
|
||||
|| existing.eligibility_claim != eligibility_claim
|
||||
|| existing.request_fingerprint != fingerprint
|
||||
{
|
||||
return Ok(MutationOutcome {
|
||||
disposition: MutationDisposition::Conflict,
|
||||
status: MutationStatus::Rejected,
|
||||
response: Value::Null,
|
||||
});
|
||||
}
|
||||
let status = MutationStatus::parse(&existing.response_status)?;
|
||||
Ok(MutationOutcome {
|
||||
disposition: MutationDisposition::Replay,
|
||||
status,
|
||||
response: existing.response_body,
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_record(
|
||||
tx: &mut PgTx<'_>,
|
||||
scope: MutationScope<'_>,
|
||||
) -> Result<Option<StoredRecord>, DurableError> {
|
||||
let row = sqlx::query_as::<_, (String, Vec<u8>, Vec<u8>, String, Value)>(
|
||||
"SELECT kind, eligibility_claim, request_fingerprint, response_status, response_body
|
||||
FROM mutation_records
|
||||
WHERE scope_subject = $1
|
||||
AND scope_operation = $2
|
||||
AND operation_key = $3
|
||||
FOR UPDATE",
|
||||
)
|
||||
.bind(scope.subject.as_str())
|
||||
.bind(scope.operation)
|
||||
.bind(scope.key)
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
|
||||
Ok(row.map(
|
||||
|(kind, eligibility_claim, request_fingerprint, response_status, response_body)| {
|
||||
StoredRecord {
|
||||
kind,
|
||||
eligibility_claim,
|
||||
request_fingerprint,
|
||||
response_status,
|
||||
response_body,
|
||||
}
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
async fn insert_record(tx: &mut PgTx<'_>, record: &MutationInsert<'_>) -> Result<(), DurableError> {
|
||||
sqlx::query(
|
||||
"INSERT INTO mutation_records (
|
||||
scope_subject,
|
||||
scope_operation,
|
||||
operation_key,
|
||||
kind,
|
||||
eligibility_claim,
|
||||
request_fingerprint,
|
||||
response_status,
|
||||
response_body,
|
||||
effect_applied
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
|
||||
)
|
||||
.bind(record.scope.subject.as_str())
|
||||
.bind(record.scope.operation)
|
||||
.bind(record.scope.key)
|
||||
.bind(record.kind)
|
||||
.bind(record.eligibility_claim)
|
||||
.bind(record.fingerprint)
|
||||
.bind(record.status.as_str())
|
||||
.bind(record.response_body)
|
||||
.bind(record.effect_applied)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
29
crates/remora-durable/src/lib.rs
Normal file
29
crates/remora-durable/src/lib.rs
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
//! PostgreSQL DurableStore substrate.
|
||||
//!
|
||||
//! Provides the pool, migration runner, transaction helpers, generic mutation
|
||||
//! ledger refining `RemoraApiMutation`, durable outbox claim/projection, and
|
||||
//! integrity-protected keyset cursor primitives. Feature packages own their
|
||||
//! domain schema additions.
|
||||
|
||||
mod cas;
|
||||
mod cursor;
|
||||
mod error;
|
||||
mod ledger;
|
||||
mod migrate;
|
||||
mod outbox;
|
||||
mod pool;
|
||||
mod tx;
|
||||
|
||||
pub use cas::{cas_set_text, cas_set_text_pool};
|
||||
pub use cursor::{CursorCodec, KeysetCursor, KeysetPosition, first_page_cursor, next_page_cursor};
|
||||
pub use error::DurableError;
|
||||
pub use ledger::{
|
||||
MutationDisposition, MutationOutcome, MutationStatus, execute_mutation, execute_mutation_in_tx,
|
||||
};
|
||||
pub use migrate::{migrate, migrator};
|
||||
pub use outbox::{
|
||||
OutboxClaim, OutboxClaimFilter, OutboxMessage, claim_batch, claim_batch_in_tx, enqueue,
|
||||
enqueue_after, mark_projected, mark_projected_pool, release_with_error, unprojected_count,
|
||||
};
|
||||
pub use pool::{PoolConfig, connect};
|
||||
pub use tx::{BoxFuture, PgTx, advisory_xact_lock, set_lock_timeout, with_transaction};
|
||||
14
crates/remora-durable/src/migrate.rs
Normal file
14
crates/remora-durable/src/migrate.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
use sqlx::PgPool;
|
||||
|
||||
use crate::error::DurableError;
|
||||
|
||||
static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations");
|
||||
|
||||
pub async fn migrate(pool: &PgPool) -> Result<(), DurableError> {
|
||||
MIGRATOR.run(pool).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn migrator() -> &'static sqlx::migrate::Migrator {
|
||||
&MIGRATOR
|
||||
}
|
||||
294
crates/remora-durable/src/outbox.rs
Normal file
294
crates/remora-durable/src/outbox.rs
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
use serde_json::Value;
|
||||
use sqlx::PgPool;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::error::DurableError;
|
||||
use crate::tx::{self, PgTx};
|
||||
|
||||
/// Ownership token returned by claim; required for projection and release.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct OutboxClaim {
|
||||
pub message_id: i64,
|
||||
pub worker_id: String,
|
||||
pub generation: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct OutboxMessage {
|
||||
pub id: i64,
|
||||
pub topic: String,
|
||||
pub payload: Value,
|
||||
pub created_at: OffsetDateTime,
|
||||
pub available_at: OffsetDateTime,
|
||||
pub attempt_count: i32,
|
||||
pub claim: OutboxClaim,
|
||||
}
|
||||
|
||||
/// Selects which outbox rows a claim may take.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum OutboxClaimFilter<'a> {
|
||||
/// Any claimable topic.
|
||||
Any,
|
||||
/// Exact topic match.
|
||||
Topic(&'a str),
|
||||
}
|
||||
|
||||
pub async fn enqueue(tx: &mut PgTx<'_>, topic: &str, payload: Value) -> Result<i64, DurableError> {
|
||||
let (id,): (i64,) = sqlx::query_as(
|
||||
"INSERT INTO outbox_messages (topic, payload, available_at)
|
||||
VALUES ($1, $2, now())
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(topic)
|
||||
.bind(payload)
|
||||
.fetch_one(tx.as_mut())
|
||||
.await?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Enqueue work available after a delay measured by the database clock.
|
||||
pub async fn enqueue_after(
|
||||
tx: &mut PgTx<'_>,
|
||||
topic: &str,
|
||||
payload: Value,
|
||||
delay: time::Duration,
|
||||
) -> Result<i64, DurableError> {
|
||||
let delay_secs = delay.as_seconds_f64().max(0.0);
|
||||
let (id,): (i64,) = sqlx::query_as(
|
||||
"INSERT INTO outbox_messages (topic, payload, available_at)
|
||||
VALUES ($1, $2, now() + ($3 * interval '1 second'))
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(topic)
|
||||
.bind(payload)
|
||||
.bind(delay_secs)
|
||||
.fetch_one(tx.as_mut())
|
||||
.await?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Claim claimable outbox rows for exclusive projection using
|
||||
/// `FOR UPDATE SKIP LOCKED`. Lease eligibility and expiry use the database
|
||||
/// clock. Each successful claim advances `claim_generation` and returns an
|
||||
/// ownership token required for projection or release.
|
||||
pub async fn claim_batch(
|
||||
pool: &PgPool,
|
||||
worker_id: &str,
|
||||
filter: OutboxClaimFilter<'_>,
|
||||
limit: i64,
|
||||
lease: time::Duration,
|
||||
) -> Result<Vec<OutboxMessage>, DurableError> {
|
||||
let worker_id = worker_id.to_owned();
|
||||
let topic = match filter {
|
||||
OutboxClaimFilter::Any => None,
|
||||
OutboxClaimFilter::Topic(topic) => Some(topic.to_owned()),
|
||||
};
|
||||
tx::with_transaction(pool, move |tx| {
|
||||
Box::pin(async move {
|
||||
let filter = match topic.as_deref() {
|
||||
None => OutboxClaimFilter::Any,
|
||||
Some(topic) => OutboxClaimFilter::Topic(topic),
|
||||
};
|
||||
claim_batch_in_tx(tx, &worker_id, filter, limit, lease).await
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn claim_batch_in_tx(
|
||||
tx: &mut PgTx<'_>,
|
||||
worker_id: &str,
|
||||
filter: OutboxClaimFilter<'_>,
|
||||
limit: i64,
|
||||
lease: time::Duration,
|
||||
) -> Result<Vec<OutboxMessage>, DurableError> {
|
||||
let lease_secs = lease.as_seconds_f64().max(0.0);
|
||||
let topic = match filter {
|
||||
OutboxClaimFilter::Any => None,
|
||||
OutboxClaimFilter::Topic(topic) => Some(topic),
|
||||
};
|
||||
let rows = sqlx::query_as::<
|
||||
_,
|
||||
(
|
||||
i64,
|
||||
String,
|
||||
Value,
|
||||
OffsetDateTime,
|
||||
OffsetDateTime,
|
||||
i32,
|
||||
i64,
|
||||
String,
|
||||
),
|
||||
>(
|
||||
"WITH candidates AS (
|
||||
SELECT id
|
||||
FROM outbox_messages
|
||||
WHERE projected_at IS NULL
|
||||
AND available_at <= now()
|
||||
AND (claimed_until IS NULL OR claimed_until < now())
|
||||
AND ($4::text IS NULL OR topic = $4)
|
||||
ORDER BY id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT $1
|
||||
)
|
||||
UPDATE outbox_messages AS messages
|
||||
SET claimed_by = $2,
|
||||
claimed_until = now() + ($3 * interval '1 second'),
|
||||
claim_generation = messages.claim_generation + 1,
|
||||
attempt_count = messages.attempt_count + 1
|
||||
FROM candidates
|
||||
WHERE messages.id = candidates.id
|
||||
RETURNING
|
||||
messages.id,
|
||||
messages.topic,
|
||||
messages.payload,
|
||||
messages.created_at,
|
||||
messages.available_at,
|
||||
messages.attempt_count,
|
||||
messages.claim_generation,
|
||||
messages.claimed_by",
|
||||
)
|
||||
.bind(limit)
|
||||
.bind(worker_id)
|
||||
.bind(lease_secs)
|
||||
.bind(topic)
|
||||
.fetch_all(tx.as_mut())
|
||||
.await?;
|
||||
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(
|
||||
|(
|
||||
id,
|
||||
topic,
|
||||
payload,
|
||||
created_at,
|
||||
available_at,
|
||||
attempt_count,
|
||||
claim_generation,
|
||||
claimed_by,
|
||||
)| {
|
||||
OutboxMessage {
|
||||
id,
|
||||
topic,
|
||||
payload,
|
||||
created_at,
|
||||
available_at,
|
||||
attempt_count,
|
||||
claim: OutboxClaim {
|
||||
message_id: id,
|
||||
worker_id: claimed_by,
|
||||
generation: claim_generation,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Mark a claimed message projected. Idempotent when the row is already
|
||||
/// projected. Requires the live claim token when still pending.
|
||||
pub async fn mark_projected(tx: &mut PgTx<'_>, claim: &OutboxClaim) -> Result<(), DurableError> {
|
||||
let result = sqlx::query(
|
||||
"UPDATE outbox_messages
|
||||
SET projected_at = now(),
|
||||
claimed_by = NULL,
|
||||
claimed_until = NULL,
|
||||
last_error = NULL
|
||||
WHERE id = $1
|
||||
AND projected_at IS NULL
|
||||
AND claimed_by = $2
|
||||
AND claim_generation = $3",
|
||||
)
|
||||
.bind(claim.message_id)
|
||||
.bind(&claim.worker_id)
|
||||
.bind(claim.generation)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if result.rows_affected() == 1 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let projected: Option<(bool,)> = sqlx::query_as(
|
||||
"SELECT projected_at IS NOT NULL
|
||||
FROM outbox_messages
|
||||
WHERE id = $1",
|
||||
)
|
||||
.bind(claim.message_id)
|
||||
.fetch_optional(tx.as_mut())
|
||||
.await?;
|
||||
|
||||
match projected {
|
||||
Some((true,)) => Ok(()),
|
||||
Some((false,)) => Err(DurableError::OutboxClaimMismatch {
|
||||
message_id: claim.message_id,
|
||||
}),
|
||||
None => Err(DurableError::OutboxMissing {
|
||||
message_id: claim.message_id,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mark_projected_pool(pool: &PgPool, claim: &OutboxClaim) -> Result<(), DurableError> {
|
||||
let claim = claim.clone();
|
||||
tx::with_transaction(pool, move |tx| {
|
||||
Box::pin(async move { mark_projected(tx, &claim).await })
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Release a claimed message for later retry. Fenced to the claim token so a
|
||||
/// stale claimant cannot clear a reclaimed lease. Retry availability uses the
|
||||
/// database clock.
|
||||
pub async fn release_with_error(
|
||||
tx: &mut PgTx<'_>,
|
||||
claim: &OutboxClaim,
|
||||
error: &str,
|
||||
retry_delay: time::Duration,
|
||||
) -> Result<(), DurableError> {
|
||||
let delay_secs = retry_delay.as_seconds_f64().max(0.0);
|
||||
let result = sqlx::query(
|
||||
"UPDATE outbox_messages
|
||||
SET claimed_by = NULL,
|
||||
claimed_until = NULL,
|
||||
available_at = now() + ($4 * interval '1 second'),
|
||||
last_error = $5
|
||||
WHERE id = $1
|
||||
AND projected_at IS NULL
|
||||
AND claimed_by = $2
|
||||
AND claim_generation = $3",
|
||||
)
|
||||
.bind(claim.message_id)
|
||||
.bind(&claim.worker_id)
|
||||
.bind(claim.generation)
|
||||
.bind(delay_secs)
|
||||
.bind(error)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
if result.rows_affected() != 1 {
|
||||
return Err(DurableError::OutboxClaimMismatch {
|
||||
message_id: claim.message_id,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn unprojected_count(
|
||||
pool: &PgPool,
|
||||
filter: OutboxClaimFilter<'_>,
|
||||
) -> Result<i64, DurableError> {
|
||||
let topic = match filter {
|
||||
OutboxClaimFilter::Any => None,
|
||||
OutboxClaimFilter::Topic(topic) => Some(topic),
|
||||
};
|
||||
let (count,): (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*)::bigint
|
||||
FROM outbox_messages
|
||||
WHERE projected_at IS NULL
|
||||
AND ($1::text IS NULL OR topic = $1)",
|
||||
)
|
||||
.bind(topic)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok(count)
|
||||
}
|
||||
41
crates/remora-durable/src/pool.rs
Normal file
41
crates/remora-durable/src/pool.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
use sqlx::postgres::{PgConnectOptions, PgPoolOptions};
|
||||
use sqlx::{ConnectOptions, PgPool};
|
||||
use std::str::FromStr;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::DurableError;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PoolConfig {
|
||||
pub max_connections: u32,
|
||||
pub acquire_timeout: Duration,
|
||||
pub idle_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
impl Default for PoolConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_connections: 10,
|
||||
acquire_timeout: Duration::from_secs(5),
|
||||
idle_timeout: Some(Duration::from_secs(60)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn connect(database_url: &str, config: PoolConfig) -> Result<PgPool, DurableError> {
|
||||
let mut options = PgConnectOptions::from_str(database_url)
|
||||
.map_err(|error| DurableError::Config(format!("invalid database URL: {error}")))?;
|
||||
options = options
|
||||
.application_name("remora-durable")
|
||||
.statement_cache_capacity(100);
|
||||
options = options.disable_statement_logging();
|
||||
|
||||
let mut builder = PgPoolOptions::new()
|
||||
.max_connections(config.max_connections)
|
||||
.acquire_timeout(config.acquire_timeout);
|
||||
if let Some(idle_timeout) = config.idle_timeout {
|
||||
builder = builder.idle_timeout(idle_timeout);
|
||||
}
|
||||
|
||||
Ok(builder.connect_with(options).await?)
|
||||
}
|
||||
42
crates/remora-durable/src/tx.rs
Normal file
42
crates/remora-durable/src/tx.rs
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
|
||||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
|
||||
use crate::error::DurableError;
|
||||
|
||||
pub type PgTx<'a> = Transaction<'a, Postgres>;
|
||||
pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
pub async fn with_transaction<T, F>(pool: &PgPool, work: F) -> Result<T, DurableError>
|
||||
where
|
||||
F: for<'c> FnOnce(&'c mut PgTx<'_>) -> BoxFuture<'c, Result<T, DurableError>>,
|
||||
{
|
||||
let mut tx = pool.begin().await?;
|
||||
match work(&mut tx).await {
|
||||
Ok(value) => {
|
||||
tx.commit().await?;
|
||||
Ok(value)
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = tx.rollback().await;
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Bound how long this transaction waits for row or advisory locks.
|
||||
pub async fn set_lock_timeout(tx: &mut PgTx<'_>) -> Result<(), DurableError> {
|
||||
sqlx::query("SET LOCAL lock_timeout = '5s'")
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn advisory_xact_lock(tx: &mut PgTx<'_>, key: i64) -> Result<(), DurableError> {
|
||||
sqlx::query("SELECT pg_advisory_xact_lock($1)")
|
||||
.bind(key)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Reference in a new issue