ms2k/remora
SHA256
Watch
0
Code Releases Activity

Harden durable replay and lock recovery

Browse source
This commit is contained in:
ms2k 2026-07-16 14:30:47 +09:00
commit 182b4d58c7
1 changed files with 156 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

100
crates/remora-durable/tests/substrate.rs
View file

@ -4,11 +4,11 @@ use std::time::Duration;
use remora_domain::ports::MutationScope;
use remora_domain::{OpaqueId, fingerprint};
use remora_durable::{
CursorCodec, DurableError, KeysetPosition, MutationDisposition, MutationStatus, OutboxClaim,
OutboxClaimFilter, PoolConfig, advisory_xact_lock, cas_set_text, claim_batch,
claim_batch_in_tx, connect, enqueue, enqueue_after, execute_mutation, first_page_cursor,
mark_projected, mark_projected_pool, migrate, next_page_cursor, release_with_error,
unprojected_count, with_transaction,
ADVISORY_CLASS_APP, CursorCodec, DurableError, KeysetPosition, MutationDisposition,
MutationStatus, OutboxClaim, OutboxClaimFilter, PoolConfig, advisory_key, advisory_xact_lock,
cas_set_text, claim_batch, claim_batch_in_tx, connect, enqueue, enqueue_after,
execute_mutation, first_page_cursor, mark_projected, mark_projected_pool, migrate,
next_page_cursor, release_with_error, unprojected_count, with_transaction,
};
use serde_json::json;
use sqlx::PgPool;
@ -60,6 +60,26 @@ fn eligibility(attempt: &str, fence: u64, worker: &str) -> Vec<u8> {
format!("attempt={attempt}|fence={fence}|worker={worker}").into_bytes()
}
async fn ensure_effect_probe_table(pool: &PgPool) {
sqlx::query(
"CREATE TABLE IF NOT EXISTS substrate_effect_probe (
id TEXT PRIMARY KEY,
hits BIGINT NOT NULL
)",
)
.execute(pool)
.await
.expect("create effect probe table");
}
async fn clear_mutation_subject(pool: &PgPool, owner: &OpaqueId) {
sqlx::query("DELETE FROM mutation_records WHERE scope_subject = $1")
.bind(owner.as_str())
.execute(pool)
.await
.expect("clear mutation rows for subject");
}
#[tokio::test]
async fn migration_round_trip_is_idempotent() {
let Some(pool) = test_pool().await else {
@ -131,13 +151,20 @@ async fn mutation_ledger_exact_replay_conflict_and_rejection() {
let Some(pool) = test_pool().await else {
return;
};
ensure_effect_probe_table(&pool).await;
let owner = subject("replay");
clear_mutation_subject(&pool, &owner).await;
let operation = "submit";
let key = "op-replay-0001";
let request = json!({"source": "http", "bucket": "demo"});
let digest = fingerprint(&request).expect("fingerprint");
let claim = eligibility("attempt-aaaaaaaa", 1, "worker-aaaaaaaa");
let probe = format!("effect-{key}");
sqlx::query("DELETE FROM substrate_effect_probe WHERE id = $1")
.bind(&probe)
.execute(&pool)
.await
.expect("clear probe");
let first = execute_mutation(
&pool,
@ -207,6 +234,7 @@ async fn mutation_ledger_exact_replay_conflict_and_rejection() {
.expect("conflict observation");
assert_eq!(conflict.disposition, MutationDisposition::Conflict);
assert_eq!(conflict.status, MutationStatus::Rejected);
assert!(conflict.response.is_null());
let kind_conflict = execute_mutation(
&pool,
@ -309,12 +337,19 @@ async fn concurrent_first_observation_serializes_to_one_effect() {
let Some(pool) = test_pool().await else {
return;
};
ensure_effect_probe_table(&pool).await;
let owner = subject("race");
clear_mutation_subject(&pool, &owner).await;
let operation = "submit";
let key = "op-race-0001";
let digest = fingerprint(&json!({"race": true})).expect("fingerprint");
let claim = eligibility("attempt-raceaaaa", 1, "worker-raceaaaa");
let probe = format!("effect-{key}");
sqlx::query("DELETE FROM substrate_effect_probe WHERE id = $1")
.bind(&probe)
.execute(&pool)
.await
.expect("clear probe");
let workers = 8_usize;
let barrier = Arc::new(Barrier::new(workers));
let mut handles = Vec::with_capacity(workers);
@ -382,14 +417,6 @@ async fn apply_effect(
tx: &mut remora_durable::PgTx<'_>,
probe_id: &str,
) -> Result<(), DurableError> {
sqlx::query(
"CREATE TABLE IF NOT EXISTS substrate_effect_probe (
id TEXT PRIMARY KEY,
hits BIGINT NOT NULL
)",
)
.execute(tx.as_mut())
.await?;
sqlx::query(
"INSERT INTO substrate_effect_probe (id, hits) VALUES ($1, 1)
ON CONFLICT (id) DO UPDATE SET hits = substrate_effect_probe.hits + 1",
@ -679,6 +706,38 @@ async fn outbox_claim_projection_recovery_and_claim_fencing() {
.await
.expect("release with error");
// Ambiguous-commit: same claim token may retry release successfully.
with_transaction(&pool, |tx| {
let claim = failed_claim[0].claim.clone();
Box::pin(async move {
release_with_error(tx, &claim, "projection failed", TimeDuration::seconds(60)).await
})
})
.await
.expect("idempotent release_with_error");
// Missing message id is distinct from claim mismatch.
let missing = with_transaction(&pool, |tx| {
Box::pin(async move {
release_with_error(
tx,
&OutboxClaim {
message_id: i64::MAX,
worker_id: "worker-fail".into(),
generation: 1,
},
"missing",
TimeDuration::seconds(1),
)
.await
})
})
.await;
assert!(
matches!(missing, Err(DurableError::OutboxMissing { .. })),
"expected OutboxMissing, got {missing:?}"
);
let early = claim_batch(
&pool,
"worker-early",
@ -739,6 +798,19 @@ async fn outbox_claim_projection_recovery_and_claim_fencing() {
unprojected_count(&pool, retry_filter).await.expect("count"),
0
);
let bad_limit = claim_batch(
&pool,
"worker-bad-limit",
batch_filter,
0,
TimeDuration::seconds(30),
)
.await;
assert!(
matches!(bad_limit, Err(DurableError::Config(_))),
"non-positive claim limit must fail closed: {bad_limit:?}"
);
}
#[tokio::test]
@ -947,7 +1019,7 @@ async fn advisory_lock_serializes_critical_section() {
barrier.wait().await;
with_transaction(&pool, |tx| {
Box::pin(async move {
advisory_xact_lock(tx, 42).await?;
advisory_xact_lock(tx, advisory_key(ADVISORY_CLASS_APP, 42)).await?;
let (value,): (i64,) = sqlx::query_as(
"SELECT value FROM substrate_lock_probe WHERE id = 'lock-row'",
)