ms2k/remora
SHA256
Watch
0
Code Releases Activity

Implement replay-safe authorized public HTTP API

Browse source
This commit is contained in:
ms2k 2026-07-17 21:23:18 +09:00
commit 4c93c25692
20 changed files with 3940 additions and 345 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Cargo.lock generated
View file

@ -3263,13 +3263,16 @@ version = "0.0.0"
dependencies = [
"actix-tls",
"actix-web",
"base64",
"getrandom 0.3.4",
"jsonwebtoken",
"rcgen",
"remora-admission",
"remora-attempt",
"remora-contracts",
"remora-coordination",
"remora-credentials",
"remora-deletion",
"remora-domain",
"remora-durable",
"remora-identity",

7
Dockerfile.test
View file

@ -314,6 +314,13 @@ COPY crates crates
COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && export REMORA_WORKER_API_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast"]
FROM rust-tools AS kind-public-api-runner
WORKDIR /workspace
COPY Cargo.toml ./
COPY crates crates
COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_OIDC_ISSUER\" && test -n \"$REMORA_TARGET_ENDPOINT\" && test -n \"$REMORA_VAULT_ADDR\" && case $REMORA_COORDINATION_PROVIDER in redis|dragonfly) ;; *) exit 1 ;; esac && case $REMORA_OIDC_PROVIDER in keycloak|zitadel) ;; *) exit 1 ;; esac && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_PUBLIC_API_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'binary(public_api)'"]
FROM rust-tools AS kind-worker-build
WORKDIR /workspace
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./

5
Makefile
View file

@ -1,4 +1,4 @@
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
CONTAINER_ENGINE ?= nerdctl
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
@ -83,6 +83,9 @@ check-recovery:
check-worker-api:
./tests/kind/run-worker-api-job.sh
check-public-api:
./tests/kind/run-public-api-job.sh
check-worker:
./tests/kind/run-worker-job.sh

111
crates/remora-admission/src/service.rs
View file

@ -4,7 +4,8 @@ use remora_credentials::{
};
use remora_domain::{OpaqueId, Target, fingerprint, ports::MutationScope};
use remora_durable::{
DurableError, MutationDisposition, MutationStatus, enqueue, execute_mutation, with_transaction,
DurableError, MutationDisposition, MutationStatus, PgTx, enqueue, execute_mutation,
with_transaction,
};
use remora_identity::{
DefaultTarget, Principal, TargetOverride, may_request_execution_grant_revocation,
@ -99,56 +100,12 @@ impl AdmissionService {
if !may_request_execution_grant_revocation(principal) {
return Err(AdmissionError::Unauthorized);
}
let principal = principal.clone();
let grant_id = grant_id.clone();
with_transaction(&self.pool, move |tx| {
Box::pin(async move {
let row = sqlx::query_as::<_, (String, Option<String>)>(
"SELECT status, credential_id
FROM execution_grants
WHERE grant_id = $1
FOR UPDATE",
)
.bind(grant_id.as_str())
.fetch_optional(tx.as_mut())
.await?
.ok_or_else(|| DurableError::Effect("grant_not_found".into()))?;
if row.0 == "revoked" {
return Ok(());
}
let now = OffsetDateTime::now_utc();
sqlx::query(
"UPDATE execution_grants
SET status = 'revoked', revoked_at = $2
WHERE grant_id = $1 AND status = 'active'",
)
.bind(grant_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
if row.1.is_some() {
let changed = sqlx::query(
"UPDATE execution_grant_credential_pins
SET status = 'revoked', revoked_at = COALESCE(revoked_at, $2)
WHERE grant_id = $1 AND status = 'active'",
)
.bind(grant_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
if changed.rows_affected() != 1 {
return Err(DurableError::Effect(
"active credential grant is missing its pin".into(),
));
}
}
enqueue(
tx,
"execution_grant.revoked",
json!({ "grant_id": grant_id }),
)
.await?;
Ok(())
})
Box::pin(
async move { Self::revoke_execution_grant_in_tx(tx, &principal, &grant_id).await },
)
})
.await
.map_err(|error| match &error {
@ -158,6 +115,62 @@ impl AdmissionService {
_ => AdmissionError::Durable(error),
})
}
pub async fn revoke_execution_grant_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
grant_id: &OpaqueId,
) -> Result<(), DurableError> {
if !may_request_execution_grant_revocation(principal) {
return Err(DurableError::Effect("grant_unauthorized".into()));
}
let row = sqlx::query_as::<_, (String, Option<String>)>(
"SELECT status, credential_id
FROM execution_grants
WHERE grant_id = $1
FOR UPDATE",
)
.bind(grant_id.as_str())
.fetch_optional(tx.as_mut())
.await?
.ok_or_else(|| DurableError::Effect("grant_not_found".into()))?;
if row.0 == "revoked" {
return Ok(());
}
let now = OffsetDateTime::now_utc();
sqlx::query(
"UPDATE execution_grants
SET status = 'revoked', revoked_at = $2
WHERE grant_id = $1 AND status = 'active'",
)
.bind(grant_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
if row.1.is_some() {
let changed = sqlx::query(
"UPDATE execution_grant_credential_pins
SET status = 'revoked', revoked_at = COALESCE(revoked_at, $2)
WHERE grant_id = $1 AND status = 'active'",
)
.bind(grant_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
if changed.rows_affected() != 1 {
return Err(DurableError::Effect(
"active credential grant is missing its pin".into(),
));
}
}
enqueue(
tx,
"execution_grant.revoked",
json!({ "grant_id": grant_id }),
)
.await?;
Ok(())
}
}
async fn decide_submission(

13
crates/remora-api/Cargo.toml
View file

@ -9,9 +9,14 @@ version.workspace = true
actix-web = { version = "=4.14.0", features = ["rustls-0_23"] }
actix-tls = "=3.5.0"
getrandom = "0.3.4"
remora-admission = { path = "../remora-admission" }
remora-attempt = { path = "../remora-attempt" }
remora-contracts = { path = "../remora-contracts" }
remora-credentials = { path = "../remora-credentials" }
remora-deletion = { path = "../remora-deletion" }
remora-domain.workspace = true
remora-durable.workspace = true
remora-identity = { path = "../remora-identity" }
remora-result = { path = "../remora-result" }
remora-target = { path = "../remora-target" }
serde.workspace = true
@ -22,12 +27,10 @@ thiserror.workspace = true
time.workspace = true
[dev-dependencies]
base64 = "=0.22.1"
jsonwebtoken = "=9.3.1"
rcgen = "=0.14.8"
remora-admission = { path = "../remora-admission" }
remora-coordination = { path = "../remora-coordination" }
remora-credentials = { path = "../remora-credentials" }
remora-durable.workspace = true
remora-identity = { path = "../remora-identity" }
reqwest = { version = "=0.13.4", default-features = false, features = ["json", "rustls"] }
reqwest = { version = "=0.13.4", default-features = false, features = ["form", "json", "rustls"] }
rustls = "=0.23.42"
tokio.workspace = true

2
crates/remora-api/src/lib.rs
View file

@ -1,5 +1,7 @@
mod public;
mod worker;
pub use public::{PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api};
pub use worker::{
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
};

2030
crates/remora-api/src/public.rs Normal file
View file

File diff suppressed because it is too large Load diff

1288
crates/remora-api/tests/public_api.rs Normal file
View file

File diff suppressed because it is too large Load diff

2
crates/remora-attempt/src/lib.rs
View file

@ -12,7 +12,7 @@ pub use migrate::migrate;
pub use mutation_types::{
AttemptMutationOutcome, CheckpointMultipart, CheckpointProjection, CheckpointRecord,
CommittedManifestRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
JobMutationProjection, MutationRejection, ProgressRecord,
JobMutationProjection, MutationRejection, ProgressRecord, ProgressSnapshot,
};
pub use recovery_types::{
CancelOutcome, CleanupClaim, CleanupCompletion, CleanupIdentity, CooperativeStopOutcome,

8
crates/remora-attempt/src/mutation_types.rs
View file

@ -59,6 +59,14 @@ pub struct ProgressRecord {
pub total_members: Option<u64>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ProgressSnapshot {
pub attempt_id: OpaqueId,
pub fence: u64,
pub progress: ProgressRecord,
pub observed_at: time::OffsetDateTime,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MutationRejection {

98
crates/remora-attempt/src/recovery.rs
View file

@ -5,7 +5,8 @@ use remora_domain::{
FailureClass, FailureDisposition, JobPhase, OpaqueId, fingerprint, ports::MutationScope,
};
use remora_durable::{
DurableError, MutationDisposition, MutationStatus, enqueue, execute_mutation, with_transaction,
DurableError, MutationDisposition, MutationStatus, PgTx, enqueue, execute_mutation,
with_transaction,
};
use remora_target::{attempt_key_attempt_id, attempt_staging_prefix, validate_attempt_key};
use serde::Serialize;
@ -24,56 +25,61 @@ impl AttemptService {
pub async fn request_cancel(&self, job_id: &OpaqueId) -> Result<CancelOutcome, AttemptError> {
with_transaction(&self.pool, |tx| {
let job_id = job_id.clone();
Box::pin(async move {
let Some(row) = sqlx::query_as::<_, CancelRow>(
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
)
.bind(job_id.as_str())
.fetch_optional(tx.as_mut())
.await?
else {
return Ok(CancelOutcome::NotFound);
};
let phase = parse_phase(&row.phase)?;
if phase.is_terminal() {
return Ok(CancelOutcome::AlreadyTerminal);
}
if row.cancel_requested {
return Ok(CancelOutcome::AlreadyRequested);
}
let now = OffsetDateTime::now_utc();
if phase == JobPhase::Running {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
} else {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
authority_attempt_id = NULL, dispatch_needed = FALSE,
terminal_at = $2, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
}
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(CancelOutcome::Requested)
})
Box::pin(async move { Self::request_cancel_in_tx(tx, &job_id).await })
})
.await
.map_err(Into::into)
}
pub async fn request_cancel_in_tx(
tx: &mut PgTx<'_>,
job_id: &OpaqueId,
) -> Result<CancelOutcome, DurableError> {
let Some(row) = sqlx::query_as::<_, CancelRow>(
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
)
.bind(job_id.as_str())
.fetch_optional(tx.as_mut())
.await?
else {
return Ok(CancelOutcome::NotFound);
};
let phase = parse_phase(&row.phase)?;
if phase.is_terminal() {
return Ok(CancelOutcome::AlreadyTerminal);
}
if row.cancel_requested {
return Ok(CancelOutcome::AlreadyRequested);
}
let now = OffsetDateTime::now_utc();
if phase == JobPhase::Running {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
} else {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
authority_attempt_id = NULL, dispatch_needed = FALSE,
terminal_at = $2, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
}
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(CancelOutcome::Requested)
}
pub async fn cooperative_stop(
&self,
authority: &AttemptAuthority,

25
crates/remora-attempt/src/service.rs
View file

@ -16,8 +16,8 @@ use time::OffsetDateTime;
use crate::{
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptError,
AttemptFaultInjector, AttemptFaultPoint, AttemptState, CheckpointProjection, ClaimOutcome,
ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, RecoveryOutcome,
RenewalMutationOutcome, RenewalOutcome,
ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, ProgressRecord, ProgressSnapshot,
RecoveryOutcome, RenewalMutationOutcome, RenewalOutcome,
};
struct NoFaults;
@ -87,6 +87,27 @@ impl AttemptService {
})
}
pub async fn progress_snapshot(
&self,
job_id: &OpaqueId,
) -> Result<Option<ProgressSnapshot>, AttemptError> {
let Some(hint) = self.coordination.get_progress(job_id.as_str()).await? else {
return Ok(None);
};
Ok(Some(ProgressSnapshot {
attempt_id: OpaqueId::parse(hint.attempt_id)
.map_err(|error| AttemptError::Invalid(error.to_string()))?,
fence: hint.fence,
progress: ProgressRecord {
completed_bytes: hint.bytes,
total_bytes: hint.total_bytes,
completed_members: hint.completed_members,
total_members: hint.total_members,
},
observed_at: OffsetDateTime::now_utc(),
}))
}
pub async fn claim(
&self,
job_id: &OpaqueId,

4
crates/remora-credentials/src/lib.rs
View file

@ -18,7 +18,9 @@ mod vault;
pub use authz::{CredentialCaller, may_manage, require_manage};
pub use error::CredentialError;
pub use migrate::{migrate, migrator};
pub use registry::CredentialRegistry;
pub use registry::{
CredentialRegistry, PreparedCredentialCreate, PreparedCredentialRotate, RotateTxOutcome,
};
pub use resolve::GrantBoundResolver;
pub use types::{
CreateCredential, CredentialGrantVersion, SecretMaterial, SourceCredential,

505
crates/remora-credentials/src/registry.rs
View file

@ -24,6 +24,30 @@ pub struct CredentialRegistry {
reconcile_min_age: time::Duration,
}
pub struct PreparedCredentialCreate {
intent_id: OpaqueId,
credential_id: OpaqueId,
version_id: OpaqueId,
owner: OpaqueId,
label: String,
kind: SourceCredentialKind,
vault_kv_version: u64,
secret_mount: String,
credential_path: String,
secret_path: String,
created_at: OffsetDateTime,
}
pub struct PreparedCredentialRotate {
intent_id: OpaqueId,
credential_id: OpaqueId,
version_id: OpaqueId,
vault_kv_version: u64,
secret_mount: String,
secret_path: String,
created_at: OffsetDateTime,
}
impl CredentialRegistry {
#[must_use]
pub fn new(pool: PgPool, vault: VaultClient) -> Self {
@ -46,6 +70,38 @@ impl CredentialRegistry {
owner: &OpaqueId,
request: CreateCredential,
) -> Result<SourceCredential, CredentialError> {
let prepared = self.prepare_create(owner, request).await?;
let credential_id = prepared.credential_id.clone();
let intent_id = prepared.intent_id.clone();
let now = prepared.created_at;
let result = with_transaction(&self.pool, move |tx| {
Box::pin(async move { Self::commit_prepared_create_in_tx(tx, prepared).await })
})
.await;
match result {
Ok(credential) => Ok(credential),
Err(error) => {
if let Ok(Some(credential)) = self.get_internal(&credential_id).await {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
return Ok(credential);
}
Err(error.into())
}
}
}
pub async fn prepare_create(
&self,
owner: &OpaqueId,
request: CreateCredential,
) -> Result<PreparedCredentialCreate, CredentialError> {
validate_label(&request.label)?;
request.material.validate()?;
let kind = request.material.kind();
@ -89,44 +145,53 @@ impl CredentialRegistry {
return Err(error);
}
};
// Durable KV version must be recorded before any later step; retry so a
// transient DB blip does not leave a Vault secret without vault_written.
record_vault_written_with_retry(&self.pool, &intent_id, vault_kv_version, now).await?;
Ok(PreparedCredentialCreate {
intent_id,
credential_id,
version_id,
owner: owner.clone(),
label: request.label,
kind,
vault_kv_version,
secret_mount,
credential_path,
secret_path,
created_at: now,
})
}
match self
.materialize_create_intent(
&intent_id,
&credential_id,
&version_id,
owner,
&request.label,
kind,
vault_kv_version,
&secret_mount,
&credential_path,
&secret_path,
now,
)
.await
pub async fn commit_prepared_create_in_tx(
tx: &mut PgTx<'_>,
prepared: PreparedCredentialCreate,
) -> Result<SourceCredential, DurableError> {
advisory_xact_lock(tx, credential_advisory_key(&prepared.credential_id)).await?;
if !claim_intent_durable_committed_in_tx(tx, &prepared.intent_id, prepared.created_at)
.await?
{
Ok(credential) => Ok(credential),
Err(error) => {
// Never destroy Vault on materialize failure: the durable commit
// may have succeeded with a lost response. Recover if rows exist.
if let Ok(Some(credential)) = self.get_internal(&credential_id).await {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
return Ok(credential);
}
Err(error)
}
return Err(DurableError::Effect(
"create intent was claimed by reconciliation".into(),
));
}
insert_credential_with_version(
tx,
InsertCredential {
credential_id: &prepared.credential_id,
owner: &prepared.owner,
label: &prepared.label,
kind: prepared.kind,
version_id: &prepared.version_id,
vault_kv_version: prepared.vault_kv_version,
secret_mount: &prepared.secret_mount,
credential_path: &prepared.credential_path,
secret_path: &prepared.secret_path,
now: prepared.created_at,
},
)
.await?;
get_internal_in_tx(tx, &prepared.credential_id)
.await?
.ok_or_else(|| DurableError::Effect("created credential is missing".into()))
}
pub async fn rotate(
@ -135,6 +200,66 @@ impl CredentialRegistry {
credential_id: &OpaqueId,
material: SecretMaterial,
) -> Result<SourceCredential, CredentialError> {
let prepared = self.prepare_rotate(caller, credential_id, material).await?;
let intent_id = prepared.intent_id.clone();
let version_id = prepared.version_id.clone();
let secret_mount = prepared.secret_mount.clone();
let secret_path = prepared.secret_path.clone();
let vault_kv_version = prepared.vault_kv_version;
let now = prepared.created_at;
let result = with_transaction(&self.pool, move |tx| {
Box::pin(async move { Self::commit_prepared_rotate_in_tx(tx, prepared).await })
})
.await;
match result {
Ok(RotateTxOutcome::Applied(credential)) => Ok(credential),
Ok(RotateTxOutcome::Revoked) => {
if self
.vault
.destroy_version(&secret_mount, &secret_path, vault_kv_version)
.await
.is_ok()
{
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::Abandoning,
IntentStatus::Abandoned,
now,
)
.await;
}
Err(CredentialError::Revoked)
}
Ok(RotateTxOutcome::IntentClaimed) => Err(CredentialError::Config(
"rotate intent was claimed by reconciliation".into(),
)),
Err(error) => {
if version_row_exists(&self.pool, &version_id).await? {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
if let Ok(Some(credential)) = self.get_internal(credential_id).await {
return Ok(credential);
}
}
Err(error.into())
}
}
}
pub async fn prepare_rotate(
&self,
caller: &CredentialCaller,
credential_id: &OpaqueId,
material: SecretMaterial,
) -> Result<PreparedCredentialRotate, CredentialError> {
material.validate()?;
let existing = self
.get(caller, credential_id)
@ -181,113 +306,71 @@ impl CredentialRegistry {
.put_version(&secret_mount, &secret_path, &material)
.await?;
record_vault_written_with_retry(&self.pool, &intent_id, vault_kv_version, now).await?;
let result = with_transaction(&self.pool, {
let credential_id = credential_id.clone();
let version_id = version_id.clone();
let intent_id = intent_id.clone();
let secret_mount = secret_mount.clone();
let secret_path = secret_path.clone();
move |tx| {
Box::pin(async move {
lock_credential(tx, &credential_id).await?;
let status: String = sqlx::query_scalar(
"SELECT status FROM source_credentials WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.fetch_optional(tx.as_mut())
.await?
.ok_or_else(|| DurableError::Effect("credential missing under lock".into()))?;
if status != "active" {
claim_intent_abandoning_in_tx(tx, &intent_id, vault_kv_version, now)
.await?;
return Ok(RotateTxOutcome::Revoked);
}
// Refuse to materialize if reconciler already claimed the intent.
if !claim_intent_durable_committed_in_tx(tx, &intent_id, now).await? {
return Ok(RotateTxOutcome::IntentClaimed);
}
sqlx::query(
"INSERT INTO source_credential_versions (
version_id, credential_id, vault_kv_version,
secret_mount, secret_path, created_at, destroyed_at
) VALUES ($1, $2, $3, $4, $5, $6, NULL)",
)
.bind(version_id.as_str())
.bind(credential_id.as_str())
.bind(vault_kv_version as i64)
.bind(&secret_mount)
.bind(&secret_path)
.bind(now)
.execute(tx.as_mut())
.await?;
sqlx::query(
"UPDATE source_credentials
SET active_version_id = $2, updated_at = $3
WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.bind(version_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
Ok(RotateTxOutcome::Applied)
})
}
Ok(PreparedCredentialRotate {
intent_id,
credential_id: credential_id.clone(),
version_id,
vault_kv_version,
secret_mount,
secret_path,
created_at: now,
})
.await;
}
match result {
Ok(RotateTxOutcome::Applied) => self
.get_internal(credential_id)
pub async fn commit_prepared_rotate_in_tx(
tx: &mut PgTx<'_>,
prepared: PreparedCredentialRotate,
) -> Result<RotateTxOutcome, DurableError> {
lock_credential(tx, &prepared.credential_id).await?;
let status: String =
sqlx::query_scalar("SELECT status FROM source_credentials WHERE credential_id = $1")
.bind(prepared.credential_id.as_str())
.fetch_optional(tx.as_mut())
.await?
.ok_or_else(|| CredentialError::Config("rotated credential is missing".into())),
Ok(RotateTxOutcome::Revoked) => {
match self
.vault
.destroy_version(&secret_mount, &secret_path, vault_kv_version)
.await
{
Ok(()) => {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::Abandoning,
IntentStatus::Abandoned,
now,
)
.await;
}
Err(_) => {
// Leave abandoning so reconciliation retries cleanup.
}
}
Err(CredentialError::Revoked)
}
Ok(RotateTxOutcome::IntentClaimed) => {
// Reconciler owns the intent; do not destroy or materialize.
Err(CredentialError::Config(
"rotate intent was claimed by reconciliation".into(),
))
}
Err(error) => {
// Ambiguous commit: do not destroy. Recover if the version row landed.
if version_row_exists(&self.pool, &version_id).await? {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
if let Ok(Some(credential)) = self.get_internal(credential_id).await {
return Ok(credential);
}
}
Err(error.into())
}
.ok_or_else(|| DurableError::Effect("credential missing under lock".into()))?;
if status != "active" {
claim_intent_abandoning_in_tx(
tx,
&prepared.intent_id,
prepared.vault_kv_version,
prepared.created_at,
)
.await?;
return Ok(RotateTxOutcome::Revoked);
}
if !claim_intent_durable_committed_in_tx(tx, &prepared.intent_id, prepared.created_at)
.await?
{
return Ok(RotateTxOutcome::IntentClaimed);
}
sqlx::query(
"INSERT INTO source_credential_versions (
version_id, credential_id, vault_kv_version,
secret_mount, secret_path, created_at, destroyed_at
) VALUES ($1, $2, $3, $4, $5, $6, NULL)",
)
.bind(prepared.version_id.as_str())
.bind(prepared.credential_id.as_str())
.bind(prepared.vault_kv_version as i64)
.bind(&prepared.secret_mount)
.bind(&prepared.secret_path)
.bind(prepared.created_at)
.execute(tx.as_mut())
.await?;
sqlx::query(
"UPDATE source_credentials
SET active_version_id = $2, updated_at = $3
WHERE credential_id = $1",
)
.bind(prepared.credential_id.as_str())
.bind(prepared.version_id.as_str())
.bind(prepared.created_at)
.execute(tx.as_mut())
.await?;
let credential = get_internal_in_tx(tx, &prepared.credential_id)
.await?
.ok_or_else(|| DurableError::Effect("rotated credential is missing".into()))?;
Ok(RotateTxOutcome::Applied(credential))
}
pub async fn revoke(
@ -295,38 +378,43 @@ impl CredentialRegistry {
caller: &CredentialCaller,
credential_id: &OpaqueId,
) -> Result<SourceCredential, CredentialError> {
let existing = self
.get(caller, credential_id)
let caller = caller.clone();
let credential_id = credential_id.clone();
with_transaction(&self.pool, {
move |tx| Box::pin(async move { Self::revoke_in_tx(tx, &caller, &credential_id).await })
})
.await
.map_err(Into::into)
}
pub async fn revoke_in_tx(
tx: &mut PgTx<'_>,
caller: &CredentialCaller,
credential_id: &OpaqueId,
) -> Result<SourceCredential, DurableError> {
lock_credential(tx, credential_id).await?;
let existing = get_internal_in_tx(tx, credential_id)
.await?
.ok_or(CredentialError::NotFound)?;
.ok_or_else(|| DurableError::Effect(CredentialError::NotFound.to_string()))?;
require_manage(caller, Some(&existing)).map_err(DurableError::effect)?;
if existing.status == SourceCredentialStatus::Revoked {
return Ok(existing);
}
let now = OffsetDateTime::now_utc();
with_transaction(&self.pool, {
let credential_id = credential_id.clone();
move |tx| {
Box::pin(async move {
lock_credential(tx, &credential_id).await?;
sqlx::query(
"UPDATE source_credentials
SET status = 'revoked',
active_version_id = NULL,
updated_at = $2
WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
Ok(())
})
}
})
sqlx::query(
"UPDATE source_credentials
SET status = 'revoked',
active_version_id = NULL,
updated_at = $2
WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
self.get_internal(credential_id)
get_internal_in_tx(tx, credential_id)
.await?
.ok_or_else(|| CredentialError::Config("revoked credential is missing".into()))
.ok_or_else(|| DurableError::Effect("revoked credential is missing".into()))
}
pub async fn get(
@ -473,7 +561,7 @@ impl CredentialRegistry {
})
}
/// Pin under an existing transaction for atomic composition (e.g. WP-14 admission).
/// Pin under an existing transaction for atomic composition with admission.
///
/// Takes the same credential advisory lock as revoke. Caller must already
/// authorize the owner/administrator relationship.
@ -557,8 +645,8 @@ impl CredentialRegistry {
/// Bind a pin to an execution grant for worker resolution.
///
/// WP-14 calls this when accepting a job. Resolution always re-reads this
/// durable binding; revocation fails closed even if workers retain the grant id.
/// Resolution always re-reads this durable binding, so revocation fails
/// closed even if workers retain the grant id.
pub async fn bind_execution_grant(
&self,
grant_id: &OpaqueId,
@ -950,66 +1038,6 @@ impl CredentialRegistry {
row.map(VersionCoordRow::into_coord).transpose()
}
#[allow(clippy::too_many_arguments)]
async fn materialize_create_intent(
&self,
intent_id: &OpaqueId,
credential_id: &OpaqueId,
version_id: &OpaqueId,
owner: &OpaqueId,
label: &str,
kind: SourceCredentialKind,
vault_kv_version: u64,
secret_mount: &str,
credential_path: &str,
secret_path: &str,
now: OffsetDateTime,
) -> Result<SourceCredential, CredentialError> {
with_transaction(&self.pool, {
let intent_id = intent_id.clone();
let credential_id = credential_id.clone();
let version_id = version_id.clone();
let owner = owner.clone();
let label = label.to_owned();
let secret_mount = secret_mount.to_owned();
let credential_path = credential_path.to_owned();
let secret_path = secret_path.to_owned();
move |tx| {
Box::pin(async move {
// Serialize with reconcile claim on this credential id.
advisory_xact_lock(tx, credential_advisory_key(&credential_id)).await?;
if !claim_intent_durable_committed_in_tx(tx, &intent_id, now).await? {
return Err(DurableError::Effect(
"create intent was claimed by reconciliation".into(),
));
}
insert_credential_with_version(
tx,
InsertCredential {
credential_id: &credential_id,
owner: &owner,
label: &label,
kind,
version_id: &version_id,
vault_kv_version,
secret_mount: &secret_mount,
credential_path: &credential_path,
secret_path: &secret_path,
now,
},
)
.await
.map_err(DurableError::from)?;
Ok(())
})
}
})
.await?;
self.get_internal(credential_id)
.await?
.ok_or_else(|| CredentialError::Config("created credential is missing".into()))
}
async fn get_internal(
&self,
credential_id: &OpaqueId,
@ -1048,6 +1076,27 @@ impl CredentialRegistry {
}
}
async fn get_internal_in_tx(
tx: &mut PgTx<'_>,
credential_id: &OpaqueId,
) -> Result<Option<SourceCredential>, DurableError> {
let row = sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE c.credential_id = $1",
)
.bind(credential_id.as_str())
.fetch_optional(tx.as_mut())
.await?;
row.map(CredentialRow::into_record)
.transpose()
.map_err(DurableError::effect)
}
const EFFECT_REVOKED: &str = "remora.credentials.revoked";
const EFFECT_NO_ACTIVE: &str = "remora.credentials.no_active_version";
const EFFECT_MISSING: &str = "remora.credentials.missing";
@ -1209,9 +1258,9 @@ async fn pin_current_version_in_tx(
})
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RotateTxOutcome {
Applied,
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum RotateTxOutcome {
Applied(SourceCredential),
Revoked,
IntentClaimed,
}

2
crates/remora-deletion/src/registry.rs
View file

@ -36,7 +36,7 @@ impl TargetRegistry {
.remove(&(target_store.to_owned(), bucket.to_owned()))
}
pub(crate) fn resolve(&self, target_store: &str, bucket: &str) -> Option<TargetStore> {
pub fn resolve(&self, target_store: &str, bucket: &str) -> Option<TargetStore> {
self.stores
.read()
.expect("target registry poisoned")

64
crates/remora-deletion/src/service.rs
View file

@ -2,7 +2,7 @@ use std::time::Duration;
use futures_util::TryStreamExt;
use remora_domain::{BucketName, OpaqueId, Target, TargetPrefix, TargetStoreId};
use remora_durable::enqueue;
use remora_durable::{PgTx, enqueue};
use remora_identity::{
AuthorizedResource, ManagementOperation, Principal, ResourceVisibility, resource_visibility,
};
@ -52,14 +52,23 @@ impl DeletionService {
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let mut tx = self.pool.begin().await?;
let row = lock_job(&mut tx, job_id).await?;
let outcome = Self::soft_delete_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn soft_delete_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::SoftDelete)?;
require_terminal(&row.phase)?;
if row.deletion_operation_id.is_some() {
return Err(DeletionError::HardDeleteStarted);
}
if row.lifecycle == "soft_deleted" {
tx.commit().await?;
return Ok(LifecycleMutation::Unchanged);
}
let now = OffsetDateTime::now_utc();
@ -72,8 +81,7 @@ impl DeletionService {
.bind(now)
.execute(tx.as_mut())
.await?;
enqueue(&mut tx, "job.updated", json!({ "job_id": job_id })).await?;
tx.commit().await?;
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(LifecycleMutation::Changed)
}
@ -83,14 +91,23 @@ impl DeletionService {
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let mut tx = self.pool.begin().await?;
let row = lock_job(&mut tx, job_id).await?;
let outcome = Self::restore_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn restore_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::Restore)?;
require_terminal(&row.phase)?;
if row.deletion_operation_id.is_some() {
return Err(DeletionError::HardDeleteStarted);
}
if row.lifecycle == "visible" {
tx.commit().await?;
return Ok(LifecycleMutation::Unchanged);
}
let now = OffsetDateTime::now_utc();
@ -103,8 +120,7 @@ impl DeletionService {
.bind(now)
.execute(tx.as_mut())
.await?;
enqueue(&mut tx, "job.updated", json!({ "job_id": job_id })).await?;
tx.commit().await?;
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(LifecycleMutation::Changed)
}
@ -113,21 +129,30 @@ impl DeletionService {
principal: &Principal,
job_id: &OpaqueId,
) -> Result<HardDeleteOutcome, DeletionError> {
let operation_id = new_id('d')?;
let mut tx = self.pool.begin().await?;
let row = lock_job(&mut tx, job_id).await?;
let outcome = Self::hard_delete_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn hard_delete_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<HardDeleteOutcome, DeletionError> {
let operation_id = new_id('d')?;
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::HardDelete)?;
require_terminal(&row.phase)?;
if let Some(existing) = row.deletion_operation_id {
let operation = load_operation_in_tx(&mut tx, &existing).await?;
tx.commit().await?;
let operation = load_operation_in_tx(tx, &existing).await?;
return Ok(HardDeleteOutcome {
mutation: LifecycleMutation::Unchanged,
operation,
});
}
validate_committed_inventory(&mut tx, &row).await?;
validate_deletion_identities(&mut tx, &row).await?;
validate_committed_inventory(tx, &row).await?;
validate_deletion_identities(tx, &row).await?;
let now = OffsetDateTime::now_utc();
sqlx::query(
"INSERT INTO deletion_operations (
@ -150,16 +175,15 @@ impl DeletionService {
.bind(now)
.execute(tx.as_mut())
.await?;
materialize_items(&mut tx, job_id, &operation_id, now).await?;
finalize_if_empty(&mut tx, job_id, &operation_id, now).await?;
materialize_items(tx, job_id, &operation_id, now).await?;
finalize_if_empty(tx, job_id, &operation_id, now).await?;
enqueue(
&mut tx,
tx,
"job.updated",
json!({ "job_id": job_id, "deletion_operation_id": operation_id }),
)
.await?;
let operation = load_operation_in_tx(&mut tx, operation_id.as_str()).await?;
tx.commit().await?;
let operation = load_operation_in_tx(tx, operation_id.as_str()).await?;
Ok(HardDeleteOutcome {
mutation: LifecycleMutation::Changed,
operation,

45
crates/remora-durable/src/ledger.rs
View file

@ -48,6 +48,51 @@ pub struct MutationOutcome {
pub response: Value,
}
pub async fn observe_mutation(
pool: &PgPool,
scope: MutationScope<'_>,
kind: &str,
eligibility_claim: &[u8],
request_fingerprint: RequestFingerprint,
) -> Result<Option<MutationOutcome>, DurableError> {
let existing = sqlx::query_as::<_, (String, Vec<u8>, Vec<u8>, String, Value)>(
"SELECT kind, eligibility_claim, request_fingerprint, response_status, response_body
FROM mutation_records
WHERE scope_subject = $1
AND scope_operation = $2
AND operation_key = $3",
)
.bind(scope.subject.as_str())
.bind(scope.operation)
.bind(scope.key)
.fetch_optional(pool)
.await?;
existing
.map(
|(
stored_kind,
stored_eligibility_claim,
stored_request_fingerprint,
response_status,
response_body,
)| {
observe_existing(
kind,
eligibility_claim,
request_fingerprint.as_bytes(),
StoredRecord {
kind: stored_kind,
eligibility_claim: stored_eligibility_claim,
request_fingerprint: stored_request_fingerprint,
response_status,
response_body,
},
)
},
)
.transpose()
}
#[derive(Clone, Debug)]
struct StoredRecord {
kind: String,

1
crates/remora-durable/src/lib.rs
View file

@ -19,6 +19,7 @@ pub use cursor::{CursorCodec, KeysetCursor, KeysetPosition, first_page_cursor, n
pub use error::DurableError;
pub use ledger::{
MutationDisposition, MutationOutcome, MutationStatus, execute_mutation, execute_mutation_in_tx,
observe_mutation,
};
pub use migrate::{migrate, migrator};
pub use outbox::{

40
tests/kind/public-api-job.yaml.in Normal file
View file

@ -0,0 +1,40 @@
apiVersion: batch/v1
kind: Job
metadata:
name: @@JOB_NAME@@
namespace: remora-testbed
spec:
activeDeadlineSeconds: 900
backoffLimit: 0
ttlSecondsAfterFinished: 300
template:
spec:
automountServiceAccountToken: false
enableServiceLinks: false
restartPolicy: Never
dnsPolicy: None
dnsConfig:
nameservers: [10.96.0.53]
searches: [remora-testbed.svc.cluster.local, svc.cluster.local, cluster.local]
options:
- {name: ndots, value: "5"}
containers:
- name: tests
image: @@TEST_IMAGE@@
imagePullPolicy: Never
envFrom:
- configMapRef: {name: @@PROFILE_NAME@@}
env:
- name: REMORA_PUBLIC_API_REQUIRE
value: "1"
volumeMounts:
- {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
resources:
requests: {cpu: 250m, memory: 512Mi}
limits: {cpu: "2", memory: 2Gi}
volumes:
- name: fixture-profile
configMap:
name: @@PROFILE_NAME@@-file
items:
- {key: profile.env, path: profile.env}

50
tests/kind/run-public-api-job.sh Executable file
View file

@ -0,0 +1,50 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
source "$ROOT/tests/kind/lib.sh"
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
SUCCEEDED=false
OWNS_FIXTURE=false
cleanup() {
local status=$?
trap - EXIT INT TERM
if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
printf 'retained context: %s\n' "$CONTEXT"
exit "$status"
fi
if [[ "$OWNS_FIXTURE" = true ]]; then
destroy_fixture || status=1
fi
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
printf 'user kubeconfig changed unexpectedly\n' >&2
status=1
fi
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
acquire_fixture_lock
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
export REMORA_REAL_NERDCTL
"$REMORA_REAL_NERDCTL" --namespace default build \
--file "$ROOT/Dockerfile.test" \
--target kind-public-api-runner \
--tag remora-kind-public-api:local \
"$ROOT"
OWNS_FIXTURE=true
setup_fixture durable coordination oidc target secret
load_local_image remora-kind-public-api:local
run_job_matrix \
public-api \
"$TESTBED/public-api-job.yaml.in" \
remora-kind-public-api:local \
durable coordination oidc target secret
SUCCEEDED=true
printf 'public API package test Jobs passed\n'