ms2k/remora
SHA256
Watch
0
Code Releases Activity

Implement replay-safe authorized public HTTP API

Browse source
This commit is contained in:
ms2k 2026-07-17 21:23:18 +09:00
commit 4c93c25692
2 changed files with 3940 additions and 345 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/remora-credentials/src/lib.rs
View file

@ -18,7 +18,9 @@ mod vault;
pub use authz::{CredentialCaller, may_manage, require_manage};
pub use error::CredentialError;
pub use migrate::{migrate, migrator};
pub use registry::CredentialRegistry;
pub use registry::{
CredentialRegistry, PreparedCredentialCreate, PreparedCredentialRotate, RotateTxOutcome,
};
pub use resolve::GrantBoundResolver;
pub use types::{
CreateCredential, CredentialGrantVersion, SecretMaterial, SourceCredential,

505
crates/remora-credentials/src/registry.rs
View file

@ -24,6 +24,30 @@ pub struct CredentialRegistry {
reconcile_min_age: time::Duration,
}
pub struct PreparedCredentialCreate {
intent_id: OpaqueId,
credential_id: OpaqueId,
version_id: OpaqueId,
owner: OpaqueId,
label: String,
kind: SourceCredentialKind,
vault_kv_version: u64,
secret_mount: String,
credential_path: String,
secret_path: String,
created_at: OffsetDateTime,
}
pub struct PreparedCredentialRotate {
intent_id: OpaqueId,
credential_id: OpaqueId,
version_id: OpaqueId,
vault_kv_version: u64,
secret_mount: String,
secret_path: String,
created_at: OffsetDateTime,
}
impl CredentialRegistry {
#[must_use]
pub fn new(pool: PgPool, vault: VaultClient) -> Self {
@ -46,6 +70,38 @@ impl CredentialRegistry {
owner: &OpaqueId,
request: CreateCredential,
) -> Result<SourceCredential, CredentialError> {
let prepared = self.prepare_create(owner, request).await?;
let credential_id = prepared.credential_id.clone();
let intent_id = prepared.intent_id.clone();
let now = prepared.created_at;
let result = with_transaction(&self.pool, move |tx| {
Box::pin(async move { Self::commit_prepared_create_in_tx(tx, prepared).await })
})
.await;
match result {
Ok(credential) => Ok(credential),
Err(error) => {
if let Ok(Some(credential)) = self.get_internal(&credential_id).await {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
return Ok(credential);
}
Err(error.into())
}
}
}
pub async fn prepare_create(
&self,
owner: &OpaqueId,
request: CreateCredential,
) -> Result<PreparedCredentialCreate, CredentialError> {
validate_label(&request.label)?;
request.material.validate()?;
let kind = request.material.kind();
@ -89,44 +145,53 @@ impl CredentialRegistry {
return Err(error);
}
};
// Durable KV version must be recorded before any later step; retry so a
// transient DB blip does not leave a Vault secret without vault_written.
record_vault_written_with_retry(&self.pool, &intent_id, vault_kv_version, now).await?;
Ok(PreparedCredentialCreate {
intent_id,
credential_id,
version_id,
owner: owner.clone(),
label: request.label,
kind,
vault_kv_version,
secret_mount,
credential_path,
secret_path,
created_at: now,
})
}
match self
.materialize_create_intent(
&intent_id,
&credential_id,
&version_id,
owner,
&request.label,
kind,
vault_kv_version,
&secret_mount,
&credential_path,
&secret_path,
now,
)
.await
pub async fn commit_prepared_create_in_tx(
tx: &mut PgTx<'_>,
prepared: PreparedCredentialCreate,
) -> Result<SourceCredential, DurableError> {
advisory_xact_lock(tx, credential_advisory_key(&prepared.credential_id)).await?;
if !claim_intent_durable_committed_in_tx(tx, &prepared.intent_id, prepared.created_at)
.await?
{
Ok(credential) => Ok(credential),
Err(error) => {
// Never destroy Vault on materialize failure: the durable commit
// may have succeeded with a lost response. Recover if rows exist.
if let Ok(Some(credential)) = self.get_internal(&credential_id).await {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
return Ok(credential);
}
Err(error)
}
return Err(DurableError::Effect(
"create intent was claimed by reconciliation".into(),
));
}
insert_credential_with_version(
tx,
InsertCredential {
credential_id: &prepared.credential_id,
owner: &prepared.owner,
label: &prepared.label,
kind: prepared.kind,
version_id: &prepared.version_id,
vault_kv_version: prepared.vault_kv_version,
secret_mount: &prepared.secret_mount,
credential_path: &prepared.credential_path,
secret_path: &prepared.secret_path,
now: prepared.created_at,
},
)
.await?;
get_internal_in_tx(tx, &prepared.credential_id)
.await?
.ok_or_else(|| DurableError::Effect("created credential is missing".into()))
}
pub async fn rotate(
@ -135,6 +200,66 @@ impl CredentialRegistry {
credential_id: &OpaqueId,
material: SecretMaterial,
) -> Result<SourceCredential, CredentialError> {
let prepared = self.prepare_rotate(caller, credential_id, material).await?;
let intent_id = prepared.intent_id.clone();
let version_id = prepared.version_id.clone();
let secret_mount = prepared.secret_mount.clone();
let secret_path = prepared.secret_path.clone();
let vault_kv_version = prepared.vault_kv_version;
let now = prepared.created_at;
let result = with_transaction(&self.pool, move |tx| {
Box::pin(async move { Self::commit_prepared_rotate_in_tx(tx, prepared).await })
})
.await;
match result {
Ok(RotateTxOutcome::Applied(credential)) => Ok(credential),
Ok(RotateTxOutcome::Revoked) => {
if self
.vault
.destroy_version(&secret_mount, &secret_path, vault_kv_version)
.await
.is_ok()
{
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::Abandoning,
IntentStatus::Abandoned,
now,
)
.await;
}
Err(CredentialError::Revoked)
}
Ok(RotateTxOutcome::IntentClaimed) => Err(CredentialError::Config(
"rotate intent was claimed by reconciliation".into(),
)),
Err(error) => {
if version_row_exists(&self.pool, &version_id).await? {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
if let Ok(Some(credential)) = self.get_internal(credential_id).await {
return Ok(credential);
}
}
Err(error.into())
}
}
}
pub async fn prepare_rotate(
&self,
caller: &CredentialCaller,
credential_id: &OpaqueId,
material: SecretMaterial,
) -> Result<PreparedCredentialRotate, CredentialError> {
material.validate()?;
let existing = self
.get(caller, credential_id)
@ -181,113 +306,71 @@ impl CredentialRegistry {
.put_version(&secret_mount, &secret_path, &material)
.await?;
record_vault_written_with_retry(&self.pool, &intent_id, vault_kv_version, now).await?;
let result = with_transaction(&self.pool, {
let credential_id = credential_id.clone();
let version_id = version_id.clone();
let intent_id = intent_id.clone();
let secret_mount = secret_mount.clone();
let secret_path = secret_path.clone();
move |tx| {
Box::pin(async move {
lock_credential(tx, &credential_id).await?;
let status: String = sqlx::query_scalar(
"SELECT status FROM source_credentials WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.fetch_optional(tx.as_mut())
.await?
.ok_or_else(|| DurableError::Effect("credential missing under lock".into()))?;
if status != "active" {
claim_intent_abandoning_in_tx(tx, &intent_id, vault_kv_version, now)
.await?;
return Ok(RotateTxOutcome::Revoked);
}
// Refuse to materialize if reconciler already claimed the intent.
if !claim_intent_durable_committed_in_tx(tx, &intent_id, now).await? {
return Ok(RotateTxOutcome::IntentClaimed);
}
sqlx::query(
"INSERT INTO source_credential_versions (
version_id, credential_id, vault_kv_version,
secret_mount, secret_path, created_at, destroyed_at
) VALUES ($1, $2, $3, $4, $5, $6, NULL)",
)
.bind(version_id.as_str())
.bind(credential_id.as_str())
.bind(vault_kv_version as i64)
.bind(&secret_mount)
.bind(&secret_path)
.bind(now)
.execute(tx.as_mut())
.await?;
sqlx::query(
"UPDATE source_credentials
SET active_version_id = $2, updated_at = $3
WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.bind(version_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
Ok(RotateTxOutcome::Applied)
})
}
Ok(PreparedCredentialRotate {
intent_id,
credential_id: credential_id.clone(),
version_id,
vault_kv_version,
secret_mount,
secret_path,
created_at: now,
})
.await;
}
match result {
Ok(RotateTxOutcome::Applied) => self
.get_internal(credential_id)
pub async fn commit_prepared_rotate_in_tx(
tx: &mut PgTx<'_>,
prepared: PreparedCredentialRotate,
) -> Result<RotateTxOutcome, DurableError> {
lock_credential(tx, &prepared.credential_id).await?;
let status: String =
sqlx::query_scalar("SELECT status FROM source_credentials WHERE credential_id = $1")
.bind(prepared.credential_id.as_str())
.fetch_optional(tx.as_mut())
.await?
.ok_or_else(|| CredentialError::Config("rotated credential is missing".into())),
Ok(RotateTxOutcome::Revoked) => {
match self
.vault
.destroy_version(&secret_mount, &secret_path, vault_kv_version)
.await
{
Ok(()) => {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::Abandoning,
IntentStatus::Abandoned,
now,
)
.await;
}
Err(_) => {
// Leave abandoning so reconciliation retries cleanup.
}
}
Err(CredentialError::Revoked)
}
Ok(RotateTxOutcome::IntentClaimed) => {
// Reconciler owns the intent; do not destroy or materialize.
Err(CredentialError::Config(
"rotate intent was claimed by reconciliation".into(),
))
}
Err(error) => {
// Ambiguous commit: do not destroy. Recover if the version row landed.
if version_row_exists(&self.pool, &version_id).await? {
let _ = mark_intent_status_if(
&self.pool,
&intent_id,
IntentStatus::VaultWritten,
IntentStatus::DurableCommitted,
now,
)
.await;
if let Ok(Some(credential)) = self.get_internal(credential_id).await {
return Ok(credential);
}
}
Err(error.into())
}
.ok_or_else(|| DurableError::Effect("credential missing under lock".into()))?;
if status != "active" {
claim_intent_abandoning_in_tx(
tx,
&prepared.intent_id,
prepared.vault_kv_version,
prepared.created_at,
)
.await?;
return Ok(RotateTxOutcome::Revoked);
}
if !claim_intent_durable_committed_in_tx(tx, &prepared.intent_id, prepared.created_at)
.await?
{
return Ok(RotateTxOutcome::IntentClaimed);
}
sqlx::query(
"INSERT INTO source_credential_versions (
version_id, credential_id, vault_kv_version,
secret_mount, secret_path, created_at, destroyed_at
) VALUES ($1, $2, $3, $4, $5, $6, NULL)",
)
.bind(prepared.version_id.as_str())
.bind(prepared.credential_id.as_str())
.bind(prepared.vault_kv_version as i64)
.bind(&prepared.secret_mount)
.bind(&prepared.secret_path)
.bind(prepared.created_at)
.execute(tx.as_mut())
.await?;
sqlx::query(
"UPDATE source_credentials
SET active_version_id = $2, updated_at = $3
WHERE credential_id = $1",
)
.bind(prepared.credential_id.as_str())
.bind(prepared.version_id.as_str())
.bind(prepared.created_at)
.execute(tx.as_mut())
.await?;
let credential = get_internal_in_tx(tx, &prepared.credential_id)
.await?
.ok_or_else(|| DurableError::Effect("rotated credential is missing".into()))?;
Ok(RotateTxOutcome::Applied(credential))
}
pub async fn revoke(
@ -295,38 +378,43 @@ impl CredentialRegistry {
caller: &CredentialCaller,
credential_id: &OpaqueId,
) -> Result<SourceCredential, CredentialError> {
let existing = self
.get(caller, credential_id)
let caller = caller.clone();
let credential_id = credential_id.clone();
with_transaction(&self.pool, {
move |tx| Box::pin(async move { Self::revoke_in_tx(tx, &caller, &credential_id).await })
})
.await
.map_err(Into::into)
}
pub async fn revoke_in_tx(
tx: &mut PgTx<'_>,
caller: &CredentialCaller,
credential_id: &OpaqueId,
) -> Result<SourceCredential, DurableError> {
lock_credential(tx, credential_id).await?;
let existing = get_internal_in_tx(tx, credential_id)
.await?
.ok_or(CredentialError::NotFound)?;
.ok_or_else(|| DurableError::Effect(CredentialError::NotFound.to_string()))?;
require_manage(caller, Some(&existing)).map_err(DurableError::effect)?;
if existing.status == SourceCredentialStatus::Revoked {
return Ok(existing);
}
let now = OffsetDateTime::now_utc();
with_transaction(&self.pool, {
let credential_id = credential_id.clone();
move |tx| {
Box::pin(async move {
lock_credential(tx, &credential_id).await?;
sqlx::query(
"UPDATE source_credentials
SET status = 'revoked',
active_version_id = NULL,
updated_at = $2
WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
Ok(())
})
}
})
sqlx::query(
"UPDATE source_credentials
SET status = 'revoked',
active_version_id = NULL,
updated_at = $2
WHERE credential_id = $1",
)
.bind(credential_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
self.get_internal(credential_id)
get_internal_in_tx(tx, credential_id)
.await?
.ok_or_else(|| CredentialError::Config("revoked credential is missing".into()))
.ok_or_else(|| DurableError::Effect("revoked credential is missing".into()))
}
pub async fn get(
@ -473,7 +561,7 @@ impl CredentialRegistry {
})
}
/// Pin under an existing transaction for atomic composition (e.g. WP-14 admission).
/// Pin under an existing transaction for atomic composition with admission.
///
/// Takes the same credential advisory lock as revoke. Caller must already
/// authorize the owner/administrator relationship.
@ -557,8 +645,8 @@ impl CredentialRegistry {
/// Bind a pin to an execution grant for worker resolution.
///
/// WP-14 calls this when accepting a job. Resolution always re-reads this
/// durable binding; revocation fails closed even if workers retain the grant id.
/// Resolution always re-reads this durable binding, so revocation fails
/// closed even if workers retain the grant id.
pub async fn bind_execution_grant(
&self,
grant_id: &OpaqueId,
@ -950,66 +1038,6 @@ impl CredentialRegistry {
row.map(VersionCoordRow::into_coord).transpose()
}
#[allow(clippy::too_many_arguments)]
async fn materialize_create_intent(
&self,
intent_id: &OpaqueId,
credential_id: &OpaqueId,
version_id: &OpaqueId,
owner: &OpaqueId,
label: &str,
kind: SourceCredentialKind,
vault_kv_version: u64,
secret_mount: &str,
credential_path: &str,
secret_path: &str,
now: OffsetDateTime,
) -> Result<SourceCredential, CredentialError> {
with_transaction(&self.pool, {
let intent_id = intent_id.clone();
let credential_id = credential_id.clone();
let version_id = version_id.clone();
let owner = owner.clone();
let label = label.to_owned();
let secret_mount = secret_mount.to_owned();
let credential_path = credential_path.to_owned();
let secret_path = secret_path.to_owned();
move |tx| {
Box::pin(async move {
// Serialize with reconcile claim on this credential id.
advisory_xact_lock(tx, credential_advisory_key(&credential_id)).await?;
if !claim_intent_durable_committed_in_tx(tx, &intent_id, now).await? {
return Err(DurableError::Effect(
"create intent was claimed by reconciliation".into(),
));
}
insert_credential_with_version(
tx,
InsertCredential {
credential_id: &credential_id,
owner: &owner,
label: &label,
kind,
version_id: &version_id,
vault_kv_version,
secret_mount: &secret_mount,
credential_path: &credential_path,
secret_path: &secret_path,
now,
},
)
.await
.map_err(DurableError::from)?;
Ok(())
})
}
})
.await?;
self.get_internal(credential_id)
.await?
.ok_or_else(|| CredentialError::Config("created credential is missing".into()))
}
async fn get_internal(
&self,
credential_id: &OpaqueId,
@ -1048,6 +1076,27 @@ impl CredentialRegistry {
}
}
async fn get_internal_in_tx(
tx: &mut PgTx<'_>,
credential_id: &OpaqueId,
) -> Result<Option<SourceCredential>, DurableError> {
let row = sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE c.credential_id = $1",
)
.bind(credential_id.as_str())
.fetch_optional(tx.as_mut())
.await?;
row.map(CredentialRow::into_record)
.transpose()
.map_err(DurableError::effect)
}
const EFFECT_REVOKED: &str = "remora.credentials.revoked";
const EFFECT_NO_ACTIVE: &str = "remora.credentials.no_active_version";
const EFFECT_MISSING: &str = "remora.credentials.missing";
@ -1209,9 +1258,9 @@ async fn pin_current_version_in_tx(
})
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum RotateTxOutcome {
Applied,
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum RotateTxOutcome {
Applied(SourceCredential),
Revoked,
IntentClaimed,
}