use std::sync::Arc; use std::time::Duration; use remora_credentials::{ ActiveCredentialGrant, CreateCredential, CredentialCaller, CredentialGrantVersion, CredentialRegistry, GrantBoundResolver, SecretMaterial, VaultClient, VaultConfig, }; use remora_domain::OpaqueId; use remora_domain::ports::CredentialResolver; use remora_durable::{PoolConfig, connect}; use remora_schema::migrate as migrate_schema; use sqlx::PgPool; use tokio::sync::Barrier; fn database_url() -> Option { std::env::var("DATABASE_URL") .ok() .filter(|value| !value.is_empty()) } fn require_env() -> bool { std::env::var("REMORA_CREDENTIALS_REQUIRE").is_ok() } fn vault_ready() -> bool { std::env::var("REMORA_VAULT_ADDR") .ok() .filter(|v| !v.is_empty()) .is_some() && std::env::var("REMORA_VAULT_TOKEN") .ok() .filter(|v| !v.is_empty()) .is_some() && std::env::var("REMORA_VAULT_PATH") .ok() .filter(|v| !v.is_empty()) .is_some() } async fn test_pool() -> Option { let Some(url) = database_url() else { if require_env() { panic!("DATABASE_URL is required when REMORA_CREDENTIALS_REQUIRE is set"); } eprintln!("skipping credentials durable test: DATABASE_URL is unset"); return None; }; let pool = connect( &url, PoolConfig { max_connections: 8, acquire_timeout: Duration::from_secs(10), idle_timeout: Some(Duration::from_secs(30)), }, ) .await .expect("connect postgres"); migrate_schema(&pool).await.expect("schema migrate"); Some(pool) } fn vault_client() -> Option { if !vault_ready() { if require_env() { panic!("vault env is required when REMORA_CREDENTIALS_REQUIRE is set"); } eprintln!("skipping credentials vault test: vault env is unset"); return None; } let config = VaultConfig::from_env().expect("vault config"); Some(VaultClient::new(config).expect("vault client")) } fn owner(label: &str) -> OpaqueId { OpaqueId::parse(format!("owner-{label}-aaaaaaaa")).expect("owner id") } fn grant_id(label: &str) -> OpaqueId { OpaqueId::parse(format!("grant-{label}-aaaaaaaa")).expect("grant id") } fn caller(owner: &OpaqueId) -> CredentialCaller { CredentialCaller { principal_id: owner.clone(), administrator: false, } } fn admin() -> CredentialCaller { CredentialCaller { principal_id: OpaqueId::parse("admin-principal-aaaaaa").unwrap(), administrator: true, } } fn token_material(value: &str) -> SecretMaterial { SecretMaterial::BearerToken { token: value.to_owned(), } } async fn bind_grant( pool: &PgPool, registry: &CredentialRegistry, label: &str, pin: &CredentialGrantVersion, ) -> ActiveCredentialGrant { let grant_id = grant_id(label); let job_id = format!("job-{label}-aaaaaaaa"); let sensitive_source_ref = format!("source-{label}-aaaaaaaa"); let mut tx = pool.begin().await.expect("begin authoritative grant"); sqlx::query( "INSERT INTO sensitive_source_inputs (sensitive_source_ref, source_input, created_at) VALUES ($1, '{}'::jsonb, now())", ) .bind(&sensitive_source_ref) .execute(&mut *tx) .await .expect("insert sensitive source"); sqlx::query( "INSERT INTO execution_grants ( grant_id, owner_id, target_store, target_bucket, target_prefix, policy_version, credential_id, credential_version_id, status, created_at ) VALUES ($1, 'owner-fixture-aaaaaaaa', 'fixture', 'fixture', '[]'::jsonb, 'fixture', $2, $3, 'active', now())", ) .bind(grant_id.as_str()) .bind(pin.credential_id.as_str()) .bind(pin.version_id.as_str()) .execute(&mut *tx) .await .expect("insert authoritative grant"); sqlx::query( "INSERT INTO jobs ( job_id, owner_id, phase, lifecycle, source_projection, sensitive_source_ref, target_store, target_bucket, target_prefix, execution_grant_id, dispatch_needed, created_at, updated_at ) VALUES ($1, 'owner-fixture-aaaaaaaa', 'queued', 'visible', '{}'::jsonb, $2, 'fixture', 'fixture', '[]'::jsonb, $3, false, now(), now())", ) .bind(job_id) .bind(sensitive_source_ref) .bind(grant_id.as_str()) .execute(&mut *tx) .await .expect("insert authoritative job"); tx.commit().await.expect("commit authoritative grant"); registry .bind_execution_grant(&grant_id, pin) .await .expect("bind grant") } async fn harness() -> Option<(PgPool, CredentialRegistry, GrantBoundResolver)> { let pool = test_pool().await?; let vault = vault_client()?; let registry = CredentialRegistry::new(pool.clone(), vault.clone()); let resolver = GrantBoundResolver::new(registry.clone(), vault); Some((pool, registry, resolver)) } #[tokio::test] async fn create_pin_and_exact_version_resolution() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("exact1"); let created = registry .create( &owner, CreateCredential { label: "exact-version".into(), material: token_material("secret-version-one-value"), }, ) .await .expect("create"); let pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("pin"); let access = bind_grant(&pool, ®istry, "exact1", &pin).await; let secret = resolver .resolve_access(&access) .await .expect("resolve pinned"); assert_eq!(secret, token_material("secret-version-one-value")); } #[tokio::test] async fn vault_create_only_write_rejects_an_occupied_intent_path() { let Some(vault) = vault_client() else { return; }; let credential_id = OpaqueId::parse(format!( "ccas{:028x}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() )) .expect("credential id"); let version_id = OpaqueId::parse(format!( "vcas{:028x}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() )) .expect("version id"); let mount = vault.config().mount.clone(); let path = vault.config().secret_path_for(&credential_id, &version_id); let first = token_material("secret-cas-first"); let version = vault .put_version(&mount, &path, &first) .await .expect("first CAS write"); assert_eq!(version, 1); assert!( vault .put_version(&mount, &path, &token_material("secret-cas-second")) .await .is_err() ); assert_eq!( vault .get_version(&mount, &path, version) .await .expect("first version remains"), first ); vault .destroy_version(&mount, &path, version) .await .expect("cleanup CAS fixture"); } #[tokio::test] async fn rotation_does_not_change_prior_pin_and_rejects_substitution() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("rotate"); let created = registry .create( &owner, CreateCredential { label: "rotate-me".into(), material: token_material("secret-version-one-value"), }, ) .await .expect("create"); let first_pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("first pin"); let first_access = bind_grant(&pool, ®istry, "rotate-a", &first_pin).await; registry .rotate( &caller(&owner), &created.id, token_material("secret-version-two-value"), ) .await .expect("rotate"); let second_pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("second pin"); let second_access = bind_grant(&pool, ®istry, "rotate-b", &second_pin).await; assert_eq!( resolver.resolve_access(&first_access).await.expect("first"), token_material("secret-version-one-value") ); assert_eq!( resolver .resolve_access(&second_access) .await .expect("second"), token_material("secret-version-two-value") ); // Forged grant id (never bound) fails closed. let forged = ActiveCredentialGrant::for_execution_grant(grant_id("forged0")); assert!(matches!( resolver.resolve_access(&forged).await, Err(remora_credentials::CredentialError::GrantInactive) )); } #[tokio::test] async fn unbound_grant_handle_cannot_resolve() { let Some((_, _, resolver)) = harness().await else { return; }; // Constructing a handle does not authorize resolution without durable bind. let access = ActiveCredentialGrant::for_execution_grant(grant_id("unbound")); assert!(matches!( resolver.resolve_access(&access).await, Err(remora_credentials::CredentialError::GrantInactive) )); } #[tokio::test] async fn execution_grant_revocation_fails_closed_on_retained_handle() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("grev01"); let created = registry .create( &owner, CreateCredential { label: "grant-revoke".into(), material: token_material("secret-grant-revoke"), }, ) .await .expect("create"); let pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("pin"); let gid = grant_id("grev01"); let access = bind_grant(&pool, ®istry, "grev01", &pin).await; resolver .resolve_access(&access) .await .expect("resolve while active"); let mut tx = pool.begin().await.expect("begin revocation"); sqlx::query( "UPDATE execution_grants SET status = 'revoked', revoked_at = now() WHERE grant_id = $1", ) .bind(gid.as_str()) .execute(&mut *tx) .await .expect("revoke authoritative grant"); sqlx::query( "UPDATE execution_grant_credential_pins SET status = 'revoked', revoked_at = now() WHERE grant_id = $1", ) .bind(gid.as_str()) .execute(&mut *tx) .await .expect("revoke grant pin"); tx.commit().await.expect("commit revocation"); // Same retained handle must fail after durable revocation. assert!(matches!( resolver.resolve_access(&access).await, Err(remora_credentials::CredentialError::GrantInactive) )); } #[tokio::test] async fn credential_revocation_blocks_new_pins() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("revoke"); let created = registry .create( &owner, CreateCredential { label: "revocable".into(), material: token_material("secret-revocable-value"), }, ) .await .expect("create"); let pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("pin before revoke"); let access = bind_grant(&pool, ®istry, "crevoke", &pin).await; registry .revoke(&caller(&owner), &created.id) .await .expect("revoke"); // Prior execution grant still active until grant revoke. assert_eq!( resolver.resolve_access(&access).await.expect("prior grant"), token_material("secret-revocable-value") ); let new_pin = registry .pin_current_version(&caller(&owner), &created.id) .await; assert!(matches!( new_pin, Err(remora_credentials::CredentialError::Revoked) )); } #[tokio::test] async fn pin_and_revoke_serialize_under_lock() { let Some((_, registry, _)) = harness().await else { return; }; let owner = owner("race01"); let created = registry .create( &owner, CreateCredential { label: "race".into(), material: token_material("secret-race-value"), }, ) .await .expect("create"); let registry = Arc::new(registry); let barrier = Arc::new(Barrier::new(2)); let pin_registry = Arc::clone(®istry); let rev_registry = Arc::clone(®istry); let pin_barrier = Arc::clone(&barrier); let rev_barrier = Arc::clone(&barrier); let pin_owner = owner.clone(); let rev_owner = owner.clone(); let credential_id = created.id.clone(); let credential_id_rev = created.id.clone(); let pin_handle = tokio::spawn(async move { pin_barrier.wait().await; pin_registry .pin_current_version(&caller(&pin_owner), &credential_id) .await }); let rev_handle = tokio::spawn(async move { rev_barrier.wait().await; rev_registry .revoke(&caller(&rev_owner), &credential_id_rev) .await }); let pin_result = pin_handle.await.expect("join pin"); rev_handle.await.expect("join rev").expect("revoke"); match pin_result { Ok(_) => { let after = registry .pin_current_version(&caller(&owner), &created.id) .await; assert!(matches!( after, Err(remora_credentials::CredentialError::Revoked) | Err(remora_credentials::CredentialError::NoActiveVersion) )); } Err(remora_credentials::CredentialError::Revoked) | Err(remora_credentials::CredentialError::NoActiveVersion) => {} Err(other) => panic!("unexpected pin error: {other:?}"), } } #[tokio::test] async fn destroyed_version_fails_closed() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("destroy"); let created = registry .create( &owner, CreateCredential { label: "destroy-me".into(), material: token_material("secret-destroy-value"), }, ) .await .expect("create"); let pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("pin"); let access = bind_grant(&pool, ®istry, "destroy", &pin).await; registry .destroy_version(&caller(&owner), &pin.credential_id, &pin.version_id) .await .expect("destroy"); assert!(matches!( resolver.resolve_access(&access).await, Err(remora_credentials::CredentialError::Unresolvable) )); assert!(matches!( registry .pin_current_version(&caller(&owner), &created.id) .await, Err(remora_credentials::CredentialError::NoActiveVersion) )); } #[tokio::test] async fn pin_after_destroying_prior_version_uses_new_active() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("pindest"); let created = registry .create( &owner, CreateCredential { label: "pin-destroy".into(), material: token_material("secret-pin-destroy-one"), }, ) .await .expect("create"); let first_pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("first pin"); let first_access = bind_grant(&pool, ®istry, "pindest-a", &first_pin).await; registry .rotate( &caller(&owner), &created.id, token_material("secret-pin-destroy-two"), ) .await .expect("rotate"); let second_pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("second pin"); let second_access = bind_grant(&pool, ®istry, "pindest-b", &second_pin).await; registry .destroy_version( &caller(&owner), &first_pin.credential_id, &first_pin.version_id, ) .await .expect("destroy prior"); assert!(matches!( resolver.resolve_access(&first_access).await, Err(remora_credentials::CredentialError::Unresolvable) )); assert_eq!( resolver .resolve_access(&second_access) .await .expect("current"), token_material("secret-pin-destroy-two") ); } #[tokio::test] async fn secret_store_outage_on_resolve_fails_closed() { let Some((pool, good_registry, _)) = harness().await else { return; }; let owner = owner("outage2"); let created = good_registry .create( &owner, CreateCredential { label: "outage2".into(), material: token_material("secret-outage-two-value"), }, ) .await .expect("create"); let pin = good_registry .pin_current_version(&caller(&owner), &created.id) .await .expect("pin"); let access = bind_grant(&pool, &good_registry, "outage2", &pin).await; let bad_vault = VaultClient::new(VaultConfig { address: "http://127.0.0.1:1".into(), token: "unused".into(), mount: "secret".into(), path_prefix: "remora/outage".into(), timeout: Duration::from_millis(250), }) .expect("bad client"); let resolver = GrantBoundResolver::new( CredentialRegistry::new( pool, vault_client().expect("good vault for registry metadata"), ), bad_vault, ); assert!(matches!( resolver.resolve_access(&access).await, Err(remora_credentials::CredentialError::SecretStoreUnavailable) )); } #[tokio::test] async fn authorization_hides_peer_credentials() { let Some((_, registry, _)) = harness().await else { return; }; let owner_id = owner("authz1"); let peer_id = owner("authz2"); let created = registry .create( &owner_id, CreateCredential { label: "private".into(), material: token_material("secret-private-value"), }, ) .await .expect("create"); assert!( registry .get(&caller(&peer_id), &created.id) .await .expect("get") .is_none() ); assert!(matches!( registry .pin_current_version(&caller(&peer_id), &created.id) .await, Err(remora_credentials::CredentialError::NotFound) )); assert_eq!( registry .get(&admin(), &created.id) .await .expect("admin") .expect("visible") .id, created.id ); } #[tokio::test] async fn concurrent_rotates_all_succeed_and_prior_pin_holds() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("conc01"); let created = registry .create( &owner, CreateCredential { label: "concurrent".into(), material: token_material("secret-concurrent-base"), }, ) .await .expect("create"); let first_pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("first pin"); let first_access = bind_grant(&pool, ®istry, "conc01", &first_pin).await; let registry = Arc::new(registry); let barrier = Arc::new(Barrier::new(4)); let mut handles = Vec::new(); for index in 0..4 { let registry = Arc::clone(®istry); let barrier = Arc::clone(&barrier); let owner = owner.clone(); let credential_id = created.id.clone(); handles.push(tokio::spawn(async move { barrier.wait().await; registry .rotate( &caller(&owner), &credential_id, token_material(&format!("secret-concurrent-rot-{index}")), ) .await })); } for handle in handles { handle.await.expect("join").expect("rotate"); } let version_count: i64 = sqlx::query_scalar( "SELECT COUNT(*) FROM source_credential_versions WHERE credential_id = $1", ) .bind(created.id.as_str()) .fetch_one(&pool) .await .expect("count"); assert_eq!(version_count, 5); let distinct_paths: i64 = sqlx::query_scalar( "SELECT COUNT(DISTINCT secret_path) FROM source_credential_versions WHERE credential_id = $1 AND vault_kv_version = 1", ) .bind(created.id.as_str()) .fetch_one(&pool) .await .expect("distinct paths"); assert_eq!(distinct_paths, 5); assert_eq!( resolver .resolve_access(&first_access) .await .expect("original pin"), token_material("secret-concurrent-base") ); } #[tokio::test] async fn domain_port_resolve_requires_durable_grant() { let Some((pool, registry, resolver)) = harness().await else { return; }; let owner = owner("port01"); let created = registry .create( &owner, CreateCredential { label: "port".into(), material: token_material("secret-port-value"), }, ) .await .expect("create"); let pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("pin"); let access = bind_grant(&pool, ®istry, "port01", &pin).await; let via_port = CredentialResolver::resolve(&resolver, &access) .await .expect("port resolve"); assert_eq!(via_port, token_material("secret-port-value")); } #[tokio::test] async fn reconcile_skips_fresh_vault_written_intents() { let Some((pool, registry, _)) = harness().await else { return; }; // Age-gate: a vault_written row younger than min age must not be abandoned. let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos(); let intent_id = format!("ifresh{nanos:028x}"); let credential_id = "creconyoung00000000000000001"; let version_id = "vreconyoung00000000000000001"; sqlx::query( "INSERT INTO secret_write_intents ( intent_id, operation, credential_id, version_id, owner_id, label, credential_kind, secret_mount, secret_path, vault_kv_version, status, created_at, updated_at ) VALUES ($1, 'rotate', $2, $3, NULL, NULL, NULL, 'secret', 'orphan/path', 9, 'vault_written', now(), now())", ) .bind(&intent_id) .bind(credential_id) .bind(version_id) .execute(&pool) .await .expect("insert fresh intent"); let _ = registry .reconcile_secret_write_intents() .await .expect("reconcile"); let status: String = sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1") .bind(&intent_id) .fetch_one(&pool) .await .expect("status"); assert_eq!( status, "vault_written", "fresh vault_written intents must not be cleaned by reconcile" ); } #[tokio::test] async fn reconcile_destroys_aged_orphan_rotate_under_lock() { let Some((pool, registry, _)) = harness().await else { return; }; let owner = owner("recon2"); let created = registry .create( &owner, CreateCredential { label: "recon-rotate".into(), material: token_material("secret-recon-base"), }, ) .await .expect("create"); // Perform a rotate, then manually roll the intent back to vault_written without // the version row to simulate stuck materialize after age boundary. let rotated = registry .rotate( &caller(&owner), &created.id, token_material("secret-recon-rot"), ) .await .expect("rotate"); let version_id = rotated.active_version.expect("active").id; // Delete durable version + reset intent to vault_written aged (simulates lost commit // recovery candidate). Keep vault secret. let (secret_mount, secret_path, vault_kv): (String, String, i64) = sqlx::query_as( "SELECT v.secret_mount, v.secret_path, v.vault_kv_version FROM source_credential_versions v WHERE v.version_id = $1", ) .bind(version_id.as_str()) .fetch_one(&pool) .await .expect("coords"); sqlx::query( "UPDATE source_credentials SET active_version_id = ( SELECT version_id FROM source_credential_versions WHERE credential_id = $1 AND version_id <> $2 ORDER BY created_at ASC LIMIT 1 ) WHERE credential_id = $1", ) .bind(created.id.as_str()) .bind(version_id.as_str()) .execute(&pool) .await .expect("point active at first"); sqlx::query("DELETE FROM source_credential_versions WHERE version_id = $1") .bind(version_id.as_str()) .execute(&pool) .await .expect("drop version row"); let intent_id: String = sqlx::query_scalar( "SELECT intent_id FROM secret_write_intents WHERE version_id = $1 ORDER BY created_at DESC LIMIT 1", ) .bind(version_id.as_str()) .fetch_one(&pool) .await .expect("intent"); sqlx::query( "UPDATE secret_write_intents SET status = 'vault_written', vault_kv_version = $2, updated_at = now() - interval '2 hours' WHERE intent_id = $1", ) .bind(&intent_id) .bind(vault_kv) .execute(&pool) .await .expect("age orphan intent"); let handled = registry .reconcile_secret_write_intents() .await .expect("reconcile orphan"); assert!(handled >= 1, "expected orphan cleanup"); let status: String = sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1") .bind(&intent_id) .fetch_one(&pool) .await .expect("status"); assert_eq!(status, "abandoned"); // Vault version destroyed: read fails closed. let vault = vault_client().expect("vault"); assert!( vault .get_version(&secret_mount, &secret_path, vault_kv as u64) .await .is_err() ); } #[tokio::test] async fn reconcile_claim_blocks_concurrent_materialize() { let Some((pool, registry, _)) = harness().await else { return; }; let owner = owner("claim1"); let created = registry .create( &owner, CreateCredential { label: "claim-rotate".into(), material: token_material("secret-claim-base"), }, ) .await .expect("create"); let rotated = registry .rotate( &caller(&owner), &created.id, token_material("secret-claim-rot"), ) .await .expect("rotate"); let version_id = rotated.active_version.expect("active").id; let (secret_mount, secret_path, vault_kv): (String, String, i64) = sqlx::query_as( "SELECT v.secret_mount, v.secret_path, v.vault_kv_version FROM source_credential_versions v WHERE v.version_id = $1", ) .bind(version_id.as_str()) .fetch_one(&pool) .await .expect("coords"); sqlx::query( "UPDATE source_credentials SET active_version_id = ( SELECT version_id FROM source_credential_versions WHERE credential_id = $1 AND version_id <> $2 ORDER BY created_at ASC LIMIT 1 ) WHERE credential_id = $1", ) .bind(created.id.as_str()) .bind(version_id.as_str()) .execute(&pool) .await .expect("point active"); sqlx::query("DELETE FROM source_credential_versions WHERE version_id = $1") .bind(version_id.as_str()) .execute(&pool) .await .expect("drop version"); let intent_id: String = sqlx::query_scalar( "SELECT intent_id FROM secret_write_intents WHERE version_id = $1 ORDER BY created_at DESC LIMIT 1", ) .bind(version_id.as_str()) .fetch_one(&pool) .await .expect("intent"); // Reconciler ownership claim (abandoning) without destroy yet. sqlx::query( "UPDATE secret_write_intents SET status = 'abandoning', vault_kv_version = $2, updated_at = now() WHERE intent_id = $1", ) .bind(&intent_id) .bind(vault_kv) .execute(&pool) .await .expect("claim abandoning"); // Simulate late materialize attempt: only vault_written may become durable. let claimed: i64 = sqlx::query_scalar( "UPDATE secret_write_intents SET status = 'durable_committed', updated_at = now() WHERE intent_id = $1 AND status = 'vault_written' RETURNING 1", ) .bind(&intent_id) .fetch_optional(&pool) .await .expect("claim attempt") .unwrap_or(0); assert_eq!( claimed, 0, "materialize must not steal an abandoning intent" ); // Finish cleanup; abandoning is eligible without age gate. let handled = registry .reconcile_secret_write_intents() .await .expect("reconcile claimed"); assert!(handled >= 1); let status: String = sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1") .bind(&intent_id) .fetch_one(&pool) .await .expect("status"); assert_eq!(status, "abandoned"); let vault = vault_client().expect("vault"); assert!( vault .get_version(&secret_mount, &secret_path, vault_kv as u64) .await .is_err() ); } #[tokio::test] async fn prepared_intent_fence_blocks_a_late_exact_path_write() { let Some((pool, registry, _)) = harness().await else { return; }; let vault = vault_client().expect("vault"); let owner = owner("disc1"); let created = registry .create( &owner, CreateCredential { label: "discover-safe".into(), material: token_material("secret-discover-base"), }, ) .await .expect("create"); let active_version = created.active_version.expect("active version"); let (active_mount, active_path, active_kv): (String, String, i64) = sqlx::query_as( "SELECT secret_mount, secret_path, vault_kv_version FROM source_credential_versions WHERE version_id = $1", ) .bind(active_version.id.as_str()) .fetch_one(&pool) .await .expect("active coordinate"); let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos(); let aged_intent = format!("iaged{nanos:028x}"); let aged_version = OpaqueId::parse(format!("vaged{nanos:028x}")).expect("version id"); let late_path = vault.config().secret_path_for(&created.id, &aged_version); sqlx::query( "INSERT INTO secret_write_intents ( intent_id, operation, credential_id, version_id, owner_id, label, credential_kind, secret_mount, secret_path, vault_kv_version, status, created_at, updated_at ) VALUES ($1, 'rotate', $2, $3, NULL, NULL, NULL, $4, $5, NULL, 'prepared', now() - interval '2 hours', now() - interval '2 hours')", ) .bind(&aged_intent) .bind(created.id.as_str()) .bind(aged_version.as_str()) .bind(&active_mount) .bind(&late_path) .execute(&pool) .await .expect("insert aged prepared"); let handled = registry .reconcile_secret_write_intents() .await .expect("reconcile empty prepared"); assert!(handled >= 1); let abandoned_status: String = sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1") .bind(&aged_intent) .fetch_one(&pool) .await .expect("abandoned status"); assert_eq!(abandoned_status, "abandoned"); assert!( vault .put_version( &active_mount, &late_path, &token_material("secret-discover-late"), ) .await .is_err(), "the CAS fence must exclude a delayed writer" ); assert!( vault .list_live_versions(&active_mount, &late_path) .await .expect("fenced metadata") .is_empty() ); vault .get_version(&active_mount, &active_path, active_kv as u64) .await .expect("active version remains intact"); } #[tokio::test] async fn migration_is_idempotent() { let Some(pool) = test_pool().await else { return; }; migrate_schema(&pool).await.expect("second"); migrate_schema(&pool).await.expect("third"); } #[tokio::test] async fn version_coordinate_migration_backfills_populated_schema() { let Some(pool) = test_pool().await else { return; }; let schema = format!( "credentials_upgrade_{:x}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() ); let mut connection = pool.acquire().await.expect("migration connection"); sqlx::query(sqlx::AssertSqlSafe(format!("CREATE SCHEMA \"{schema}\""))) .execute(&mut *connection) .await .expect("create migration schema"); sqlx::query(sqlx::AssertSqlSafe(format!( "SET search_path TO \"{schema}\"" ))) .execute(&mut *connection) .await .expect("select migration schema"); sqlx::raw_sql(include_str!( "../../remora-schema/migrations/credentials/0001_credentials.sql" )) .execute(&mut *connection) .await .expect("pre-coordinate schema"); sqlx::query( "INSERT INTO source_credentials ( credential_id, owner_id, label, kind, status, active_version_id, secret_mount, secret_path, created_at, updated_at ) VALUES ( 'cupgrade00000000000000000001', 'oupgrade00000000000000000001', 'upgrade', 'bearer_token', 'revoked', NULL, 'fixture-mount', 'fixture/legacy/path', now(), now() )", ) .execute(&mut *connection) .await .expect("legacy credential"); sqlx::query( "INSERT INTO source_credential_versions ( version_id, credential_id, vault_kv_version, created_at, destroyed_at ) VALUES ( 'vupgrade00000000000000000001', 'cupgrade00000000000000000001', 7, now(), NULL )", ) .execute(&mut *connection) .await .expect("legacy version"); sqlx::raw_sql(include_str!( "../../remora-schema/migrations/credentials/0005_version_secret_coordinates.sql" )) .execute(&mut *connection) .await .expect("coordinate migration"); let coordinates: (String, String) = sqlx::query_as( "SELECT secret_mount, secret_path FROM source_credential_versions WHERE version_id = 'vupgrade00000000000000000001'", ) .fetch_one(&mut *connection) .await .expect("backfilled coordinates"); assert_eq!( coordinates, ("fixture-mount".into(), "fixture/legacy/path".into()) ); sqlx::query("SET search_path TO public") .execute(&mut *connection) .await .expect("restore search path"); sqlx::query(sqlx::AssertSqlSafe(format!( "DROP SCHEMA \"{schema}\" CASCADE" ))) .execute(&mut *connection) .await .expect("drop migration schema"); } #[tokio::test] async fn transcript_does_not_contain_secret_literals() { let Some((pool, registry, resolver)) = harness().await else { return; }; let marker = "unique-secret-marker-9f3c2a1b"; let owner = owner("redact"); let created = registry .create( &owner, CreateCredential { label: "redaction".into(), material: token_material(marker), }, ) .await .expect("create"); let pin = registry .pin_current_version(&caller(&owner), &created.id) .await .expect("pin"); let access = bind_grant(&pool, ®istry, "redact", &pin).await; let secret = resolver.resolve_access(&access).await.expect("resolve"); let transcript = format!("{created:?}{secret:?}"); assert!(!transcript.contains(marker)); }