remora/crates/remora-credentials/tests/substrate.rs

use std::io;
use std::sync::Arc;
use std::time::Duration;

use remora_credentials::{
    ActiveCredentialGrant, CreateCredential, CredentialCaller, CredentialGrantVersion,
    CredentialRegistry, GrantBoundResolver, SecretMaterial, VaultClient, VaultConfig,
};
use remora_domain::OpaqueId;
use remora_domain::ports::CredentialResolver;
use remora_durable::{PoolConfig, connect};
use remora_schema::migrate as migrate_schema;
use sqlx::PgPool;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{Barrier, oneshot};

fn database_url() -> Option<String> {
    std::env::var("DATABASE_URL")
        .ok()
        .filter(|value| !value.is_empty())
}

fn require_env() -> bool {
    std::env::var("REMORA_CREDENTIALS_REQUIRE").is_ok()
}

fn vault_ready() -> bool {
    std::env::var("REMORA_VAULT_ADDR")
        .ok()
        .filter(|v| !v.is_empty())
        .is_some()
        && std::env::var("REMORA_VAULT_TOKEN")
            .ok()
            .filter(|v| !v.is_empty())
            .is_some()
        && std::env::var("REMORA_VAULT_PATH")
            .ok()
            .filter(|v| !v.is_empty())
            .is_some()
}

async fn test_pool() -> Option<PgPool> {
    let Some(url) = database_url() else {
        if require_env() {
            panic!("DATABASE_URL is required when REMORA_CREDENTIALS_REQUIRE is set");
        }
        eprintln!("skipping credentials durable test: DATABASE_URL is unset");
        return None;
    };
    let pool = connect(
        &url,
        PoolConfig {
            max_connections: 8,
            acquire_timeout: Duration::from_secs(10),
            idle_timeout: Some(Duration::from_secs(30)),
        },
    )
    .await
    .expect("connect postgres");
    migrate_schema(&pool).await.expect("schema migrate");
    Some(pool)
}

fn vault_client() -> Option<VaultClient> {
    if !vault_ready() {
        if require_env() {
            panic!("vault env is required when REMORA_CREDENTIALS_REQUIRE is set");
        }
        eprintln!("skipping credentials vault test: vault env is unset");
        return None;
    }
    let config = VaultConfig::from_env().expect("vault config");
    Some(VaultClient::new(config).expect("vault client"))
}

fn owner(label: &str) -> OpaqueId {
    OpaqueId::parse(format!("owner-{label}-aaaaaaaa")).expect("owner id")
}

fn grant_id(label: &str) -> OpaqueId {
    OpaqueId::parse(format!("grant-{label}-aaaaaaaa")).expect("grant id")
}

fn caller(owner: &OpaqueId) -> CredentialCaller {
    CredentialCaller {
        principal_id: owner.clone(),
        administrator: false,
    }
}

fn admin() -> CredentialCaller {
    CredentialCaller {
        principal_id: OpaqueId::parse("admin-principal-aaaaaa").unwrap(),
        administrator: true,
    }
}

fn token_material(value: &str) -> SecretMaterial {
    SecretMaterial::BearerToken {
        token: value.to_owned(),
    }
}

async fn bind_grant(
    pool: &PgPool,
    registry: &CredentialRegistry,
    label: &str,
    pin: &CredentialGrantVersion,
) -> ActiveCredentialGrant {
    let grant_id = grant_id(label);
    let job_id = format!("job-{label}-aaaaaaaa");
    let sensitive_source_ref = format!("source-{label}-aaaaaaaa");
    let mut tx = pool.begin().await.expect("begin authoritative grant");
    sqlx::query(
        "INSERT INTO sensitive_source_inputs (sensitive_source_ref, source_input, created_at)
         VALUES ($1, '{}'::jsonb, now())",
    )
    .bind(&sensitive_source_ref)
    .execute(&mut *tx)
    .await
    .expect("insert sensitive source");
    sqlx::query(
        "INSERT INTO execution_grants (
           grant_id, owner_id, target_store, target_bucket, target_prefix,
           policy_version, credential_id, credential_version_id, status, created_at
         )
         VALUES ($1, 'owner-fixture-aaaaaaaa', 'fixture', 'fixture', '[]'::jsonb,
                 'fixture', $2, $3, 'active', now())",
    )
    .bind(grant_id.as_str())
    .bind(pin.credential_id.as_str())
    .bind(pin.version_id.as_str())
    .execute(&mut *tx)
    .await
    .expect("insert authoritative grant");
    sqlx::query(
        "INSERT INTO jobs (
           job_id, owner_id, phase, lifecycle, source_projection, sensitive_source_ref,
           target_store, target_bucket, target_prefix, execution_grant_id,
           dispatch_needed, created_at, updated_at
         )
         VALUES ($1, 'owner-fixture-aaaaaaaa', 'queued', 'visible', '{}'::jsonb, $2,
                 'fixture', 'fixture', '[]'::jsonb, $3, false, now(), now())",
    )
    .bind(job_id)
    .bind(sensitive_source_ref)
    .bind(grant_id.as_str())
    .execute(&mut *tx)
    .await
    .expect("insert authoritative job");
    tx.commit().await.expect("commit authoritative grant");
    registry
        .bind_execution_grant(&grant_id, pin)
        .await
        .expect("bind grant")
}

async fn harness() -> Option<(PgPool, CredentialRegistry, GrantBoundResolver)> {
    let pool = test_pool().await?;
    let vault = vault_client()?;
    let registry = CredentialRegistry::new(pool.clone(), vault.clone());
    let resolver = GrantBoundResolver::new(registry.clone(), vault);
    Some((pool, registry, resolver))
}

async fn held_vault_response_proxy(
    upstream: &str,
) -> (String, oneshot::Receiver<()>, oneshot::Sender<()>) {
    let authority = upstream
        .strip_prefix("http://")
        .expect("fixture Vault must use HTTP")
        .trim_end_matches('/')
        .to_owned();
    let listener = TcpListener::bind("127.0.0.1:0").await.expect("proxy bind");
    let address = listener.local_addr().expect("proxy address");
    let (applied_tx, applied_rx) = oneshot::channel();
    let (release_tx, release_rx) = oneshot::channel();
    tokio::spawn(async move {
        let (mut client, _) = listener.accept().await.expect("proxy accept");
        let request = read_http_message(&mut client).await.expect("proxy request");
        let mut vault = TcpStream::connect(&authority)
            .await
            .expect("connect fixture Vault");
        vault.write_all(&request).await.expect("forward request");
        let response = read_http_message(&mut vault).await.expect("Vault response");
        let _ = applied_tx.send(());
        let _ = release_rx.await;
        client.write_all(&response).await.expect("release response");
    });
    (format!("http://{address}"), applied_rx, release_tx)
}

async fn read_http_message(stream: &mut TcpStream) -> io::Result<Vec<u8>> {
    let mut message = Vec::with_capacity(8192);
    let mut chunk = [0u8; 4096];
    let mut expected = None;
    loop {
        let read = stream.read(&mut chunk).await?;
        if read == 0 {
            return Ok(message);
        }
        message.extend_from_slice(&chunk[..read]);
        if expected.is_none()
            && let Some(header_end) = message.windows(4).position(|window| window == b"\r\n\r\n")
        {
            let headers = std::str::from_utf8(&message[..header_end])
                .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
            let content_length = headers
                .lines()
                .find_map(|line| {
                    let (name, value) = line.split_once(':')?;
                    name.eq_ignore_ascii_case("content-length")
                        .then(|| value.trim().parse::<usize>().ok())
                        .flatten()
                })
                .unwrap_or(0);
            expected = Some(header_end + 4 + content_length);
        }
        if expected.is_some_and(|length| message.len() >= length) {
            return Ok(message);
        }
    }
}

#[tokio::test]
async fn create_pin_and_exact_version_resolution() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("exact1");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "exact-version".into(),
                material: token_material("secret-version-one-value"),
            },
        )
        .await
        .expect("create");
    let pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("pin");
    let access = bind_grant(&pool, &registry, "exact1", &pin).await;

    let secret = resolver
        .resolve_access(&access)
        .await
        .expect("resolve pinned");
    assert_eq!(secret, token_material("secret-version-one-value"));
}

#[tokio::test]
async fn vault_create_only_write_rejects_an_occupied_intent_path() {
    let Some(vault) = vault_client() else {
        return;
    };
    let credential_id = OpaqueId::parse(format!(
        "ccas{:028x}",
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_nanos()
    ))
    .expect("credential id");
    let version_id = OpaqueId::parse(format!(
        "vcas{:028x}",
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_nanos()
    ))
    .expect("version id");
    let mount = vault.config().mount.clone();
    let path = vault.config().secret_path_for(&credential_id, &version_id);
    let first = token_material("secret-cas-first");
    let version = vault
        .put_version(&mount, &path, &first)
        .await
        .expect("first CAS write");
    assert_eq!(version, 1);
    assert!(
        vault
            .put_version(&mount, &path, &token_material("secret-cas-second"))
            .await
            .is_err()
    );
    assert_eq!(
        vault
            .get_version(&mount, &path, version)
            .await
            .expect("first version remains"),
        first
    );
    vault
        .destroy_version(&mount, &path, version)
        .await
        .expect("cleanup CAS fixture");
}

#[tokio::test]
async fn rotation_does_not_change_prior_pin_and_rejects_substitution() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("rotate");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "rotate-me".into(),
                material: token_material("secret-version-one-value"),
            },
        )
        .await
        .expect("create");
    let first_pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("first pin");
    let first_access = bind_grant(&pool, &registry, "rotate-a", &first_pin).await;

    registry
        .rotate(
            &caller(&owner),
            &created.id,
            token_material("secret-version-two-value"),
        )
        .await
        .expect("rotate");
    let second_pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("second pin");
    let second_access = bind_grant(&pool, &registry, "rotate-b", &second_pin).await;

    assert_eq!(
        resolver.resolve_access(&first_access).await.expect("first"),
        token_material("secret-version-one-value")
    );
    assert_eq!(
        resolver
            .resolve_access(&second_access)
            .await
            .expect("second"),
        token_material("secret-version-two-value")
    );

    // Forged grant id (never bound) fails closed.
    let forged = ActiveCredentialGrant::for_execution_grant(grant_id("forged0"));
    assert!(matches!(
        resolver.resolve_access(&forged).await,
        Err(remora_credentials::CredentialError::GrantInactive)
    ));
}

#[tokio::test]
async fn unbound_grant_handle_cannot_resolve() {
    let Some((_, _, resolver)) = harness().await else {
        return;
    };
    // Constructing a handle does not authorize resolution without durable bind.
    let access = ActiveCredentialGrant::for_execution_grant(grant_id("unbound"));
    assert!(matches!(
        resolver.resolve_access(&access).await,
        Err(remora_credentials::CredentialError::GrantInactive)
    ));
}

#[tokio::test]
async fn execution_grant_revocation_fails_closed_on_retained_handle() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("grev01");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "grant-revoke".into(),
                material: token_material("secret-grant-revoke"),
            },
        )
        .await
        .expect("create");
    let pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("pin");
    let gid = grant_id("grev01");
    let access = bind_grant(&pool, &registry, "grev01", &pin).await;

    resolver
        .resolve_access(&access)
        .await
        .expect("resolve while active");

    let mut tx = pool.begin().await.expect("begin revocation");
    sqlx::query(
        "UPDATE execution_grants
         SET status = 'revoked', revoked_at = now()
         WHERE grant_id = $1",
    )
    .bind(gid.as_str())
    .execute(&mut *tx)
    .await
    .expect("revoke authoritative grant");
    sqlx::query(
        "UPDATE execution_grant_credential_pins
         SET status = 'revoked', revoked_at = now()
         WHERE grant_id = $1",
    )
    .bind(gid.as_str())
    .execute(&mut *tx)
    .await
    .expect("revoke grant pin");
    tx.commit().await.expect("commit revocation");

    // Same retained handle must fail after durable revocation.
    assert!(matches!(
        resolver.resolve_access(&access).await,
        Err(remora_credentials::CredentialError::GrantInactive)
    ));
}

#[tokio::test]
async fn credential_revocation_blocks_new_pins() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("revoke");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "revocable".into(),
                material: token_material("secret-revocable-value"),
            },
        )
        .await
        .expect("create");
    let pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("pin before revoke");
    let access = bind_grant(&pool, &registry, "crevoke", &pin).await;

    registry
        .revoke(&caller(&owner), &created.id)
        .await
        .expect("revoke");

    // Prior execution grant still active until grant revoke.
    assert_eq!(
        resolver.resolve_access(&access).await.expect("prior grant"),
        token_material("secret-revocable-value")
    );

    let new_pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await;
    assert!(matches!(
        new_pin,
        Err(remora_credentials::CredentialError::Revoked)
    ));
}

#[tokio::test]
async fn pin_and_revoke_serialize_under_lock() {
    let Some((_, registry, _)) = harness().await else {
        return;
    };
    let owner = owner("race01");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "race".into(),
                material: token_material("secret-race-value"),
            },
        )
        .await
        .expect("create");

    let registry = Arc::new(registry);
    let barrier = Arc::new(Barrier::new(2));
    let pin_registry = Arc::clone(&registry);
    let rev_registry = Arc::clone(&registry);
    let pin_barrier = Arc::clone(&barrier);
    let rev_barrier = Arc::clone(&barrier);
    let pin_owner = owner.clone();
    let rev_owner = owner.clone();
    let credential_id = created.id.clone();
    let credential_id_rev = created.id.clone();

    let pin_handle = tokio::spawn(async move {
        pin_barrier.wait().await;
        pin_registry
            .pin_current_version(&caller(&pin_owner), &credential_id)
            .await
    });
    let rev_handle = tokio::spawn(async move {
        rev_barrier.wait().await;
        rev_registry
            .revoke(&caller(&rev_owner), &credential_id_rev)
            .await
    });

    let pin_result = pin_handle.await.expect("join pin");
    rev_handle.await.expect("join rev").expect("revoke");

    match pin_result {
        Ok(_) => {
            let after = registry
                .pin_current_version(&caller(&owner), &created.id)
                .await;
            assert!(matches!(
                after,
                Err(remora_credentials::CredentialError::Revoked)
                    | Err(remora_credentials::CredentialError::NoActiveVersion)
            ));
        }
        Err(remora_credentials::CredentialError::Revoked)
        | Err(remora_credentials::CredentialError::NoActiveVersion) => {}
        Err(other) => panic!("unexpected pin error: {other:?}"),
    }
}

#[tokio::test]
async fn destroyed_version_fails_closed() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("destroy");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "destroy-me".into(),
                material: token_material("secret-destroy-value"),
            },
        )
        .await
        .expect("create");
    let pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("pin");
    let access = bind_grant(&pool, &registry, "destroy", &pin).await;

    registry
        .destroy_version(&caller(&owner), &pin.credential_id, &pin.version_id)
        .await
        .expect("destroy");

    assert!(matches!(
        resolver.resolve_access(&access).await,
        Err(remora_credentials::CredentialError::Unresolvable)
    ));
    assert!(matches!(
        registry
            .pin_current_version(&caller(&owner), &created.id)
            .await,
        Err(remora_credentials::CredentialError::NoActiveVersion)
    ));
}

#[tokio::test]
async fn pin_after_destroying_prior_version_uses_new_active() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("pindest");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "pin-destroy".into(),
                material: token_material("secret-pin-destroy-one"),
            },
        )
        .await
        .expect("create");
    let first_pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("first pin");
    let first_access = bind_grant(&pool, &registry, "pindest-a", &first_pin).await;

    registry
        .rotate(
            &caller(&owner),
            &created.id,
            token_material("secret-pin-destroy-two"),
        )
        .await
        .expect("rotate");
    let second_pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("second pin");
    let second_access = bind_grant(&pool, &registry, "pindest-b", &second_pin).await;

    registry
        .destroy_version(
            &caller(&owner),
            &first_pin.credential_id,
            &first_pin.version_id,
        )
        .await
        .expect("destroy prior");

    assert!(matches!(
        resolver.resolve_access(&first_access).await,
        Err(remora_credentials::CredentialError::Unresolvable)
    ));
    assert_eq!(
        resolver
            .resolve_access(&second_access)
            .await
            .expect("current"),
        token_material("secret-pin-destroy-two")
    );
}

#[tokio::test]
async fn secret_store_outage_on_resolve_fails_closed() {
    let Some((pool, good_registry, _)) = harness().await else {
        return;
    };
    let owner = owner("outage2");
    let created = good_registry
        .create(
            &owner,
            CreateCredential {
                label: "outage2".into(),
                material: token_material("secret-outage-two-value"),
            },
        )
        .await
        .expect("create");
    let pin = good_registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("pin");
    let access = bind_grant(&pool, &good_registry, "outage2", &pin).await;

    let bad_vault = VaultClient::new(VaultConfig {
        address: "http://127.0.0.1:1".into(),
        token: "unused".into(),
        mount: "secret".into(),
        path_prefix: "remora/outage".into(),
        timeout: Duration::from_millis(250),
    })
    .expect("bad client");
    let resolver = GrantBoundResolver::new(
        CredentialRegistry::new(
            pool,
            vault_client().expect("good vault for registry metadata"),
        ),
        bad_vault,
    );
    assert!(matches!(
        resolver.resolve_access(&access).await,
        Err(remora_credentials::CredentialError::SecretStoreUnavailable)
    ));
}

#[tokio::test]
async fn authorization_hides_peer_credentials() {
    let Some((_, registry, _)) = harness().await else {
        return;
    };
    let owner_id = owner("authz1");
    let peer_id = owner("authz2");
    let created = registry
        .create(
            &owner_id,
            CreateCredential {
                label: "private".into(),
                material: token_material("secret-private-value"),
            },
        )
        .await
        .expect("create");

    assert!(
        registry
            .get(&caller(&peer_id), &created.id)
            .await
            .expect("get")
            .is_none()
    );
    assert!(matches!(
        registry
            .pin_current_version(&caller(&peer_id), &created.id)
            .await,
        Err(remora_credentials::CredentialError::NotFound)
    ));
    assert_eq!(
        registry
            .get(&admin(), &created.id)
            .await
            .expect("admin")
            .expect("visible")
            .id,
        created.id
    );
}

#[tokio::test]
async fn concurrent_rotates_all_succeed_and_prior_pin_holds() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("conc01");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "concurrent".into(),
                material: token_material("secret-concurrent-base"),
            },
        )
        .await
        .expect("create");
    let first_pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("first pin");
    let first_access = bind_grant(&pool, &registry, "conc01", &first_pin).await;

    let registry = Arc::new(registry);
    let barrier = Arc::new(Barrier::new(4));
    let mut handles = Vec::new();
    for index in 0..4 {
        let registry = Arc::clone(&registry);
        let barrier = Arc::clone(&barrier);
        let owner = owner.clone();
        let credential_id = created.id.clone();
        handles.push(tokio::spawn(async move {
            barrier.wait().await;
            registry
                .rotate(
                    &caller(&owner),
                    &credential_id,
                    token_material(&format!("secret-concurrent-rot-{index}")),
                )
                .await
        }));
    }
    for handle in handles {
        handle.await.expect("join").expect("rotate");
    }

    let version_count: i64 = sqlx::query_scalar(
        "SELECT COUNT(*) FROM source_credential_versions WHERE credential_id = $1",
    )
    .bind(created.id.as_str())
    .fetch_one(&pool)
    .await
    .expect("count");
    assert_eq!(version_count, 5);
    let distinct_paths: i64 = sqlx::query_scalar(
        "SELECT COUNT(DISTINCT secret_path) FROM source_credential_versions
         WHERE credential_id = $1 AND vault_kv_version = 1",
    )
    .bind(created.id.as_str())
    .fetch_one(&pool)
    .await
    .expect("distinct paths");
    assert_eq!(distinct_paths, 5);

    assert_eq!(
        resolver
            .resolve_access(&first_access)
            .await
            .expect("original pin"),
        token_material("secret-concurrent-base")
    );
}

#[tokio::test]
async fn domain_port_resolve_requires_durable_grant() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let owner = owner("port01");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "port".into(),
                material: token_material("secret-port-value"),
            },
        )
        .await
        .expect("create");
    let pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("pin");
    let access = bind_grant(&pool, &registry, "port01", &pin).await;
    let via_port = CredentialResolver::resolve(&resolver, &access)
        .await
        .expect("port resolve");
    assert_eq!(via_port, token_material("secret-port-value"));
}

#[tokio::test]
async fn reconcile_skips_fresh_vault_written_intents() {
    let Some((pool, registry, _)) = harness().await else {
        return;
    };
    // Age-gate: a vault_written row younger than min age must not be abandoned.
    let nanos = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_nanos();
    let intent_id = format!("ifresh{nanos:028x}");
    let credential_id = "creconyoung00000000000000001";
    let version_id = "vreconyoung00000000000000001";
    sqlx::query(
        "INSERT INTO secret_write_intents (
            intent_id, operation, credential_id, version_id, owner_id, label,
            credential_kind, secret_mount, secret_path, vault_kv_version, status,
            created_at, updated_at
         ) VALUES ($1, 'rotate', $2, $3, NULL, NULL, NULL, 'secret', 'orphan/path', 9,
                   'vault_written', now(), now())",
    )
    .bind(&intent_id)
    .bind(credential_id)
    .bind(version_id)
    .execute(&pool)
    .await
    .expect("insert fresh intent");

    let _ = registry
        .reconcile_secret_write_intents()
        .await
        .expect("reconcile");

    let status: String =
        sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1")
            .bind(&intent_id)
            .fetch_one(&pool)
            .await
            .expect("status");
    assert_eq!(
        status, "vault_written",
        "fresh vault_written intents must not be cleaned by reconcile"
    );
}

#[tokio::test]
async fn reconcile_destroys_aged_orphan_rotate_under_lock() {
    let Some((pool, registry, _)) = harness().await else {
        return;
    };
    let owner = owner("recon2");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "recon-rotate".into(),
                material: token_material("secret-recon-base"),
            },
        )
        .await
        .expect("create");

    // Perform a rotate, then manually roll the intent back to vault_written without
    // the version row to simulate stuck materialize after age boundary.
    let rotated = registry
        .rotate(
            &caller(&owner),
            &created.id,
            token_material("secret-recon-rot"),
        )
        .await
        .expect("rotate");
    let version_id = rotated.active_version.expect("active").id;

    // Delete durable version + reset intent to vault_written aged (simulates lost commit
    // recovery candidate). Keep vault secret.
    let (secret_mount, secret_path, vault_kv): (String, String, i64) = sqlx::query_as(
        "SELECT v.secret_mount, v.secret_path, v.vault_kv_version
         FROM source_credential_versions v
         WHERE v.version_id = $1",
    )
    .bind(version_id.as_str())
    .fetch_one(&pool)
    .await
    .expect("coords");

    sqlx::query(
        "UPDATE source_credentials SET active_version_id = (
        SELECT version_id FROM source_credential_versions
        WHERE credential_id = $1 AND version_id <> $2
        ORDER BY created_at ASC LIMIT 1
    ) WHERE credential_id = $1",
    )
    .bind(created.id.as_str())
    .bind(version_id.as_str())
    .execute(&pool)
    .await
    .expect("point active at first");

    sqlx::query("DELETE FROM source_credential_versions WHERE version_id = $1")
        .bind(version_id.as_str())
        .execute(&pool)
        .await
        .expect("drop version row");

    let intent_id: String = sqlx::query_scalar(
        "SELECT intent_id FROM secret_write_intents
         WHERE version_id = $1 ORDER BY created_at DESC LIMIT 1",
    )
    .bind(version_id.as_str())
    .fetch_one(&pool)
    .await
    .expect("intent");

    sqlx::query(
        "UPDATE secret_write_intents
         SET status = 'vault_written', vault_kv_version = $2,
             updated_at = now() - interval '2 hours'
         WHERE intent_id = $1",
    )
    .bind(&intent_id)
    .bind(vault_kv)
    .execute(&pool)
    .await
    .expect("age orphan intent");

    let handled = registry
        .reconcile_secret_write_intents()
        .await
        .expect("reconcile orphan");
    assert!(handled >= 1, "expected orphan cleanup");

    let status: String =
        sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1")
            .bind(&intent_id)
            .fetch_one(&pool)
            .await
            .expect("status");
    assert_eq!(status, "abandoned");

    // Vault version destroyed: read fails closed.
    let vault = vault_client().expect("vault");
    assert!(
        vault
            .get_version(&secret_mount, &secret_path, vault_kv as u64)
            .await
            .is_err()
    );
}

#[tokio::test]
async fn reconcile_claim_blocks_concurrent_materialize() {
    let Some((pool, registry, _)) = harness().await else {
        return;
    };
    let owner = owner("claim1");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "claim-rotate".into(),
                material: token_material("secret-claim-base"),
            },
        )
        .await
        .expect("create");
    let rotated = registry
        .rotate(
            &caller(&owner),
            &created.id,
            token_material("secret-claim-rot"),
        )
        .await
        .expect("rotate");
    let version_id = rotated.active_version.expect("active").id;
    let (secret_mount, secret_path, vault_kv): (String, String, i64) = sqlx::query_as(
        "SELECT v.secret_mount, v.secret_path, v.vault_kv_version
         FROM source_credential_versions v
         WHERE v.version_id = $1",
    )
    .bind(version_id.as_str())
    .fetch_one(&pool)
    .await
    .expect("coords");

    sqlx::query(
        "UPDATE source_credentials SET active_version_id = (
        SELECT version_id FROM source_credential_versions
        WHERE credential_id = $1 AND version_id <> $2
        ORDER BY created_at ASC LIMIT 1
    ) WHERE credential_id = $1",
    )
    .bind(created.id.as_str())
    .bind(version_id.as_str())
    .execute(&pool)
    .await
    .expect("point active");

    sqlx::query("DELETE FROM source_credential_versions WHERE version_id = $1")
        .bind(version_id.as_str())
        .execute(&pool)
        .await
        .expect("drop version");

    let intent_id: String = sqlx::query_scalar(
        "SELECT intent_id FROM secret_write_intents
         WHERE version_id = $1 ORDER BY created_at DESC LIMIT 1",
    )
    .bind(version_id.as_str())
    .fetch_one(&pool)
    .await
    .expect("intent");

    // Reconciler ownership claim (abandoning) without destroy yet.
    sqlx::query(
        "UPDATE secret_write_intents
         SET status = 'abandoning', vault_kv_version = $2, updated_at = now()
         WHERE intent_id = $1",
    )
    .bind(&intent_id)
    .bind(vault_kv)
    .execute(&pool)
    .await
    .expect("claim abandoning");

    // Simulate late materialize attempt: only vault_written may become durable.
    let claimed: i64 = sqlx::query_scalar(
        "UPDATE secret_write_intents
         SET status = 'durable_committed', updated_at = now()
         WHERE intent_id = $1 AND status = 'vault_written'
         RETURNING 1",
    )
    .bind(&intent_id)
    .fetch_optional(&pool)
    .await
    .expect("claim attempt")
    .unwrap_or(0);
    assert_eq!(
        claimed, 0,
        "materialize must not steal an abandoning intent"
    );

    // Finish cleanup; abandoning is eligible without age gate.
    let handled = registry
        .reconcile_secret_write_intents()
        .await
        .expect("reconcile claimed");
    assert!(handled >= 1);

    let status: String =
        sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1")
            .bind(&intent_id)
            .fetch_one(&pool)
            .await
            .expect("status");
    assert_eq!(status, "abandoned");

    let vault = vault_client().expect("vault");
    assert!(
        vault
            .get_version(&secret_mount, &secret_path, vault_kv as u64)
            .await
            .is_err()
    );
}

#[tokio::test]
async fn concurrent_reconcilers_claim_before_a_late_create_response() {
    let Some(pool) = test_pool().await else {
        return;
    };
    let Some(direct_vault) = vault_client() else {
        return;
    };
    let upstream = direct_vault.config().address.clone();
    let (proxy_address, applied, release) = held_vault_response_proxy(&upstream).await;
    let mut delayed_config = VaultConfig::from_env().expect("Vault config");
    delayed_config.address = proxy_address;
    delayed_config.timeout = Duration::from_secs(5);
    let delayed_registry = CredentialRegistry::new(
        pool.clone(),
        VaultClient::new(delayed_config).expect("delayed Vault client"),
    )
    .with_reconcile_min_age(Duration::ZERO);
    let reconciler = Arc::new(
        CredentialRegistry::new(pool.clone(), direct_vault.clone())
            .with_reconcile_min_age(Duration::ZERO),
    );
    let owner = owner("late-response");
    let create_owner = owner.clone();
    let create_task = tokio::spawn(async move {
        delayed_registry
            .create(
                &create_owner,
                CreateCredential {
                    label: "late-response".into(),
                    material: token_material("secret-late-response"),
                },
            )
            .await
    });

    tokio::time::timeout(Duration::from_secs(5), applied)
        .await
        .expect("Vault apply timeout")
        .expect("Vault apply signal");
    let (intent_id, secret_mount, secret_path): (String, String, String) = sqlx::query_as(
        "SELECT intent_id, secret_mount, secret_path
         FROM secret_write_intents
         WHERE operation = 'create' AND owner_id = $1
         ORDER BY created_at DESC LIMIT 1",
    )
    .bind(owner.as_str())
    .fetch_one(&pool)
    .await
    .expect("prepared create intent");

    let barrier = Arc::new(Barrier::new(4));
    let mut reconcile_tasks = Vec::new();
    for _ in 0..4 {
        let reconciler = Arc::clone(&reconciler);
        let barrier = Arc::clone(&barrier);
        reconcile_tasks.push(tokio::spawn(async move {
            barrier.wait().await;
            reconciler.reconcile_secret_write_intents().await
        }));
    }
    for task in reconcile_tasks {
        task.await.expect("reconciler join").expect("reconcile");
    }

    let status: String =
        sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1")
            .bind(&intent_id)
            .fetch_one(&pool)
            .await
            .expect("intent status");
    assert_eq!(status, "abandoned");

    release.send(()).expect("release Vault response");
    assert!(
        tokio::time::timeout(Duration::from_secs(5), create_task)
            .await
            .expect("create completion timeout")
            .expect("create join")
            .is_err()
    );
    let credential_count: i64 =
        sqlx::query_scalar("SELECT COUNT(*) FROM source_credentials WHERE owner_id = $1")
            .bind(owner.as_str())
            .fetch_one(&pool)
            .await
            .expect("credential count");
    assert_eq!(credential_count, 0);
    assert!(
        direct_vault
            .list_live_versions(&secret_mount, &secret_path)
            .await
            .expect("final Vault metadata")
            .is_empty()
    );
}

#[tokio::test]
async fn prepared_intent_fence_blocks_a_late_exact_path_write() {
    let Some((pool, registry, _)) = harness().await else {
        return;
    };
    let vault = vault_client().expect("vault");
    let owner = owner("disc1");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "discover-safe".into(),
                material: token_material("secret-discover-base"),
            },
        )
        .await
        .expect("create");
    let active_version = created.active_version.expect("active version");
    let (active_mount, active_path, active_kv): (String, String, i64) = sqlx::query_as(
        "SELECT secret_mount, secret_path, vault_kv_version
         FROM source_credential_versions WHERE version_id = $1",
    )
    .bind(active_version.id.as_str())
    .fetch_one(&pool)
    .await
    .expect("active coordinate");

    let nanos = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_nanos();
    let aged_intent = format!("iaged{nanos:028x}");
    let aged_version = OpaqueId::parse(format!("vaged{nanos:028x}")).expect("version id");
    let late_path = vault.config().secret_path_for(&created.id, &aged_version);
    sqlx::query(
        "INSERT INTO secret_write_intents (
            intent_id, operation, credential_id, version_id, owner_id, label,
            credential_kind, secret_mount, secret_path, vault_kv_version, status,
            created_at, updated_at
         ) VALUES ($1, 'rotate', $2, $3, NULL, NULL, NULL, $4, $5, NULL,
                   'prepared', now() - interval '2 hours', now() - interval '2 hours')",
    )
    .bind(&aged_intent)
    .bind(created.id.as_str())
    .bind(aged_version.as_str())
    .bind(&active_mount)
    .bind(&late_path)
    .execute(&pool)
    .await
    .expect("insert aged prepared");

    let handled = registry
        .reconcile_secret_write_intents()
        .await
        .expect("reconcile empty prepared");
    assert!(handled >= 1);

    let abandoned_status: String =
        sqlx::query_scalar("SELECT status FROM secret_write_intents WHERE intent_id = $1")
            .bind(&aged_intent)
            .fetch_one(&pool)
            .await
            .expect("abandoned status");
    assert_eq!(abandoned_status, "abandoned");

    assert!(
        vault
            .put_version(
                &active_mount,
                &late_path,
                &token_material("secret-discover-late"),
            )
            .await
            .is_err(),
        "the CAS fence must exclude a delayed writer"
    );
    assert!(
        vault
            .list_live_versions(&active_mount, &late_path)
            .await
            .expect("fenced metadata")
            .is_empty()
    );
    vault
        .get_version(&active_mount, &active_path, active_kv as u64)
        .await
        .expect("active version remains intact");
}

#[tokio::test]
async fn migration_is_idempotent() {
    let Some(pool) = test_pool().await else {
        return;
    };
    migrate_schema(&pool).await.expect("second");
    migrate_schema(&pool).await.expect("third");
}

#[tokio::test]
async fn version_coordinate_migration_backfills_populated_schema() {
    let Some(pool) = test_pool().await else {
        return;
    };
    let schema = format!(
        "credentials_upgrade_{:x}",
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_nanos()
    );
    let mut connection = pool.acquire().await.expect("migration connection");
    sqlx::query(sqlx::AssertSqlSafe(format!("CREATE SCHEMA \"{schema}\"")))
        .execute(&mut *connection)
        .await
        .expect("create migration schema");
    sqlx::query(sqlx::AssertSqlSafe(format!(
        "SET search_path TO \"{schema}\""
    )))
    .execute(&mut *connection)
    .await
    .expect("select migration schema");
    sqlx::raw_sql(include_str!(
        "../../remora-schema/migrations/credentials/0001_credentials.sql"
    ))
    .execute(&mut *connection)
    .await
    .expect("pre-coordinate schema");
    sqlx::query(
        "INSERT INTO source_credentials (
            credential_id, owner_id, label, kind, status, active_version_id,
            secret_mount, secret_path, created_at, updated_at
         ) VALUES (
            'cupgrade00000000000000000001', 'oupgrade00000000000000000001',
            'upgrade', 'bearer_token', 'revoked', NULL,
            'fixture-mount', 'fixture/legacy/path', now(), now()
         )",
    )
    .execute(&mut *connection)
    .await
    .expect("legacy credential");
    sqlx::query(
        "INSERT INTO source_credential_versions (
            version_id, credential_id, vault_kv_version, created_at, destroyed_at
         ) VALUES (
            'vupgrade00000000000000000001', 'cupgrade00000000000000000001',
            7, now(), NULL
         )",
    )
    .execute(&mut *connection)
    .await
    .expect("legacy version");
    sqlx::raw_sql(include_str!(
        "../../remora-schema/migrations/credentials/0005_version_secret_coordinates.sql"
    ))
    .execute(&mut *connection)
    .await
    .expect("coordinate migration");
    let coordinates: (String, String) = sqlx::query_as(
        "SELECT secret_mount, secret_path FROM source_credential_versions
         WHERE version_id = 'vupgrade00000000000000000001'",
    )
    .fetch_one(&mut *connection)
    .await
    .expect("backfilled coordinates");
    assert_eq!(
        coordinates,
        ("fixture-mount".into(), "fixture/legacy/path".into())
    );
    sqlx::query("SET search_path TO public")
        .execute(&mut *connection)
        .await
        .expect("restore search path");
    sqlx::query(sqlx::AssertSqlSafe(format!(
        "DROP SCHEMA \"{schema}\" CASCADE"
    )))
    .execute(&mut *connection)
    .await
    .expect("drop migration schema");
}

#[tokio::test]
async fn transcript_does_not_contain_secret_literals() {
    let Some((pool, registry, resolver)) = harness().await else {
        return;
    };
    let marker = "unique-secret-marker-9f3c2a1b";
    let owner = owner("redact");
    let created = registry
        .create(
            &owner,
            CreateCredential {
                label: "redaction".into(),
                material: token_material(marker),
            },
        )
        .await
        .expect("create");
    let pin = registry
        .pin_current_version(&caller(&owner), &created.id)
        .await
        .expect("pin");
    let access = bind_grant(&pool, &registry, "redact", &pin).await;
    let secret = resolver.resolve_access(&access).await.expect("resolve");
    let transcript = format!("{created:?}{secret:?}");
    assert!(!transcript.contains(marker));
}