348 lines
11 KiB
Rust
348 lines
11 KiB
Rust
|
use std::time::Duration;
|
||
|
|
|
||
|
|
use remora_admission::{
|
||
|
|
AdmissionConfig, AdmissionDecision, AdmissionError, AdmissionService, SubmitOutcome, migrate,
|
||
|
|
};
|
||
|
|
use remora_contracts::{
|
||
|
|
HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto, TargetOverrideDto,
|
||
|
|
};
|
||
|
|
use remora_credentials::{CredentialRegistry, migrate as credentials_migrate};
|
||
|
|
use remora_domain::{BucketName, OpaqueId, StorageScope, TargetPrefix, TargetStoreId};
|
||
|
|
use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
|
||
|
|
use remora_identity::{DefaultTarget, Principal};
|
||
|
|
use sqlx::PgPool;
|
||
|
|
|
||
|
|
fn database_url() -> Option<String> {
|
||
|
|
std::env::var("DATABASE_URL")
|
||
|
|
.ok()
|
||
|
|
.filter(|value| !value.is_empty())
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn pool() -> Option<PgPool> {
|
||
|
|
let Some(url) = database_url() else {
|
||
|
|
if std::env::var("REMORA_ADMISSION_REQUIRE").is_ok() {
|
||
|
|
panic!("DATABASE_URL is required when REMORA_ADMISSION_REQUIRE is set");
|
||
|
|
}
|
||
|
|
return None;
|
||
|
|
};
|
||
|
|
let pool = connect(
|
||
|
|
&url,
|
||
|
|
PoolConfig {
|
||
|
|
max_connections: 8,
|
||
|
|
acquire_timeout: Duration::from_secs(10),
|
||
|
|
idle_timeout: Some(Duration::from_secs(30)),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.expect("connect postgres");
|
||
|
|
durable_migrate(&pool).await.expect("durable migrate");
|
||
|
|
credentials_migrate(&pool)
|
||
|
|
.await
|
||
|
|
.expect("credentials migrate");
|
||
|
|
migrate(&pool).await.expect("admission migrate");
|
||
|
|
Some(pool)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn unique_id(prefix: &str) -> OpaqueId {
|
||
|
|
let nonce = std::time::SystemTime::now()
|
||
|
|
.duration_since(std::time::UNIX_EPOCH)
|
||
|
|
.unwrap()
|
||
|
|
.as_nanos();
|
||
|
|
OpaqueId::parse(format!("{prefix}{nonce:032x}")).unwrap()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn target_parts() -> (TargetStoreId, BucketName, TargetPrefix) {
|
||
|
|
(
|
||
|
|
TargetStoreId::parse("primary").unwrap(),
|
||
|
|
BucketName::parse("remora-results").unwrap(),
|
||
|
|
TargetPrefix::parse("users", ["_remora"]).unwrap(),
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
fn principal(id: OpaqueId, authorized: bool, administrator: bool) -> Principal {
|
||
|
|
let (store, bucket, prefix) = target_parts();
|
||
|
|
Principal {
|
||
|
|
id,
|
||
|
|
issuer: "https://issuer.example".into(),
|
||
|
|
subject: "subject".into(),
|
||
|
|
administrator,
|
||
|
|
scopes: authorized
|
||
|
|
.then_some(StorageScope {
|
||
|
|
store,
|
||
|
|
bucket,
|
||
|
|
prefix,
|
||
|
|
})
|
||
|
|
.into_iter()
|
||
|
|
.collect(),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn service(pool: PgPool) -> AdmissionService {
|
||
|
|
let (store, bucket, prefix) = target_parts();
|
||
|
|
AdmissionService::new(
|
||
|
|
pool,
|
||
|
|
AdmissionConfig {
|
||
|
|
default_target: DefaultTarget {
|
||
|
|
store,
|
||
|
|
bucket,
|
||
|
|
prefix,
|
||
|
|
},
|
||
|
|
reserved_target_segments: vec!["_remora".into()],
|
||
|
|
policy_version: "source-policy-v1".into(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
.unwrap()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn request(url: String) -> JobSubmissionDto {
|
||
|
|
JobSubmissionDto {
|
||
|
|
source: SourceSubmissionDto::Http(HttpSourceSubmissionDto {
|
||
|
|
url,
|
||
|
|
result_path: "artifact.bin".into(),
|
||
|
|
credential_id: None,
|
||
|
|
}),
|
||
|
|
target: Some(TargetOverrideDto {
|
||
|
|
bucket: None,
|
||
|
|
prefix: Some("users/alice".into()),
|
||
|
|
}),
|
||
|
|
label: Some("artifact".into()),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fn accepted(outcome: SubmitOutcome) -> remora_admission::AcceptedJob {
|
||
|
|
match outcome.decision {
|
||
|
|
AdmissionDecision::Accepted(job) => *job,
|
||
|
|
AdmissionDecision::Rejected { .. } => panic!("submission rejected"),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn exact_replay_and_lost_response_recovery_create_one_durable_acceptance() {
|
||
|
|
let Some(pool) = pool().await else { return };
|
||
|
|
let service = service(pool.clone());
|
||
|
|
let owner = principal(unique_id("owner"), true, false);
|
||
|
|
let key = unique_id("key").to_string();
|
||
|
|
let submission = request("https://downloads.example/archive?token=private".into());
|
||
|
|
let first = service.submit(&owner, &key, submission).await.unwrap();
|
||
|
|
assert_eq!(first.disposition, MutationDisposition::First);
|
||
|
|
let first_job = accepted(first);
|
||
|
|
|
||
|
|
let replay = service
|
||
|
|
.submit(
|
||
|
|
&owner,
|
||
|
|
&key,
|
||
|
|
request("https://downloads.example/archive?token=private".into()),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(replay.disposition, MutationDisposition::Replay);
|
||
|
|
let replay_job = accepted(replay);
|
||
|
|
assert_eq!(first_job.job_id, replay_job.job_id);
|
||
|
|
assert_eq!(first_job.execution_grant_id, replay_job.execution_grant_id);
|
||
|
|
|
||
|
|
let counts: (i64, i64, i64) = sqlx::query_as(
|
||
|
|
"SELECT
|
||
|
|
(SELECT count(*) FROM jobs WHERE job_id = $1),
|
||
|
|
(SELECT count(*) FROM execution_grants WHERE grant_id = $2),
|
||
|
|
(SELECT count(*) FROM outbox_messages WHERE topic = 'job.dispatch' AND payload->>'job_id' = $1)",
|
||
|
|
)
|
||
|
|
.bind(first_job.job_id.as_str())
|
||
|
|
.bind(first_job.execution_grant_id.as_str())
|
||
|
|
.fetch_one(&pool)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(counts, (1, 1, 1));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn changed_fingerprint_conflicts_and_current_authorization_is_rechecked_for_new_keys() {
|
||
|
|
let Some(pool) = pool().await else { return };
|
||
|
|
let service = service(pool);
|
||
|
|
let owner_id = unique_id("owner");
|
||
|
|
let owner = principal(owner_id.clone(), true, false);
|
||
|
|
let key = unique_id("key").to_string();
|
||
|
|
service
|
||
|
|
.submit(
|
||
|
|
&owner,
|
||
|
|
&key,
|
||
|
|
request("https://downloads.example/one".into()),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert!(matches!(
|
||
|
|
service
|
||
|
|
.submit(
|
||
|
|
&owner,
|
||
|
|
&key,
|
||
|
|
request("https://downloads.example/two".into())
|
||
|
|
)
|
||
|
|
.await,
|
||
|
|
Err(AdmissionError::IdempotencyConflict)
|
||
|
|
));
|
||
|
|
|
||
|
|
let lost_authorization = principal(owner_id, false, false);
|
||
|
|
let replay = service
|
||
|
|
.submit(
|
||
|
|
&lost_authorization,
|
||
|
|
&key,
|
||
|
|
request("https://downloads.example/one".into()),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(replay.disposition, MutationDisposition::Replay);
|
||
|
|
let fresh = service
|
||
|
|
.submit(
|
||
|
|
&lost_authorization,
|
||
|
|
unique_id("key").as_str(),
|
||
|
|
request("https://downloads.example/one".into()),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert!(matches!(fresh.decision, AdmissionDecision::Rejected { .. }));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn public_projection_and_outbox_do_not_leak_sensitive_url() {
|
||
|
|
let Some(pool) = pool().await else { return };
|
||
|
|
let service = service(pool.clone());
|
||
|
|
let owner = principal(unique_id("owner"), true, false);
|
||
|
|
let secret = unique_id("secret").to_string();
|
||
|
|
let accepted = accepted(
|
||
|
|
service
|
||
|
|
.submit(
|
||
|
|
&owner,
|
||
|
|
unique_id("key").as_str(),
|
||
|
|
request(format!(
|
||
|
|
"https://downloads.example/archive?token={secret}&signature=hidden"
|
||
|
|
)),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap(),
|
||
|
|
);
|
||
|
|
let public_text: String = sqlx::query_scalar(
|
||
|
|
"SELECT concat_ws(' ',
|
||
|
|
(SELECT source_projection::text FROM jobs WHERE job_id = $1),
|
||
|
|
(SELECT payload::text FROM outbox_messages WHERE topic = 'job.dispatch' AND payload->>'job_id' = $1),
|
||
|
|
(SELECT response_body::text FROM mutation_records WHERE response_body->'accepted'->>'job_id' = $1)
|
||
|
|
)",
|
||
|
|
)
|
||
|
|
.bind(accepted.job_id.as_str())
|
||
|
|
.fetch_one(&pool)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert!(!public_text.contains(&secret));
|
||
|
|
assert!(!public_text.contains("signature"));
|
||
|
|
let private_text: String = sqlx::query_scalar(
|
||
|
|
"SELECT source_input::text FROM sensitive_source_inputs
|
||
|
|
WHERE sensitive_source_ref = (SELECT sensitive_source_ref FROM jobs WHERE job_id = $1)",
|
||
|
|
)
|
||
|
|
.bind(accepted.job_id.as_str())
|
||
|
|
.fetch_one(&pool)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert!(private_text.contains(&secret));
|
||
|
|
}
|
||
|
|
|
||
|
|
#[tokio::test]
|
||
|
|
async fn credential_pin_and_accepted_grant_revoke_atomically_without_reactivation() {
|
||
|
|
let Some(pool) = pool().await else { return };
|
||
|
|
let service = service(pool.clone());
|
||
|
|
let owner_id = unique_id("owner");
|
||
|
|
let credential_id = unique_id("cred");
|
||
|
|
let version_id = unique_id("version");
|
||
|
|
let now = time::OffsetDateTime::now_utc();
|
||
|
|
let mut tx = pool.begin().await.unwrap();
|
||
|
|
sqlx::query(
|
||
|
|
"INSERT INTO source_credentials (
|
||
|
|
credential_id, owner_id, label, kind, status, active_version_id,
|
||
|
|
secret_mount, secret_path, created_at, updated_at
|
||
|
|
) VALUES ($1, $2, 'fixture', 'bearer_token', 'active', $3, 'secret', 'fixture', $4, $4)",
|
||
|
|
)
|
||
|
|
.bind(credential_id.as_str())
|
||
|
|
.bind(owner_id.as_str())
|
||
|
|
.bind(version_id.as_str())
|
||
|
|
.bind(now)
|
||
|
|
.execute(&mut *tx)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
sqlx::query(
|
||
|
|
"INSERT INTO source_credential_versions (
|
||
|
|
version_id, credential_id, vault_kv_version, created_at, destroyed_at, secret_mount, secret_path
|
||
|
|
) VALUES ($1, $2, 1, $3, NULL, 'secret', 'fixture/version')",
|
||
|
|
)
|
||
|
|
.bind(version_id.as_str())
|
||
|
|
.bind(credential_id.as_str())
|
||
|
|
.bind(now)
|
||
|
|
.execute(&mut *tx)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
tx.commit().await.unwrap();
|
||
|
|
|
||
|
|
let mut request = request("https://downloads.example/private".into());
|
||
|
|
let SourceSubmissionDto::Http(source) = &mut request.source else {
|
||
|
|
unreachable!()
|
||
|
|
};
|
||
|
|
source.credential_id = Some(credential_id.clone());
|
||
|
|
let accepted = accepted(
|
||
|
|
service
|
||
|
|
.submit(
|
||
|
|
&principal(owner_id, true, false),
|
||
|
|
unique_id("key").as_str(),
|
||
|
|
request,
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.unwrap(),
|
||
|
|
);
|
||
|
|
let binding: (String, String, String) = sqlx::query_as(
|
||
|
|
"SELECT credential_id, version_id, status
|
||
|
|
FROM execution_grant_credential_pins WHERE grant_id = $1",
|
||
|
|
)
|
||
|
|
.bind(accepted.execution_grant_id.as_str())
|
||
|
|
.fetch_one(&pool)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(
|
||
|
|
binding,
|
||
|
|
(
|
||
|
|
credential_id.to_string(),
|
||
|
|
version_id.to_string(),
|
||
|
|
"active".into()
|
||
|
|
)
|
||
|
|
);
|
||
|
|
|
||
|
|
let admin = principal(unique_id("admin"), false, true);
|
||
|
|
let (left, right) = tokio::join!(
|
||
|
|
service.revoke_execution_grant(&admin, &accepted.execution_grant_id),
|
||
|
|
service.revoke_execution_grant(&admin, &accepted.execution_grant_id)
|
||
|
|
);
|
||
|
|
left.unwrap();
|
||
|
|
right.unwrap();
|
||
|
|
let statuses: (String, String, i64) = sqlx::query_as(
|
||
|
|
"SELECT
|
||
|
|
(SELECT status FROM execution_grants WHERE grant_id = $1),
|
||
|
|
(SELECT status FROM execution_grant_credential_pins WHERE grant_id = $1),
|
||
|
|
(SELECT count(*) FROM outbox_messages WHERE topic = 'execution_grant.revoked' AND payload->>'grant_id' = $1)",
|
||
|
|
)
|
||
|
|
.bind(accepted.execution_grant_id.as_str())
|
||
|
|
.fetch_one(&pool)
|
||
|
|
.await
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(statuses, ("revoked".into(), "revoked".into(), 1));
|
||
|
|
|
||
|
|
let mut tx = pool.begin().await.unwrap();
|
||
|
|
let pin = remora_credentials::CredentialGrantVersion {
|
||
|
|
credential_id,
|
||
|
|
version_id,
|
||
|
|
};
|
||
|
|
assert!(
|
||
|
|
CredentialRegistry::bind_execution_grant_in_tx(
|
||
|
|
&mut tx,
|
||
|
|
&accepted.execution_grant_id,
|
||
|
|
&pin,
|
||
|
|
time::OffsetDateTime::now_utc(),
|
||
|
|
)
|
||
|
|
.await
|
||
|
|
.is_err()
|
||
|
|
);
|
||
|
|
tx.rollback().await.unwrap();
|
||
|
|
}
|