1288 lines
43 KiB
Rust
1288 lines
43 KiB
Rust
use std::{net::TcpListener, time::Duration};
|
|
|
|
use actix_web::{App, HttpServer, web};
|
|
use base64::{
|
|
Engine as _,
|
|
engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD},
|
|
};
|
|
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
|
|
use remora_admission::{AdmissionConfig, AdmissionService, migrate as admission_migrate};
|
|
use remora_api::{PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api};
|
|
use remora_attempt::{AttemptConfig, AttemptService, migrate as attempt_migrate};
|
|
use remora_contracts::{
|
|
AttemptPageDto, DeletionOperationDto, JobPageDto, JobResourceDto, ResultDescriptorDto,
|
|
ResultMemberPageDto, SourceCredentialDto,
|
|
};
|
|
use remora_coordination::{
|
|
ActivateOptions, CoordinationStore, LeaseOwner, ProgressValue, StoreConfig,
|
|
};
|
|
use remora_credentials::{
|
|
CredentialRegistry, VaultClient, VaultConfig, migrate as credentials_migrate,
|
|
};
|
|
use remora_deletion::{
|
|
DeletionConfig, DeletionService, TargetRegistry, migrate as deletion_migrate,
|
|
};
|
|
use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
|
|
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
|
use remora_identity::{
|
|
DefaultTarget, OidcConfig, OidcIdentityProvider, migrate as identity_migrate,
|
|
};
|
|
use remora_result::migrate as result_migrate;
|
|
use remora_target::{
|
|
CreateObjectRequest, TargetStore, TargetStoreConfig, attempt_manifest_key_in,
|
|
attempt_object_key_in,
|
|
};
|
|
use reqwest::{Client, StatusCode};
|
|
use serde_json::{Value, json};
|
|
use sqlx::PgPool;
|
|
use time::OffsetDateTime;
|
|
|
|
struct Harness {
|
|
pool: PgPool,
|
|
coordination: CoordinationStore,
|
|
base: String,
|
|
client: Client,
|
|
token: String,
|
|
server: actix_web::dev::ServerHandle,
|
|
}
|
|
|
|
impl Harness {
|
|
async fn new() -> Option<Self> {
|
|
if std::env::var("REMORA_PUBLIC_API_REQUIRE").is_err() {
|
|
return None;
|
|
}
|
|
let pool = connect(
|
|
&required_env("DATABASE_URL"),
|
|
PoolConfig {
|
|
max_connections: 24,
|
|
acquire_timeout: Duration::from_secs(10),
|
|
idle_timeout: Some(Duration::from_secs(30)),
|
|
},
|
|
)
|
|
.await
|
|
.unwrap();
|
|
durable_migrate(&pool).await.unwrap();
|
|
identity_migrate(&pool).await.unwrap();
|
|
credentials_migrate(&pool).await.unwrap();
|
|
admission_migrate(&pool).await.unwrap();
|
|
attempt_migrate(&pool).await.unwrap();
|
|
result_migrate(&pool).await.unwrap();
|
|
deletion_migrate(&pool).await.unwrap();
|
|
sqlx::query("TRUNCATE TABLE jobs CASCADE")
|
|
.execute(&pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("TRUNCATE TABLE source_credentials CASCADE")
|
|
.execute(&pool)
|
|
.await
|
|
.unwrap();
|
|
|
|
let target = target_store();
|
|
let target_store_id =
|
|
TargetStoreId::parse(target.config().target_store_id.clone()).unwrap();
|
|
let bucket = BucketName::parse(target.config().bucket.clone()).unwrap();
|
|
let prefix =
|
|
TargetPrefix::parse(&required_env("REMORA_TARGET_PREFIX"), ["_remora"]).unwrap();
|
|
let admission = AdmissionService::new(
|
|
pool.clone(),
|
|
AdmissionConfig {
|
|
default_target: DefaultTarget {
|
|
store: target_store_id,
|
|
bucket,
|
|
prefix,
|
|
},
|
|
reserved_target_segments: vec!["_remora".into()],
|
|
policy_version: "public-api-v1".into(),
|
|
},
|
|
)
|
|
.unwrap();
|
|
let coordination = CoordinationStore::connect(
|
|
&required_env("REMORA_COORDINATION_URL"),
|
|
format!(
|
|
"{}wp21:{}:",
|
|
required_env("REMORA_COORDINATION_NAMESPACE"),
|
|
nonce()
|
|
),
|
|
StoreConfig {
|
|
global_limit: 16,
|
|
lease_ttl: Duration::from_secs(10),
|
|
reserve_ttl: Duration::from_secs(10),
|
|
progress_ttl: Duration::from_secs(10),
|
|
cancel_ttl: None,
|
|
},
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let attempts = AttemptService::new(
|
|
pool.clone(),
|
|
coordination.clone(),
|
|
AttemptConfig {
|
|
max_attempts: 3,
|
|
local_limit: 8,
|
|
},
|
|
)
|
|
.unwrap();
|
|
let vault = VaultClient::new(VaultConfig::from_env().unwrap()).unwrap();
|
|
let credentials = CredentialRegistry::new(pool.clone(), vault);
|
|
let targets = TargetRegistry::new([target]);
|
|
let deletion =
|
|
DeletionService::new(pool.clone(), targets.clone(), DeletionConfig::default()).unwrap();
|
|
let state = PublicApiState::new(
|
|
pool.clone(),
|
|
PublicApiServices {
|
|
identity: oidc_provider(),
|
|
admission,
|
|
attempts,
|
|
credentials,
|
|
deletion,
|
|
targets,
|
|
},
|
|
PublicApiConfig {
|
|
cursor_secret: b"public-api-fixture-cursor-secret".to_vec(),
|
|
member_access_ttl: Duration::from_secs(60),
|
|
},
|
|
)
|
|
.unwrap();
|
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = HttpServer::new(move || {
|
|
App::new()
|
|
.app_data(web::Data::new(state.clone()))
|
|
.configure(configure_public_api)
|
|
})
|
|
.listen(listener)
|
|
.unwrap()
|
|
.run();
|
|
let handle = server.handle();
|
|
tokio::spawn(server);
|
|
let token = mint_provider_token().await.unwrap();
|
|
Some(Self {
|
|
pool,
|
|
coordination,
|
|
base: format!("http://{address}/v1"),
|
|
client: Client::builder()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.build()
|
|
.unwrap(),
|
|
token,
|
|
server: handle,
|
|
})
|
|
}
|
|
|
|
fn request(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder {
|
|
self.request_with_token(method, path, &self.token)
|
|
}
|
|
|
|
fn request_with_token(
|
|
&self,
|
|
method: reqwest::Method,
|
|
path: &str,
|
|
token: &str,
|
|
) -> reqwest::RequestBuilder {
|
|
self.client
|
|
.request(method, format!("{}{}", self.base, path))
|
|
.bearer_auth(token)
|
|
}
|
|
}
|
|
|
|
impl Drop for Harness {
|
|
fn drop(&mut self) {
|
|
tokio::spawn(self.server.stop(true));
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn public_routes_are_authenticated_replayable_authorized_and_redacted() {
|
|
let Some(harness) = Harness::new().await else {
|
|
return;
|
|
};
|
|
let unauthenticated = harness
|
|
.client
|
|
.get(format!("{}/jobs", harness.base))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(unauthenticated.status(), StatusCode::UNAUTHORIZED);
|
|
assert_eq!(
|
|
unauthenticated.headers()["content-type"],
|
|
"application/problem+json"
|
|
);
|
|
|
|
let submission = json!({
|
|
"source": {
|
|
"kind": "http",
|
|
"url": "https://example.invalid/private?token=never-public",
|
|
"result_path": "artifact.bin"
|
|
},
|
|
"label": "public-api-test"
|
|
});
|
|
let submit_key = idempotency_key();
|
|
let submitted = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", &submit_key)
|
|
.json(&submission)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(submitted.status(), StatusCode::ACCEPTED);
|
|
assert!(submitted.headers().contains_key("idempotency-expires-at"));
|
|
let job: Value = submitted.json().await.unwrap();
|
|
serde_json::from_value::<JobResourceDto>(job.clone()).unwrap();
|
|
assert_eq!(job["resource_kind"], "job");
|
|
assert_eq!(job["phase"], "Queued");
|
|
let transcript = serde_json::to_string(&job).unwrap();
|
|
assert!(!transcript.contains("token=never-public"));
|
|
let job_id = job["id"].as_str().unwrap();
|
|
|
|
let replay = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", &submit_key)
|
|
.json(&submission)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(replay.status(), StatusCode::ACCEPTED);
|
|
assert_eq!(replay.json::<Value>().await.unwrap(), job);
|
|
let conflict = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", &submit_key)
|
|
.json(&json!({
|
|
"source": {
|
|
"kind": "http",
|
|
"url": "https://example.invalid/changed",
|
|
"result_path": "artifact.bin"
|
|
}
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(conflict.status(), StatusCode::CONFLICT);
|
|
|
|
let second = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.json(&json!({
|
|
"source": {
|
|
"kind": "http",
|
|
"url": "https://example.invalid/second",
|
|
"result_path": "artifact.bin"
|
|
},
|
|
"label": "second-public-api-test"
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(second.status(), StatusCode::ACCEPTED);
|
|
let second_job: Value = second.json().await.unwrap();
|
|
let second_job_id = second_job["id"].as_str().unwrap();
|
|
|
|
let listed = harness
|
|
.request(reqwest::Method::GET, "/jobs?limit=1")
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(listed.status(), StatusCode::OK);
|
|
let page: Value = listed.json().await.unwrap();
|
|
serde_json::from_value::<JobPageDto>(page.clone()).unwrap();
|
|
let first_page_id = page["items"][0]["id"].as_str().unwrap();
|
|
assert!(page["event_cursor"].as_str().unwrap().len() >= 16);
|
|
let next_cursor = page["next_cursor"].as_str().unwrap();
|
|
let next_page = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?limit=1&cursor={next_cursor}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(next_page.status(), StatusCode::OK);
|
|
let next_page: Value = next_page.json().await.unwrap();
|
|
serde_json::from_value::<JobPageDto>(next_page.clone()).unwrap();
|
|
let second_page_id = next_page["items"][0]["id"].as_str().unwrap();
|
|
assert_ne!(first_page_id, second_page_id);
|
|
assert_eq!(
|
|
[first_page_id, second_page_id]
|
|
.into_iter()
|
|
.collect::<std::collections::BTreeSet<_>>(),
|
|
[job_id, second_job_id]
|
|
.into_iter()
|
|
.collect::<std::collections::BTreeSet<_>>()
|
|
);
|
|
let cross_filter = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?include_deleted=true&cursor={next_cursor}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(cross_filter.status(), StatusCode::BAD_REQUEST);
|
|
let altered_cursor = format!("{}x", page["event_cursor"].as_str().unwrap());
|
|
let invalid = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?cursor={altered_cursor}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(invalid.status(), StatusCode::BAD_REQUEST);
|
|
|
|
let fetched = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(fetched.status(), StatusCode::OK);
|
|
assert_eq!(fetched.json::<Value>().await.unwrap()["id"], job_id);
|
|
|
|
install_mutation_record_failure(&harness.pool).await;
|
|
let rolled_back = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(rolled_back.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
let phase: String = sqlx::query_scalar("SELECT phase FROM jobs WHERE job_id = $1")
|
|
.bind(job_id)
|
|
.fetch_one(&harness.pool)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(phase, "queued");
|
|
remove_mutation_record_failure(&harness.pool).await;
|
|
|
|
let cancel_key = idempotency_key();
|
|
let cancelled = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
|
|
.header("Idempotency-Key", &cancel_key)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(cancelled.status(), StatusCode::ACCEPTED);
|
|
let cancel_projection: Value = cancelled.json().await.unwrap();
|
|
let cancel_replay = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
|
|
.header("Idempotency-Key", &cancel_key)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
cancel_replay.json::<Value>().await.unwrap(),
|
|
cancel_projection
|
|
);
|
|
|
|
let credential_key = idempotency_key();
|
|
let secret = "synthetic-secret-never-return";
|
|
let created = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &credential_key)
|
|
.json(&json!({
|
|
"kind": "bearer_token",
|
|
"label": "fixture credential",
|
|
"material": { "token": secret }
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(created.status(), StatusCode::CREATED);
|
|
let credential: Value = created.json().await.unwrap();
|
|
serde_json::from_value::<SourceCredentialDto>(credential.clone()).unwrap();
|
|
assert!(!serde_json::to_string(&credential).unwrap().contains(secret));
|
|
let credential_replay = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &credential_key)
|
|
.json(&json!({
|
|
"kind": "bearer_token",
|
|
"label": "fixture credential",
|
|
"material": { "token": secret }
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(credential_replay.json::<Value>().await.unwrap(), credential);
|
|
let credential_conflict = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &credential_key)
|
|
.json(&json!({
|
|
"kind": "bearer_token",
|
|
"label": "changed fixture credential",
|
|
"material": { "token": secret }
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(credential_conflict.status(), StatusCode::CONFLICT);
|
|
let credential_id = credential["id"].as_str().unwrap();
|
|
|
|
let credential_page = harness
|
|
.request(reqwest::Method::GET, "/source-credentials?limit=1")
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(credential_page.status(), StatusCode::OK);
|
|
assert_eq!(
|
|
credential_page.json::<Value>().await.unwrap()["items"][0]["id"],
|
|
credential_id
|
|
);
|
|
let fetched_credential = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/source-credentials/{credential_id}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(fetched_credential.status(), StatusCode::OK);
|
|
let rotated = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{credential_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.json(&json!({ "material": { "token": "rotated-secret" } }))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(rotated.status(), StatusCode::CREATED);
|
|
|
|
let hugging_face_key = idempotency_key();
|
|
let hugging_face_request = json!({
|
|
"kind": "hugging_face",
|
|
"label": "fixture Hugging Face credential",
|
|
"material": { "token": "synthetic-hugging-face-token" }
|
|
});
|
|
let lost_create_response = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &hugging_face_key)
|
|
.json(&hugging_face_request)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(lost_create_response.status(), StatusCode::CREATED);
|
|
drop(lost_create_response);
|
|
let hugging_face = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &hugging_face_key)
|
|
.json(&hugging_face_request)
|
|
.send()
|
|
.await
|
|
.unwrap()
|
|
.json::<Value>()
|
|
.await
|
|
.unwrap();
|
|
serde_json::from_value::<SourceCredentialDto>(hugging_face.clone()).unwrap();
|
|
assert_eq!(hugging_face["kind"], "hugging_face");
|
|
let hugging_face_id = hugging_face["id"].as_str().unwrap();
|
|
let rotate_key = idempotency_key();
|
|
let rotate_request = json!({ "material": { "token": "rotated-hugging-face-token" } });
|
|
let lost_rotate_response = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{hugging_face_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", &rotate_key)
|
|
.json(&rotate_request)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(lost_rotate_response.status(), StatusCode::CREATED);
|
|
drop(lost_rotate_response);
|
|
let hugging_face_rotated = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{hugging_face_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", &rotate_key)
|
|
.json(&rotate_request)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hugging_face_rotated.status(), StatusCode::CREATED);
|
|
let hugging_face_rotated: Value = hugging_face_rotated.json().await.unwrap();
|
|
serde_json::from_value::<SourceCredentialDto>(hugging_face_rotated.clone()).unwrap();
|
|
assert_eq!(hugging_face_rotated["kind"], "hugging_face");
|
|
let rotate_conflict = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{hugging_face_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", &rotate_key)
|
|
.json(&json!({ "material": { "token": "changed-token" } }))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(rotate_conflict.status(), StatusCode::CONFLICT);
|
|
|
|
let revoked = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{credential_id}/revoke"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(revoked.status(), StatusCode::OK);
|
|
assert_eq!(revoked.json::<Value>().await.unwrap()["status"], "revoked");
|
|
|
|
let (attempt_id, worker_id) = seed_committed_result(&harness.pool, job_id).await;
|
|
seed_failed_attempt(&harness.pool, job_id).await;
|
|
let owner = LeaseOwner::new(attempt_id.as_str(), 1, worker_id.as_str()).unwrap();
|
|
harness.coordination.enqueue(job_id).await.unwrap();
|
|
harness
|
|
.coordination
|
|
.activate(job_id, &owner, ActivateOptions::default())
|
|
.await
|
|
.unwrap();
|
|
harness
|
|
.coordination
|
|
.report_progress_value(
|
|
job_id,
|
|
&owner,
|
|
ProgressValue {
|
|
completed_bytes: 7,
|
|
total_bytes: Some(7),
|
|
completed_members: 1,
|
|
total_members: Some(1),
|
|
},
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let with_progress = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap()
|
|
.json::<Value>()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(with_progress["progress"]["completed_bytes"], 7);
|
|
assert_eq!(with_progress["progress"]["attempt_id"], attempt_id.as_str());
|
|
harness.coordination.release(job_id, &owner).await.unwrap();
|
|
let foreign = LeaseOwner::new(
|
|
opaque_id("attempt").as_str(),
|
|
2,
|
|
opaque_id("worker").as_str(),
|
|
)
|
|
.unwrap();
|
|
harness
|
|
.coordination
|
|
.activate(job_id, &foreign, ActivateOptions::default())
|
|
.await
|
|
.unwrap();
|
|
let stale_progress = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap()
|
|
.json::<Value>()
|
|
.await
|
|
.unwrap();
|
|
assert!(stale_progress["progress"].is_null());
|
|
let result = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}/result"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(result.status(), StatusCode::OK);
|
|
let result = result.json::<Value>().await.unwrap();
|
|
serde_json::from_value::<ResultDescriptorDto>(result.clone()).unwrap();
|
|
assert_eq!(result["kind"], "manifest");
|
|
|
|
let members = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{job_id}/result/members"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(members.status(), StatusCode::OK);
|
|
let members: Value = members.json().await.unwrap();
|
|
serde_json::from_value::<ResultMemberPageDto>(members.clone()).unwrap();
|
|
let member_id = members["items"][0]["id"].as_str().unwrap();
|
|
let access = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{job_id}/result/members/{member_id}/content"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(access.status(), StatusCode::TEMPORARY_REDIRECT);
|
|
assert_eq!(access.headers()["cache-control"], "no-store");
|
|
|
|
let attempts = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}/attempts"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempts.status(), StatusCode::OK);
|
|
let attempts = attempts.json::<Value>().await.unwrap();
|
|
serde_json::from_value::<AttemptPageDto>(attempts.clone()).unwrap();
|
|
let failed_attempt = attempts["items"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.find(|attempt| !attempt["failure"].is_null())
|
|
.unwrap();
|
|
assert_eq!(failed_attempt["failure"]["class"], "internal");
|
|
assert_eq!(failed_attempt["failure"]["message"], "job failed");
|
|
assert!(
|
|
!serde_json::to_string(failed_attempt)
|
|
.unwrap()
|
|
.contains("database password")
|
|
);
|
|
|
|
let revoked_job = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/revoke"))
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(revoked_job.status(), StatusCode::ACCEPTED);
|
|
|
|
let soft_deleted = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/jobs/{job_id}/soft-delete"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(soft_deleted.status(), StatusCode::OK);
|
|
let hidden = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}/result"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hidden.status(), StatusCode::NOT_FOUND);
|
|
let restored = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/restore"))
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(restored.status(), StatusCode::OK);
|
|
|
|
let hard_deleted = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/jobs/{job_id}/hard-delete"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hard_deleted.status(), StatusCode::ACCEPTED);
|
|
let operation: Value = hard_deleted.json().await.unwrap();
|
|
serde_json::from_value::<DeletionOperationDto>(operation.clone()).unwrap();
|
|
let operation_id = operation["id"].as_str().unwrap();
|
|
let fetched_operation = harness
|
|
.request(reqwest::Method::GET, &format!("/deletions/{operation_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(fetched_operation.status(), StatusCode::OK);
|
|
let tombstone = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(tombstone.status(), StatusCode::OK);
|
|
let tombstone = tombstone.json::<Value>().await.unwrap();
|
|
serde_json::from_value::<JobResourceDto>(tombstone.clone()).unwrap();
|
|
assert_eq!(tombstone["resource_kind"], "tombstone");
|
|
|
|
let peer_owner = opaque_id("peer-principal");
|
|
sqlx::query("UPDATE jobs SET owner_id = $2 WHERE job_id = $1")
|
|
.bind(second_job_id)
|
|
.bind(peer_owner.as_str())
|
|
.execute(&harness.pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("UPDATE source_credentials SET owner_id = $2 WHERE credential_id = $1")
|
|
.bind(hugging_face_id)
|
|
.bind(peer_owner.as_str())
|
|
.execute(&harness.pool)
|
|
.await
|
|
.unwrap();
|
|
let admin_peer_job = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{second_job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(admin_peer_job.status(), StatusCode::OK);
|
|
let admin_peer_credential = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/source-credentials/{hugging_face_id}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(admin_peer_credential.status(), StatusCode::OK);
|
|
|
|
revoke_remora_grants().await.unwrap();
|
|
let unprivileged_token = mint_provider_token().await.unwrap();
|
|
let lost_scope = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{job_id}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(lost_scope.status(), StatusCode::NOT_FOUND);
|
|
let hidden_peer_job = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{second_job_id}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hidden_peer_job.status(), StatusCode::NOT_FOUND);
|
|
let hidden_peer_credential = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/source-credentials/{hugging_face_id}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hidden_peer_credential.status(), StatusCode::NOT_FOUND);
|
|
let invalidated_cursor = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?limit=1&cursor={next_cursor}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(invalidated_cursor.status(), StatusCode::BAD_REQUEST);
|
|
let forbidden_revoke = harness
|
|
.request_with_token(
|
|
reqwest::Method::POST,
|
|
&format!("/jobs/{second_job_id}/revoke"),
|
|
&unprivileged_token,
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(forbidden_revoke.status(), StatusCode::FORBIDDEN);
|
|
}
|
|
|
|
async fn seed_committed_result(pool: &PgPool, job_id: &str) -> (OpaqueId, OpaqueId) {
|
|
let attempt_id = opaque_id("attempt");
|
|
let worker_id = opaque_id("worker");
|
|
let manifest_id = opaque_id("manifest");
|
|
let member_id = opaque_id("member");
|
|
let object_id = opaque_id("object");
|
|
let prefix = TargetPrefix::parse(
|
|
&required_env("REMORA_TARGET_PREFIX"),
|
|
std::iter::empty::<&str>(),
|
|
)
|
|
.unwrap()
|
|
.canonical();
|
|
let member_key = attempt_object_key_in(&prefix, &attempt_id, &object_id, Some("artifact.bin"));
|
|
let manifest_key = attempt_manifest_key_in(&prefix, &attempt_id, &manifest_id);
|
|
let target = target_store();
|
|
let member = target
|
|
.put_create_only(
|
|
&CreateObjectRequest::for_target_attempt_object(
|
|
&prefix,
|
|
&attempt_id,
|
|
&object_id,
|
|
Some("artifact.bin"),
|
|
),
|
|
b"payload".to_vec(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let manifest = target
|
|
.put_create_only(
|
|
&CreateObjectRequest::for_target_attempt_manifest(&prefix, &attempt_id, &manifest_id),
|
|
b"manifest".to_vec(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let now = OffsetDateTime::now_utc();
|
|
sqlx::query(
|
|
"INSERT INTO attempts (
|
|
attempt_id, job_id, fence, worker_incarnation, state, initial_progress,
|
|
authorized_at, executing_at, committed_at, staging_origin_attempt_id
|
|
) VALUES ($1, $2, 1, $3, 'committed', 0, $4, $4, $4, $1)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(job_id)
|
|
.bind(worker_id.as_str())
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"INSERT INTO result_inventories (
|
|
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
|
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
|
manifest_size, member_count, total_bytes, staging_origin_attempt_id, target_prefix
|
|
) VALUES ($1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 1, 7, $1, $9)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(manifest_id.as_str())
|
|
.bind(&manifest.identity.target_store_id)
|
|
.bind(&manifest.identity.bucket)
|
|
.bind(&manifest_key)
|
|
.bind(&manifest.identity.version_id)
|
|
.bind(manifest.content_sha256.as_str())
|
|
.bind(i64::try_from(manifest.size).unwrap())
|
|
.bind(&prefix)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"INSERT INTO result_inventory_members (
|
|
attempt_id, member_id, path, size, object_store, object_bucket,
|
|
object_key, object_version_id, object_sha256
|
|
) VALUES ($1, $2, 'artifact.bin', 7, $3, $4, $5, $6, $7)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(member_id.as_str())
|
|
.bind(&member.identity.target_store_id)
|
|
.bind(&member.identity.bucket)
|
|
.bind(&member_key)
|
|
.bind(&member.identity.version_id)
|
|
.bind(member.content_sha256.as_str())
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
let committed = json!({
|
|
"attempt_id": attempt_id,
|
|
"manifest_id": manifest_id,
|
|
"schema_version": 1,
|
|
"manifest": {
|
|
"store": manifest.identity.target_store_id,
|
|
"bucket": manifest.identity.bucket,
|
|
"key": manifest.identity.key,
|
|
"version_id": manifest.identity.version_id,
|
|
"sha256": manifest.content_sha256.as_str(),
|
|
"size": manifest.size
|
|
},
|
|
"member_count": 1,
|
|
"total_bytes": 7
|
|
});
|
|
sqlx::query(
|
|
"UPDATE jobs
|
|
SET phase = 'succeeded', lifecycle = 'visible', revision = revision + 1,
|
|
current_fence = 1, authority_attempt_id = $2, committed_result = $3,
|
|
terminal_at = $4, updated_at = $4
|
|
WHERE job_id = $1",
|
|
)
|
|
.bind(job_id)
|
|
.bind(attempt_id.as_str())
|
|
.bind(committed)
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
(attempt_id, worker_id)
|
|
}
|
|
|
|
async fn seed_failed_attempt(pool: &PgPool, job_id: &str) {
|
|
let attempt_id = opaque_id("failed-attempt");
|
|
let worker_id = opaque_id("failed-worker");
|
|
let now = OffsetDateTime::now_utc();
|
|
sqlx::query(
|
|
"INSERT INTO attempts (
|
|
attempt_id, job_id, fence, worker_incarnation, state, initial_progress,
|
|
failure, authorized_at, executing_at, stopped_at, staging_origin_attempt_id
|
|
) VALUES ($1, $2, 2, $3, 'stopped', 0, $4, $5, $5, $5, $1)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(job_id)
|
|
.bind(worker_id.as_str())
|
|
.bind(json!({
|
|
"class": "internal",
|
|
"disposition": "retryable",
|
|
"details": {
|
|
"exception": "database password synthetic-secret-never-return"
|
|
}
|
|
}))
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
async fn install_mutation_record_failure(pool: &PgPool) {
|
|
sqlx::query(
|
|
"CREATE OR REPLACE FUNCTION remora_test_fail_cancel_record()
|
|
RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN
|
|
IF NEW.scope_operation = 'job.cancel' THEN
|
|
RAISE EXCEPTION 'synthetic mutation record failure';
|
|
END IF;
|
|
RETURN NEW;
|
|
END
|
|
$$",
|
|
)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"CREATE TRIGGER remora_test_fail_cancel_record
|
|
BEFORE INSERT ON mutation_records
|
|
FOR EACH ROW EXECUTE FUNCTION remora_test_fail_cancel_record()",
|
|
)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
async fn remove_mutation_record_failure(pool: &PgPool) {
|
|
sqlx::query("DROP TRIGGER remora_test_fail_cancel_record ON mutation_records")
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("DROP FUNCTION remora_test_fail_cancel_record()")
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
async fn revoke_remora_grants() -> Result<(), String> {
|
|
match required_env("REMORA_OIDC_PROVIDER").as_str() {
|
|
"keycloak" => revoke_keycloak_remora_roles().await,
|
|
"zitadel" => revoke_zitadel_remora_roles().await,
|
|
provider => Err(format!("unsupported OIDC provider {provider}")),
|
|
}
|
|
}
|
|
|
|
async fn revoke_keycloak_remora_roles() -> Result<(), String> {
|
|
let realm = required_env("REMORA_OIDC_TENANT");
|
|
let client_id = required_env("REMORA_OIDC_CLIENT_ID");
|
|
let username = required_env("REMORA_OIDC_USERNAME");
|
|
let client = Client::new();
|
|
let admin_response = client
|
|
.post("http://keycloak.remora.test:8080/realms/master/protocol/openid-connect/token")
|
|
.form(&[
|
|
("grant_type", "password"),
|
|
("client_id", "admin-cli"),
|
|
("username", "remora-admin"),
|
|
("password", "synthetic-keycloak"),
|
|
])
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
let admin_body: Value = admin_response
|
|
.json()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
let admin_token = admin_body["access_token"]
|
|
.as_str()
|
|
.ok_or_else(|| "keycloak admin token missing".to_owned())?;
|
|
let users: Value = client
|
|
.get(format!(
|
|
"http://keycloak.remora.test:8080/admin/realms/{realm}/users?username={username}&exact=true"
|
|
))
|
|
.bearer_auth(admin_token)
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?
|
|
.json()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
let user_id = users
|
|
.as_array()
|
|
.and_then(|rows| rows.first())
|
|
.and_then(|row| row["id"].as_str())
|
|
.ok_or_else(|| "keycloak user missing".to_owned())?;
|
|
let clients: Value = client
|
|
.get(format!(
|
|
"http://keycloak.remora.test:8080/admin/realms/{realm}/clients?clientId={client_id}"
|
|
))
|
|
.bearer_auth(admin_token)
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?
|
|
.json()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
let client_uuid = clients
|
|
.as_array()
|
|
.and_then(|rows| rows.first())
|
|
.and_then(|row| row["id"].as_str())
|
|
.ok_or_else(|| "keycloak client missing".to_owned())?;
|
|
let roles_url = format!(
|
|
"http://keycloak.remora.test:8080/admin/realms/{realm}/users/{user_id}/role-mappings/clients/{client_uuid}"
|
|
);
|
|
let mapped: Value = client
|
|
.get(&roles_url)
|
|
.bearer_auth(admin_token)
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?
|
|
.json()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
let roles = mapped
|
|
.as_array()
|
|
.cloned()
|
|
.unwrap_or_default()
|
|
.into_iter()
|
|
.filter(|role| {
|
|
role["name"]
|
|
.as_str()
|
|
.is_some_and(|name| name.starts_with("remora:v1:"))
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let response = client
|
|
.delete(roles_url)
|
|
.bearer_auth(admin_token)
|
|
.json(&roles)
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
if response.status().is_success() {
|
|
Ok(())
|
|
} else {
|
|
Err(format!(
|
|
"keycloak role revocation failed: {}",
|
|
response.status()
|
|
))
|
|
}
|
|
}
|
|
|
|
async fn revoke_zitadel_remora_roles() -> Result<(), String> {
|
|
let issuer = required_env("REMORA_OIDC_ISSUER");
|
|
let organization = required_env("REMORA_OIDC_TENANT");
|
|
let user_id = required_env("REMORA_OIDC_USER_ID");
|
|
let project_id = required_env("REMORA_OIDC_PROJECT_ID");
|
|
let token = required_env("REMORA_OIDC_MGMT_TOKEN");
|
|
let host = issuer
|
|
.strip_prefix("http://")
|
|
.or_else(|| issuer.strip_prefix("https://"))
|
|
.unwrap_or(&issuer);
|
|
let client = Client::new();
|
|
let grants: Value = client
|
|
.post(format!("{issuer}/management/v1/users/grants/_search"))
|
|
.header("Host", host)
|
|
.bearer_auth(&token)
|
|
.header("x-zitadel-orgid", &organization)
|
|
.json(&json!({ "queries": [{ "userIdQuery": { "userId": user_id } }] }))
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?
|
|
.json()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
let grant_id = grants["result"]
|
|
.as_array()
|
|
.and_then(|rows| {
|
|
rows.iter().find(|row| {
|
|
row["projectId"]
|
|
.as_str()
|
|
.is_some_and(|value| value == project_id)
|
|
})
|
|
})
|
|
.and_then(|row| row["id"].as_str())
|
|
.ok_or_else(|| "zitadel project grant missing".to_owned())?;
|
|
let response = client
|
|
.put(format!(
|
|
"{issuer}/management/v1/users/{user_id}/grants/{grant_id}"
|
|
))
|
|
.header("Host", host)
|
|
.bearer_auth(token)
|
|
.header("x-zitadel-orgid", organization)
|
|
.json(&json!({ "roleKeys": ["test-user"] }))
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
if response.status().is_success() {
|
|
Ok(())
|
|
} else {
|
|
Err(format!(
|
|
"zitadel role revocation failed: {}",
|
|
response.status()
|
|
))
|
|
}
|
|
}
|
|
|
|
fn oidc_provider() -> OidcIdentityProvider {
|
|
let provider = required_env("REMORA_OIDC_PROVIDER");
|
|
let audience = if provider == "zitadel" {
|
|
required_env("REMORA_OIDC_PROJECT_ID")
|
|
} else {
|
|
required_env("REMORA_OIDC_CLIENT_ID")
|
|
};
|
|
let mut config = OidcConfig::new(required_env("REMORA_OIDC_ISSUER"), audience).unwrap();
|
|
if provider == "zitadel" {
|
|
let project_id = required_env("REMORA_OIDC_PROJECT_ID");
|
|
config = config
|
|
.with_zitadel_project_roles()
|
|
.with_project_role_claim(format!("urn:zitadel:iam:org:project:{project_id}:roles"))
|
|
.with_resource_access_client(required_env("REMORA_OIDC_CLIENT_ID"));
|
|
} else {
|
|
config = config.with_resource_access_client(required_env("REMORA_OIDC_CLIENT_ID"));
|
|
}
|
|
OidcIdentityProvider::new(config).unwrap()
|
|
}
|
|
|
|
async fn mint_provider_token() -> Result<String, String> {
|
|
match required_env("REMORA_OIDC_PROVIDER").as_str() {
|
|
"keycloak" => mint_keycloak_token().await,
|
|
"zitadel" => mint_zitadel_token().await,
|
|
provider => Err(format!("unsupported OIDC provider {provider}")),
|
|
}
|
|
}
|
|
|
|
async fn mint_keycloak_token() -> Result<String, String> {
|
|
let response = Client::new()
|
|
.post(format!(
|
|
"{}/protocol/openid-connect/token",
|
|
required_env("REMORA_OIDC_ISSUER").trim_end_matches('/')
|
|
))
|
|
.form(&[
|
|
("grant_type", "password"),
|
|
("client_id", required_env("REMORA_OIDC_CLIENT_ID").as_str()),
|
|
("username", required_env("REMORA_OIDC_USERNAME").as_str()),
|
|
("password", required_env("REMORA_OIDC_PASSWORD").as_str()),
|
|
("scope", "openid"),
|
|
])
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
token_from_response(response).await
|
|
}
|
|
|
|
async fn mint_zitadel_token() -> Result<String, String> {
|
|
let issuer = required_env("REMORA_OIDC_ISSUER")
|
|
.trim_end_matches('/')
|
|
.to_owned();
|
|
let token_url = format!("{issuer}/oauth/v2/token");
|
|
let encoded_key = required_env("REMORA_OIDC_USER_KEY_B64");
|
|
let key_json = URL_SAFE_NO_PAD
|
|
.decode(encoded_key.as_bytes())
|
|
.or_else(|_| STANDARD.decode(encoded_key.as_bytes()))
|
|
.map_err(|error| format!("decode user key: {error}"))?;
|
|
let key: Value =
|
|
serde_json::from_slice(&key_json).map_err(|error| format!("parse user key: {error}"))?;
|
|
let key_id = key
|
|
.get("keyId")
|
|
.or_else(|| key.get("key_id"))
|
|
.and_then(Value::as_str)
|
|
.ok_or_else(|| "user key missing keyId".to_owned())?
|
|
.to_owned();
|
|
let subject = key
|
|
.get("userId")
|
|
.or_else(|| key.get("userID"))
|
|
.and_then(Value::as_str)
|
|
.map(str::to_owned)
|
|
.unwrap_or_else(|| required_env("REMORA_OIDC_USER_ID"));
|
|
let private_key = key["key"]
|
|
.as_str()
|
|
.ok_or_else(|| "user key missing private key".to_owned())?
|
|
.replace("\\n", "\n");
|
|
let now = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.map_err(|error| error.to_string())?
|
|
.as_secs() as i64;
|
|
let encoding =
|
|
EncodingKey::from_rsa_pem(private_key.as_bytes()).map_err(|error| error.to_string())?;
|
|
let project_id = required_env("REMORA_OIDC_PROJECT_ID");
|
|
let host = issuer
|
|
.strip_prefix("http://")
|
|
.or_else(|| issuer.strip_prefix("https://"))
|
|
.unwrap_or(&issuer);
|
|
let scope = format!(
|
|
"openid profile \
|
|
urn:zitadel:iam:org:project:id:{project_id}:aud \
|
|
urn:zitadel:iam:org:project:roles \
|
|
urn:zitadel:iam:org:projects:roles \
|
|
urn:zitadel:iam:org:project:{project_id}:roles"
|
|
);
|
|
let client = Client::new();
|
|
let mut last_error = String::new();
|
|
for audience in [&token_url, &issuer] {
|
|
let mut header = Header::new(Algorithm::RS256);
|
|
header.kid = Some(key_id.clone());
|
|
let assertion = encode(
|
|
&header,
|
|
&json!({
|
|
"iss": subject,
|
|
"sub": subject,
|
|
"aud": audience,
|
|
"iat": now,
|
|
"exp": now + 3600
|
|
}),
|
|
&encoding,
|
|
)
|
|
.map_err(|error| error.to_string())?;
|
|
let response = client
|
|
.post(&token_url)
|
|
.header("Host", host)
|
|
.form(&[
|
|
("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
|
|
("assertion", assertion.as_str()),
|
|
("scope", scope.as_str()),
|
|
])
|
|
.send()
|
|
.await
|
|
.map_err(|error| error.to_string())?;
|
|
if response.status().is_success() {
|
|
return token_from_response(response).await;
|
|
}
|
|
last_error = format!(
|
|
"token endpoint: {} {}",
|
|
response.status(),
|
|
response.text().await.unwrap_or_default()
|
|
);
|
|
}
|
|
Err(last_error)
|
|
}
|
|
|
|
async fn token_from_response(response: reqwest::Response) -> Result<String, String> {
|
|
if !response.status().is_success() {
|
|
return Err(format!(
|
|
"token endpoint: {} {}",
|
|
response.status(),
|
|
response.text().await.unwrap_or_default()
|
|
));
|
|
}
|
|
let body: Value = response.json().await.map_err(|error| error.to_string())?;
|
|
body.get("access_token")
|
|
.and_then(Value::as_str)
|
|
.map(str::to_owned)
|
|
.ok_or_else(|| "token response missing access_token".into())
|
|
}
|
|
|
|
fn required_env(name: &str) -> String {
|
|
std::env::var(name).unwrap_or_else(|_| panic!("{name} is required"))
|
|
}
|
|
|
|
fn target_store() -> TargetStore {
|
|
let mut config = TargetStoreConfig::from_env().unwrap();
|
|
config.target_store_id = "target-main".into();
|
|
TargetStore::connect(config).unwrap()
|
|
}
|
|
|
|
fn opaque_id(prefix: &str) -> OpaqueId {
|
|
OpaqueId::parse(format!("{prefix}{}", nonce())).unwrap()
|
|
}
|
|
|
|
fn idempotency_key() -> String {
|
|
format!("key-{}-replay-safe", nonce())
|
|
}
|
|
|
|
fn nonce() -> String {
|
|
let mut bytes = [0_u8; 8];
|
|
getrandom::fill(&mut bytes).unwrap();
|
|
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
|
}
|