1011 lines
34 KiB
Rust
1011 lines
34 KiB
Rust
use std::{net::TcpListener, time::Duration};
|
|
|
|
use actix_web::{App, HttpServer, web};
|
|
use remora_admission::{AdmissionConfig, AdmissionService, migrate as admission_migrate};
|
|
use remora_api::{
|
|
PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,
|
|
migrate as api_migrate,
|
|
};
|
|
use remora_attempt::{AttemptConfig, AttemptService, migrate as attempt_migrate};
|
|
use remora_contracts::{
|
|
AttemptPageDto, DeletionOperationDto, JobPageDto, JobResourceDto, ResultDescriptorDto,
|
|
ResultMemberPageDto, SourceCredentialDto,
|
|
};
|
|
use remora_coordination::{
|
|
ActivateOptions, CoordinationStore, LeaseOwner, ProgressValue, StoreConfig,
|
|
};
|
|
use remora_credentials::{
|
|
CredentialRegistry, VaultClient, VaultConfig, migrate as credentials_migrate,
|
|
};
|
|
use remora_deletion::{
|
|
DeletionConfig, DeletionService, TargetRegistry, migrate as deletion_migrate,
|
|
};
|
|
use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
|
|
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
|
use remora_identity::{DefaultTarget, migrate as identity_migrate};
|
|
use remora_result::migrate as result_migrate;
|
|
use remora_target::{
|
|
CreateObjectRequest, TargetStore, TargetStoreConfig, attempt_manifest_key_in,
|
|
attempt_object_key_in,
|
|
};
|
|
use reqwest::{Client, StatusCode};
|
|
use serde_json::{Value, json};
|
|
use sqlx::PgPool;
|
|
use time::OffsetDateTime;
|
|
|
|
mod support;
|
|
|
|
use support::{mint_provider_token, oidc_provider, required_env, revoke_remora_grants};
|
|
|
|
struct Harness {
|
|
pool: PgPool,
|
|
coordination: CoordinationStore,
|
|
base: String,
|
|
client: Client,
|
|
token: String,
|
|
server: actix_web::dev::ServerHandle,
|
|
}
|
|
|
|
impl Harness {
|
|
async fn new() -> Option<Self> {
|
|
if std::env::var("REMORA_PUBLIC_API_REQUIRE").is_err() {
|
|
return None;
|
|
}
|
|
let pool = connect(
|
|
&required_env("DATABASE_URL"),
|
|
PoolConfig {
|
|
max_connections: 24,
|
|
acquire_timeout: Duration::from_secs(10),
|
|
idle_timeout: Some(Duration::from_secs(30)),
|
|
},
|
|
)
|
|
.await
|
|
.unwrap();
|
|
durable_migrate(&pool).await.unwrap();
|
|
identity_migrate(&pool).await.unwrap();
|
|
credentials_migrate(&pool).await.unwrap();
|
|
admission_migrate(&pool).await.unwrap();
|
|
attempt_migrate(&pool).await.unwrap();
|
|
result_migrate(&pool).await.unwrap();
|
|
deletion_migrate(&pool).await.unwrap();
|
|
api_migrate(&pool).await.unwrap();
|
|
sqlx::query("TRUNCATE TABLE jobs CASCADE")
|
|
.execute(&pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("TRUNCATE TABLE source_credentials CASCADE")
|
|
.execute(&pool)
|
|
.await
|
|
.unwrap();
|
|
|
|
let target = target_store();
|
|
let target_store_id =
|
|
TargetStoreId::parse(target.config().target_store_id.clone()).unwrap();
|
|
let bucket = BucketName::parse(target.config().bucket.clone()).unwrap();
|
|
let prefix =
|
|
TargetPrefix::parse(&required_env("REMORA_TARGET_PREFIX"), ["_remora"]).unwrap();
|
|
let admission = AdmissionService::new(
|
|
pool.clone(),
|
|
AdmissionConfig {
|
|
default_target: DefaultTarget {
|
|
store: target_store_id,
|
|
bucket,
|
|
prefix,
|
|
},
|
|
reserved_target_segments: vec!["_remora".into()],
|
|
policy_version: "public-api-v1".into(),
|
|
},
|
|
)
|
|
.unwrap();
|
|
let coordination = CoordinationStore::connect(
|
|
&required_env("REMORA_COORDINATION_URL"),
|
|
format!(
|
|
"{}wp21:{}:",
|
|
required_env("REMORA_COORDINATION_NAMESPACE"),
|
|
nonce()
|
|
),
|
|
StoreConfig {
|
|
global_limit: 16,
|
|
lease_ttl: Duration::from_secs(10),
|
|
reserve_ttl: Duration::from_secs(10),
|
|
progress_ttl: Duration::from_secs(10),
|
|
cancel_ttl: None,
|
|
},
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let attempts = AttemptService::new(
|
|
pool.clone(),
|
|
coordination.clone(),
|
|
AttemptConfig {
|
|
max_attempts: 3,
|
|
local_limit: 8,
|
|
},
|
|
)
|
|
.unwrap();
|
|
let vault = VaultClient::new(VaultConfig::from_env().unwrap()).unwrap();
|
|
let credentials = CredentialRegistry::new(pool.clone(), vault);
|
|
let targets = TargetRegistry::new([target]);
|
|
let deletion =
|
|
DeletionService::new(pool.clone(), targets.clone(), DeletionConfig::default()).unwrap();
|
|
let state = PublicApiState::new(
|
|
pool.clone(),
|
|
PublicApiServices {
|
|
identity: oidc_provider(),
|
|
admission,
|
|
attempts,
|
|
credentials,
|
|
deletion,
|
|
targets,
|
|
},
|
|
PublicApiConfig {
|
|
cursor_secret: b"public-api-fixture-cursor-secret".to_vec(),
|
|
member_access_ttl: Duration::from_secs(60),
|
|
event_retention: Duration::from_secs(60),
|
|
event_poll_interval: Duration::from_millis(50),
|
|
event_heartbeat_interval: Duration::from_secs(1),
|
|
event_ticket_ttl: Duration::from_secs(30),
|
|
event_allowed_origins: vec!["https://remora.test".into()],
|
|
},
|
|
)
|
|
.unwrap();
|
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let server = HttpServer::new(move || {
|
|
App::new()
|
|
.app_data(web::Data::new(state.clone()))
|
|
.configure(configure_public_api)
|
|
})
|
|
.listen(listener)
|
|
.unwrap()
|
|
.run();
|
|
let handle = server.handle();
|
|
tokio::spawn(server);
|
|
let token = mint_provider_token().await.unwrap();
|
|
Some(Self {
|
|
pool,
|
|
coordination,
|
|
base: format!("http://{address}/v1"),
|
|
client: Client::builder()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.build()
|
|
.unwrap(),
|
|
token,
|
|
server: handle,
|
|
})
|
|
}
|
|
|
|
fn request(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder {
|
|
self.request_with_token(method, path, &self.token)
|
|
}
|
|
|
|
fn request_with_token(
|
|
&self,
|
|
method: reqwest::Method,
|
|
path: &str,
|
|
token: &str,
|
|
) -> reqwest::RequestBuilder {
|
|
self.client
|
|
.request(method, format!("{}{}", self.base, path))
|
|
.bearer_auth(token)
|
|
}
|
|
}
|
|
|
|
impl Drop for Harness {
|
|
fn drop(&mut self) {
|
|
tokio::spawn(self.server.stop(true));
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn public_routes_are_authenticated_replayable_authorized_and_redacted() {
|
|
let Some(harness) = Harness::new().await else {
|
|
return;
|
|
};
|
|
let unauthenticated = harness
|
|
.client
|
|
.get(format!("{}/jobs", harness.base))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(unauthenticated.status(), StatusCode::UNAUTHORIZED);
|
|
assert_eq!(
|
|
unauthenticated.headers()["content-type"],
|
|
"application/problem+json"
|
|
);
|
|
|
|
let submission = json!({
|
|
"source": {
|
|
"kind": "http",
|
|
"url": "https://example.invalid/private?token=never-public",
|
|
"result_path": "artifact.bin"
|
|
},
|
|
"label": "public-api-test"
|
|
});
|
|
let submit_key = idempotency_key();
|
|
let submitted = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", &submit_key)
|
|
.json(&submission)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(submitted.status(), StatusCode::ACCEPTED);
|
|
assert!(submitted.headers().contains_key("idempotency-expires-at"));
|
|
let job: Value = submitted.json().await.unwrap();
|
|
serde_json::from_value::<JobResourceDto>(job.clone()).unwrap();
|
|
assert_eq!(job["resource_kind"], "job");
|
|
assert_eq!(job["phase"], "Queued");
|
|
let transcript = serde_json::to_string(&job).unwrap();
|
|
assert!(!transcript.contains("token=never-public"));
|
|
let job_id = job["id"].as_str().unwrap();
|
|
|
|
let replay = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", &submit_key)
|
|
.json(&submission)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(replay.status(), StatusCode::ACCEPTED);
|
|
assert_eq!(replay.json::<Value>().await.unwrap(), job);
|
|
let conflict = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", &submit_key)
|
|
.json(&json!({
|
|
"source": {
|
|
"kind": "http",
|
|
"url": "https://example.invalid/changed",
|
|
"result_path": "artifact.bin"
|
|
}
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(conflict.status(), StatusCode::CONFLICT);
|
|
|
|
let second = harness
|
|
.request(reqwest::Method::POST, "/jobs")
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.json(&json!({
|
|
"source": {
|
|
"kind": "http",
|
|
"url": "https://example.invalid/second",
|
|
"result_path": "artifact.bin"
|
|
},
|
|
"label": "second-public-api-test"
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(second.status(), StatusCode::ACCEPTED);
|
|
let second_job: Value = second.json().await.unwrap();
|
|
let second_job_id = second_job["id"].as_str().unwrap();
|
|
|
|
let listed = harness
|
|
.request(reqwest::Method::GET, "/jobs?limit=1")
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(listed.status(), StatusCode::OK);
|
|
let page: Value = listed.json().await.unwrap();
|
|
serde_json::from_value::<JobPageDto>(page.clone()).unwrap();
|
|
assert_eq!(
|
|
page["management_capabilities"]["revoke_execution_grants"],
|
|
true
|
|
);
|
|
let first_page_id = page["items"][0]["id"].as_str().unwrap();
|
|
assert!(page["event_cursor"].as_str().unwrap().len() >= 16);
|
|
let next_cursor = page["next_cursor"].as_str().unwrap();
|
|
let next_page = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?limit=1&cursor={next_cursor}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(next_page.status(), StatusCode::OK);
|
|
let next_page: Value = next_page.json().await.unwrap();
|
|
serde_json::from_value::<JobPageDto>(next_page.clone()).unwrap();
|
|
let second_page_id = next_page["items"][0]["id"].as_str().unwrap();
|
|
assert_ne!(first_page_id, second_page_id);
|
|
assert_eq!(
|
|
[first_page_id, second_page_id]
|
|
.into_iter()
|
|
.collect::<std::collections::BTreeSet<_>>(),
|
|
[job_id, second_job_id]
|
|
.into_iter()
|
|
.collect::<std::collections::BTreeSet<_>>()
|
|
);
|
|
let cross_filter = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?include_deleted=true&cursor={next_cursor}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(cross_filter.status(), StatusCode::BAD_REQUEST);
|
|
let altered_cursor = format!("{}x", page["event_cursor"].as_str().unwrap());
|
|
let invalid = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?cursor={altered_cursor}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(invalid.status(), StatusCode::BAD_REQUEST);
|
|
|
|
let fetched = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(fetched.status(), StatusCode::OK);
|
|
assert_eq!(fetched.json::<Value>().await.unwrap()["id"], job_id);
|
|
|
|
install_mutation_record_failure(&harness.pool).await;
|
|
let rolled_back = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(rolled_back.status(), StatusCode::SERVICE_UNAVAILABLE);
|
|
let phase: String = sqlx::query_scalar("SELECT phase FROM jobs WHERE job_id = $1")
|
|
.bind(job_id)
|
|
.fetch_one(&harness.pool)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(phase, "queued");
|
|
remove_mutation_record_failure(&harness.pool).await;
|
|
|
|
let cancel_key = idempotency_key();
|
|
let cancelled = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
|
|
.header("Idempotency-Key", &cancel_key)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(cancelled.status(), StatusCode::ACCEPTED);
|
|
let cancel_projection: Value = cancelled.json().await.unwrap();
|
|
let cancel_replay = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/cancel"))
|
|
.header("Idempotency-Key", &cancel_key)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(
|
|
cancel_replay.json::<Value>().await.unwrap(),
|
|
cancel_projection
|
|
);
|
|
|
|
let credential_key = idempotency_key();
|
|
let secret = "synthetic-secret-never-return";
|
|
let created = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &credential_key)
|
|
.json(&json!({
|
|
"kind": "bearer_token",
|
|
"label": "fixture credential",
|
|
"material": { "token": secret }
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(created.status(), StatusCode::CREATED);
|
|
let credential: Value = created.json().await.unwrap();
|
|
serde_json::from_value::<SourceCredentialDto>(credential.clone()).unwrap();
|
|
assert!(!serde_json::to_string(&credential).unwrap().contains(secret));
|
|
let credential_replay = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &credential_key)
|
|
.json(&json!({
|
|
"kind": "bearer_token",
|
|
"label": "fixture credential",
|
|
"material": { "token": secret }
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(credential_replay.json::<Value>().await.unwrap(), credential);
|
|
let credential_conflict = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &credential_key)
|
|
.json(&json!({
|
|
"kind": "bearer_token",
|
|
"label": "changed fixture credential",
|
|
"material": { "token": secret }
|
|
}))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(credential_conflict.status(), StatusCode::CONFLICT);
|
|
let credential_id = credential["id"].as_str().unwrap();
|
|
|
|
let credential_page = harness
|
|
.request(reqwest::Method::GET, "/source-credentials?limit=1")
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(credential_page.status(), StatusCode::OK);
|
|
assert_eq!(
|
|
credential_page.json::<Value>().await.unwrap()["items"][0]["id"],
|
|
credential_id
|
|
);
|
|
let fetched_credential = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/source-credentials/{credential_id}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(fetched_credential.status(), StatusCode::OK);
|
|
let rotated = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{credential_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.json(&json!({ "material": { "token": "rotated-secret" } }))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(rotated.status(), StatusCode::CREATED);
|
|
|
|
let hugging_face_key = idempotency_key();
|
|
let hugging_face_request = json!({
|
|
"kind": "hugging_face",
|
|
"label": "fixture Hugging Face credential",
|
|
"material": { "token": "synthetic-hugging-face-token" }
|
|
});
|
|
let lost_create_response = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &hugging_face_key)
|
|
.json(&hugging_face_request)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(lost_create_response.status(), StatusCode::CREATED);
|
|
drop(lost_create_response);
|
|
let hugging_face = harness
|
|
.request(reqwest::Method::POST, "/source-credentials")
|
|
.header("Idempotency-Key", &hugging_face_key)
|
|
.json(&hugging_face_request)
|
|
.send()
|
|
.await
|
|
.unwrap()
|
|
.json::<Value>()
|
|
.await
|
|
.unwrap();
|
|
serde_json::from_value::<SourceCredentialDto>(hugging_face.clone()).unwrap();
|
|
assert_eq!(hugging_face["kind"], "hugging_face");
|
|
let hugging_face_id = hugging_face["id"].as_str().unwrap();
|
|
let rotate_key = idempotency_key();
|
|
let rotate_request = json!({ "material": { "token": "rotated-hugging-face-token" } });
|
|
let lost_rotate_response = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{hugging_face_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", &rotate_key)
|
|
.json(&rotate_request)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(lost_rotate_response.status(), StatusCode::CREATED);
|
|
drop(lost_rotate_response);
|
|
let hugging_face_rotated = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{hugging_face_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", &rotate_key)
|
|
.json(&rotate_request)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hugging_face_rotated.status(), StatusCode::CREATED);
|
|
let hugging_face_rotated: Value = hugging_face_rotated.json().await.unwrap();
|
|
serde_json::from_value::<SourceCredentialDto>(hugging_face_rotated.clone()).unwrap();
|
|
assert_eq!(hugging_face_rotated["kind"], "hugging_face");
|
|
let rotate_conflict = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{hugging_face_id}/versions"),
|
|
)
|
|
.header("Idempotency-Key", &rotate_key)
|
|
.json(&json!({ "material": { "token": "changed-token" } }))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(rotate_conflict.status(), StatusCode::CONFLICT);
|
|
|
|
let revoked = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/source-credentials/{credential_id}/revoke"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(revoked.status(), StatusCode::OK);
|
|
assert_eq!(revoked.json::<Value>().await.unwrap()["status"], "revoked");
|
|
|
|
let (attempt_id, worker_id) = seed_committed_result(&harness.pool, job_id).await;
|
|
seed_failed_attempt(&harness.pool, job_id).await;
|
|
let owner = LeaseOwner::new(attempt_id.as_str(), 1, worker_id.as_str()).unwrap();
|
|
harness.coordination.enqueue(job_id).await.unwrap();
|
|
harness
|
|
.coordination
|
|
.activate(job_id, &owner, ActivateOptions::default())
|
|
.await
|
|
.unwrap();
|
|
harness
|
|
.coordination
|
|
.report_progress_value(
|
|
job_id,
|
|
&owner,
|
|
ProgressValue {
|
|
completed_bytes: 7,
|
|
total_bytes: Some(7),
|
|
completed_members: 1,
|
|
total_members: Some(1),
|
|
},
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let with_progress = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap()
|
|
.json::<Value>()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(with_progress["progress"]["completed_bytes"], 7);
|
|
assert_eq!(with_progress["progress"]["attempt_id"], attempt_id.as_str());
|
|
harness.coordination.release(job_id, &owner).await.unwrap();
|
|
let foreign = LeaseOwner::new(
|
|
opaque_id("attempt").as_str(),
|
|
2,
|
|
opaque_id("worker").as_str(),
|
|
)
|
|
.unwrap();
|
|
harness
|
|
.coordination
|
|
.activate(job_id, &foreign, ActivateOptions::default())
|
|
.await
|
|
.unwrap();
|
|
let stale_progress = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap()
|
|
.json::<Value>()
|
|
.await
|
|
.unwrap();
|
|
assert!(stale_progress["progress"].is_null());
|
|
let result = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}/result"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(result.status(), StatusCode::OK);
|
|
let result = result.json::<Value>().await.unwrap();
|
|
serde_json::from_value::<ResultDescriptorDto>(result.clone()).unwrap();
|
|
assert_eq!(result["kind"], "manifest");
|
|
|
|
let members = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{job_id}/result/members"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(members.status(), StatusCode::OK);
|
|
let members: Value = members.json().await.unwrap();
|
|
serde_json::from_value::<ResultMemberPageDto>(members.clone()).unwrap();
|
|
let member_id = members["items"][0]["id"].as_str().unwrap();
|
|
let access = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{job_id}/result/members/{member_id}/content"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(access.status(), StatusCode::OK);
|
|
assert_eq!(access.headers()["cache-control"], "no-store");
|
|
assert_eq!(access.headers()["pragma"], "no-cache");
|
|
let access: Value = access.json().await.unwrap();
|
|
let access_url = reqwest::Url::parse(access["url"].as_str().unwrap()).unwrap();
|
|
assert!(matches!(access_url.scheme(), "http" | "https"));
|
|
assert!(access["expires_at"].is_string());
|
|
|
|
let attempts = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}/attempts"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(attempts.status(), StatusCode::OK);
|
|
let attempts = attempts.json::<Value>().await.unwrap();
|
|
serde_json::from_value::<AttemptPageDto>(attempts.clone()).unwrap();
|
|
let failed_attempt = attempts["items"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.find(|attempt| !attempt["failure"].is_null())
|
|
.unwrap();
|
|
assert_eq!(failed_attempt["failure"]["class"], "internal");
|
|
assert_eq!(failed_attempt["failure"]["message"], "job failed");
|
|
assert!(
|
|
!serde_json::to_string(failed_attempt)
|
|
.unwrap()
|
|
.contains("database password")
|
|
);
|
|
|
|
let revoked_job = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/revoke"))
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(revoked_job.status(), StatusCode::ACCEPTED);
|
|
|
|
let soft_deleted = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/jobs/{job_id}/soft-delete"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(soft_deleted.status(), StatusCode::OK);
|
|
let hidden = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}/result"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hidden.status(), StatusCode::NOT_FOUND);
|
|
let restored = harness
|
|
.request(reqwest::Method::POST, &format!("/jobs/{job_id}/restore"))
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(restored.status(), StatusCode::OK);
|
|
|
|
let hard_deleted = harness
|
|
.request(
|
|
reqwest::Method::POST,
|
|
&format!("/jobs/{job_id}/hard-delete"),
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hard_deleted.status(), StatusCode::ACCEPTED);
|
|
let operation: Value = hard_deleted.json().await.unwrap();
|
|
serde_json::from_value::<DeletionOperationDto>(operation.clone()).unwrap();
|
|
let operation_id = operation["id"].as_str().unwrap();
|
|
let fetched_operation = harness
|
|
.request(reqwest::Method::GET, &format!("/deletions/{operation_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(fetched_operation.status(), StatusCode::OK);
|
|
let tombstone = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(tombstone.status(), StatusCode::OK);
|
|
let tombstone = tombstone.json::<Value>().await.unwrap();
|
|
serde_json::from_value::<JobResourceDto>(tombstone.clone()).unwrap();
|
|
assert_eq!(tombstone["resource_kind"], "tombstone");
|
|
|
|
let peer_owner = opaque_id("peer-principal");
|
|
sqlx::query("UPDATE jobs SET owner_id = $2 WHERE job_id = $1")
|
|
.bind(second_job_id)
|
|
.bind(peer_owner.as_str())
|
|
.execute(&harness.pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("UPDATE source_credentials SET owner_id = $2 WHERE credential_id = $1")
|
|
.bind(hugging_face_id)
|
|
.bind(peer_owner.as_str())
|
|
.execute(&harness.pool)
|
|
.await
|
|
.unwrap();
|
|
let admin_peer_job = harness
|
|
.request(reqwest::Method::GET, &format!("/jobs/{second_job_id}"))
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(admin_peer_job.status(), StatusCode::OK);
|
|
let admin_peer_credential = harness
|
|
.request(
|
|
reqwest::Method::GET,
|
|
&format!("/source-credentials/{hugging_face_id}"),
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(admin_peer_credential.status(), StatusCode::OK);
|
|
|
|
revoke_remora_grants().await.unwrap();
|
|
let unprivileged_token = mint_provider_token().await.unwrap();
|
|
let lost_scope = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{job_id}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(lost_scope.status(), StatusCode::NOT_FOUND);
|
|
let unprivileged_page = harness
|
|
.request_with_token(reqwest::Method::GET, "/jobs", &unprivileged_token)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(unprivileged_page.status(), StatusCode::OK);
|
|
assert_eq!(
|
|
unprivileged_page.json::<Value>().await.unwrap()["management_capabilities"]["revoke_execution_grants"],
|
|
false
|
|
);
|
|
let hidden_submit_replay = harness
|
|
.request_with_token(reqwest::Method::POST, "/jobs", &unprivileged_token)
|
|
.header("Idempotency-Key", &submit_key)
|
|
.json(&submission)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hidden_submit_replay.status(), StatusCode::NOT_FOUND);
|
|
let hidden_peer_job = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs/{second_job_id}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hidden_peer_job.status(), StatusCode::NOT_FOUND);
|
|
let hidden_peer_credential = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/source-credentials/{hugging_face_id}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(hidden_peer_credential.status(), StatusCode::NOT_FOUND);
|
|
let invalidated_cursor = harness
|
|
.request_with_token(
|
|
reqwest::Method::GET,
|
|
&format!("/jobs?limit=1&cursor={next_cursor}"),
|
|
&unprivileged_token,
|
|
)
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(invalidated_cursor.status(), StatusCode::BAD_REQUEST);
|
|
let forbidden_revoke = harness
|
|
.request_with_token(
|
|
reqwest::Method::POST,
|
|
&format!("/jobs/{second_job_id}/revoke"),
|
|
&unprivileged_token,
|
|
)
|
|
.header("Idempotency-Key", idempotency_key())
|
|
.send()
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(forbidden_revoke.status(), StatusCode::FORBIDDEN);
|
|
}
|
|
|
|
async fn seed_committed_result(pool: &PgPool, job_id: &str) -> (OpaqueId, OpaqueId) {
|
|
let attempt_id = opaque_id("attempt");
|
|
let worker_id = opaque_id("worker");
|
|
let manifest_id = opaque_id("manifest");
|
|
let member_id = opaque_id("member");
|
|
let object_id = opaque_id("object");
|
|
let prefix = TargetPrefix::parse(
|
|
&required_env("REMORA_TARGET_PREFIX"),
|
|
std::iter::empty::<&str>(),
|
|
)
|
|
.unwrap()
|
|
.canonical();
|
|
let member_key = attempt_object_key_in(&prefix, &attempt_id, &object_id, Some("artifact.bin"));
|
|
let manifest_key = attempt_manifest_key_in(&prefix, &attempt_id, &manifest_id);
|
|
let target = target_store();
|
|
let member = target
|
|
.put_create_only(
|
|
&CreateObjectRequest::for_target_attempt_object(
|
|
&prefix,
|
|
&attempt_id,
|
|
&object_id,
|
|
Some("artifact.bin"),
|
|
),
|
|
b"payload".to_vec(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let manifest = target
|
|
.put_create_only(
|
|
&CreateObjectRequest::for_target_attempt_manifest(&prefix, &attempt_id, &manifest_id),
|
|
b"manifest".to_vec(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let now = OffsetDateTime::now_utc();
|
|
sqlx::query(
|
|
"INSERT INTO attempts (
|
|
attempt_id, job_id, fence, worker_incarnation, state, initial_progress,
|
|
authorized_at, executing_at, committed_at, staging_origin_attempt_id
|
|
) VALUES ($1, $2, 1, $3, 'committed', 0, $4, $4, $4, $1)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(job_id)
|
|
.bind(worker_id.as_str())
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"INSERT INTO result_inventories (
|
|
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
|
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
|
manifest_size, member_count, total_bytes, staging_origin_attempt_id, target_prefix
|
|
) VALUES ($1, 'sealed', 1, $2, $3, $4, $5, $6, $7, $8, 1, 7, $1, $9)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(manifest_id.as_str())
|
|
.bind(&manifest.identity.target_store_id)
|
|
.bind(&manifest.identity.bucket)
|
|
.bind(&manifest_key)
|
|
.bind(&manifest.identity.version_id)
|
|
.bind(manifest.content_sha256.as_str())
|
|
.bind(i64::try_from(manifest.size).unwrap())
|
|
.bind(&prefix)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"INSERT INTO result_inventory_members (
|
|
attempt_id, member_id, path, size, object_store, object_bucket,
|
|
object_key, object_version_id, object_sha256
|
|
) VALUES ($1, $2, 'artifact.bin', 7, $3, $4, $5, $6, $7)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(member_id.as_str())
|
|
.bind(&member.identity.target_store_id)
|
|
.bind(&member.identity.bucket)
|
|
.bind(&member_key)
|
|
.bind(&member.identity.version_id)
|
|
.bind(member.content_sha256.as_str())
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
let committed = json!({
|
|
"attempt_id": attempt_id,
|
|
"manifest_id": manifest_id,
|
|
"schema_version": 1,
|
|
"manifest": {
|
|
"store": manifest.identity.target_store_id,
|
|
"bucket": manifest.identity.bucket,
|
|
"key": manifest.identity.key,
|
|
"version_id": manifest.identity.version_id,
|
|
"sha256": manifest.content_sha256.as_str(),
|
|
"size": manifest.size
|
|
},
|
|
"member_count": 1,
|
|
"total_bytes": 7
|
|
});
|
|
sqlx::query(
|
|
"UPDATE jobs
|
|
SET phase = 'succeeded', lifecycle = 'visible', revision = revision + 1,
|
|
current_fence = 1, authority_attempt_id = $2, committed_result = $3,
|
|
terminal_at = $4, updated_at = $4
|
|
WHERE job_id = $1",
|
|
)
|
|
.bind(job_id)
|
|
.bind(attempt_id.as_str())
|
|
.bind(committed)
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
(attempt_id, worker_id)
|
|
}
|
|
|
|
async fn seed_failed_attempt(pool: &PgPool, job_id: &str) {
|
|
let attempt_id = opaque_id("failed-attempt");
|
|
let worker_id = opaque_id("failed-worker");
|
|
let now = OffsetDateTime::now_utc();
|
|
sqlx::query(
|
|
"INSERT INTO attempts (
|
|
attempt_id, job_id, fence, worker_incarnation, state, initial_progress,
|
|
failure, authorized_at, executing_at, stopped_at, staging_origin_attempt_id
|
|
) VALUES ($1, $2, 2, $3, 'stopped', 0, $4, $5, $5, $5, $1)",
|
|
)
|
|
.bind(attempt_id.as_str())
|
|
.bind(job_id)
|
|
.bind(worker_id.as_str())
|
|
.bind(json!({
|
|
"class": "internal",
|
|
"disposition": "retryable",
|
|
"details": {
|
|
"exception": "database password synthetic-secret-never-return"
|
|
}
|
|
}))
|
|
.bind(now)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
async fn install_mutation_record_failure(pool: &PgPool) {
|
|
sqlx::query(
|
|
"CREATE OR REPLACE FUNCTION remora_test_fail_cancel_record()
|
|
RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN
|
|
IF NEW.scope_operation = 'job.cancel' THEN
|
|
RAISE EXCEPTION 'synthetic mutation record failure';
|
|
END IF;
|
|
RETURN NEW;
|
|
END
|
|
$$",
|
|
)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query(
|
|
"CREATE TRIGGER remora_test_fail_cancel_record
|
|
BEFORE INSERT ON mutation_records
|
|
FOR EACH ROW EXECUTE FUNCTION remora_test_fail_cancel_record()",
|
|
)
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
async fn remove_mutation_record_failure(pool: &PgPool) {
|
|
sqlx::query("DROP TRIGGER remora_test_fail_cancel_record ON mutation_records")
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
sqlx::query("DROP FUNCTION remora_test_fail_cancel_record()")
|
|
.execute(pool)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
fn target_store() -> TargetStore {
|
|
let mut config = TargetStoreConfig::from_env().unwrap();
|
|
config.target_store_id = "target-main".into();
|
|
TargetStore::connect(config).unwrap()
|
|
}
|
|
|
|
fn opaque_id(prefix: &str) -> OpaqueId {
|
|
OpaqueId::parse(format!("{prefix}{}", nonce())).unwrap()
|
|
}
|
|
|
|
fn idempotency_key() -> String {
|
|
format!("key-{}-replay-safe", nonce())
|
|
}
|
|
|
|
fn nonce() -> String {
|
|
let mut bytes = [0_u8; 8];
|
|
getrandom::fill(&mut bytes).unwrap();
|
|
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
|
}
|