Centralize durable schema migration ownership

Cargo.lock

@@ -3264,6 +3264,7 @@ dependencies = [
  "remora-domain",
  "remora-durable",
  "remora-identity",
+ "remora-schema",
  "serde",
  "serde_json",
  "sqlx",
@@ -3296,6 +3297,7 @@ dependencies = [
  "remora-identity",
  "remora-result",
  "remora-runtime",
+ "remora-schema",
  "remora-target",
  "reqwest 0.13.4",
  "rustls",
@@ -3324,6 +3326,7 @@ dependencies = [
  "remora-durable",
  "remora-identity",
  "remora-result",
+ "remora-schema",
  "remora-target",
  "serde",
  "serde_json",
@@ -3363,6 +3366,7 @@ dependencies = [
  "getrandom 0.3.4",
  "remora-domain",
  "remora-durable",
+ "remora-schema",
  "reqwest 0.12.28",
  "serde",
  "serde_json",
@@ -3386,6 +3390,7 @@ dependencies = [
  "remora-durable",
  "remora-identity",
  "remora-result",
+ "remora-schema",
  "remora-stream",
  "remora-target",
  "serde",
@@ -3415,6 +3420,7 @@ dependencies = [
  "base64",
  "hmac",
  "remora-domain",
+ "remora-schema",
  "serde",
  "serde_json",
  "sha2 0.11.0",
@@ -3475,6 +3481,7 @@ dependencies = [
  "rand 0.8.7",
  "remora-domain",
  "remora-durable",
+ "remora-schema",
  "reqwest 0.12.28",
  "rsa",
  "serde",
@@ -3524,6 +3531,7 @@ dependencies = [
  "futures-util",
  "remora-domain",
  "remora-durable",
+ "remora-schema",
  "remora-stream",
  "remora-target",
  "serde",
@@ -3547,6 +3555,15 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "remora-schema"
+version = "0.0.0"
+dependencies = [
+ "sqlx",
+ "thiserror",
+ "tokio",
+]
+
 [[package]]
 name = "remora-stream"
 version = "0.0.0"
@@ -3609,6 +3626,7 @@ dependencies = [
  "remora-outbound",
  "remora-result",
  "remora-runtime",
+ "remora-schema",
  "remora-stream",
  "remora-target",
  "reqwest 0.13.4",

Cargo.toml

@@ -16,6 +16,7 @@ members = [
     "crates/remora-outbound",
     "crates/remora-result",
     "crates/remora-runtime",
+    "crates/remora-schema",
     "crates/remora-stream",
     "crates/remora-target",
     "crates/remora-worker",
@@ -33,6 +34,7 @@ hmac = "0.13.0"
 hf-hub = "=1.0.0"
 remora-domain = { path = "crates/remora-domain" }
 remora-durable = { path = "crates/remora-durable" }
+remora-schema = { path = "crates/remora-schema" }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.150"
 serde_json_canonicalizer = "0.3.2"

crates/remora-admission/Cargo.toml

@@ -20,4 +20,5 @@ time.workspace = true
 url = "2.5.8"
 
 [dev-dependencies]
+remora-schema.workspace = true
 tokio.workspace = true

crates/remora-admission/src/lib.rs

@@ -1,10 +1,8 @@
 mod error;
-mod migrate;
 mod service;
 mod source;
 mod types;
 
 pub use error::AdmissionError;
-pub use migrate::{migrate, migrator};
 pub use service::{AdmissionConfig, AdmissionService};
 pub use types::{AcceptedJob, AdmissionDecision, AdmissionRejection, SubmitOutcome};

crates/remora-admission/src/migrate.rs

@@ -1,17 +0,0 @@
-use remora_durable::DurableError;
-use sqlx::PgPool;
-
-const MIGRATION_TABLE: &str = "_remora_admission_migrations";
-
-pub async fn migrate(pool: &PgPool) -> Result<(), DurableError> {
-    let mut migrator = sqlx::migrate!("./migrations");
-    migrator.dangerous_set_table_name(MIGRATION_TABLE);
-    migrator.run(pool).await?;
-    Ok(())
-}
-
-pub fn migrator() -> sqlx::migrate::Migrator {
-    let mut migrator = sqlx::migrate!("./migrations");
-    migrator.dangerous_set_table_name(MIGRATION_TABLE);
-    migrator
-}

crates/remora-admission/tests/substrate.rs

@@ -1,20 +1,21 @@
 use std::time::Duration;
 
 use remora_admission::{
-    AdmissionConfig, AdmissionDecision, AdmissionError, AdmissionService, SubmitOutcome, migrate,
+    AdmissionConfig, AdmissionDecision, AdmissionError, AdmissionService, SubmitOutcome,
 };
 use remora_contracts::{
     HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto, TargetOverrideDto,
 };
 use remora_credentials::{
     CredentialError, CredentialGrantVersion, CredentialRegistry, GrantBoundResolver, VaultClient,
-    VaultConfig, migrate as credentials_migrate,
+    VaultConfig,
 };
 use remora_domain::{
     BucketName, OpaqueId, StorageScope, TargetPrefix, TargetStoreId, ports::ActiveCredentialGrant,
 };
-use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
+use remora_durable::{MutationDisposition, PoolConfig, connect};
 use remora_identity::{DefaultTarget, Principal};
+use remora_schema::migrate as migrate_schema;
 use sqlx::PgPool;
 
 fn database_url() -> Option<String> {
@@ -40,11 +41,7 @@ async fn pool() -> Option<PgPool> {
     )
     .await
     .expect("connect postgres");
-    durable_migrate(&pool).await.expect("durable migrate");
+    migrate_schema(&pool).await.expect("schema migrate");
-    credentials_migrate(&pool)
-        .await
-        .expect("credentials migrate");
-    migrate(&pool).await.expect("admission migrate");
     Some(pool)
 }
 

crates/remora-api/Cargo.toml

@@ -22,6 +22,7 @@ remora-durable.workspace = true
 remora-identity = { path = "../remora-identity" }
 remora-result = { path = "../remora-result" }
 remora-runtime = { path = "../remora-runtime" }
+remora-schema.workspace = true
 remora-target = { path = "../remora-target" }
 rustls = "=0.23.42"
 rustls-pemfile = "=2.2.0"

crates/remora-api/src/application.rs

@@ -291,22 +291,7 @@ fn admission_service(
 }
 
 async fn migrate_all(pool: &PgPool) -> Result<(), RuntimeError> {
-    remora_durable::migrate(pool).await.map_err(runtime_error)?;
+    remora_schema::migrate(pool).await.map_err(runtime_error)
-    remora_identity::migrate(pool)
-        .await
-        .map_err(runtime_error)?;
-    remora_credentials::migrate(pool)
-        .await
-        .map_err(runtime_error)?;
-    remora_admission::migrate(pool)
-        .await
-        .map_err(runtime_error)?;
-    remora_attempt::migrate(pool).await.map_err(runtime_error)?;
-    remora_result::migrate(pool).await.map_err(runtime_error)?;
-    remora_deletion::migrate(pool)
-        .await
-        .map_err(runtime_error)?;
-    crate::migrate(pool).await.map_err(runtime_error)
 }
 
 fn start_maintenance(

crates/remora-api/src/lib.rs

@@ -1,10 +1,8 @@
 mod application;
-mod migrate;
 mod public;
 mod worker;
 
 pub use application::{ApiApplication, ApiApplicationConfig};
-pub use migrate::{migrate, migrator};
 pub use public::{
     PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,
     prune_live_event_history,

crates/remora-api/src/migrate.rs

@@ -1,16 +0,0 @@
-use remora_durable::DurableError;
-use sqlx::PgPool;
-
-const MIGRATION_TABLE: &str = "_remora_api_migrations";
-
-pub async fn migrate(pool: &PgPool) -> Result<(), DurableError> {
-    let mut migrator = sqlx::migrate!("./migrations");
-    migrator.dangerous_set_table_name(MIGRATION_TABLE);
-    migrator.run(pool).await.map_err(DurableError::from)
-}
-
-pub fn migrator() -> sqlx::migrate::Migrator {
-    let mut migrator = sqlx::migrate!("./migrations");
-    migrator.dangerous_set_table_name(MIGRATION_TABLE);
-    migrator
-}

crates/remora-api/tests/live_events.rs

@@ -2,26 +2,22 @@ use std::{net::TcpListener, time::Duration};
 
 use actix_web::{App, HttpServer, web};
 use futures_util::{SinkExt as _, StreamExt as _};
-use remora_admission::{AdmissionConfig, AdmissionService, migrate as admission_migrate};
+use remora_admission::{AdmissionConfig, AdmissionService};
 use remora_api::{
     PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,
-    migrate as api_migrate, prune_live_event_history,
+    prune_live_event_history,
 };
-use remora_attempt::{AttemptConfig, AttemptService, migrate as attempt_migrate};
+use remora_attempt::{AttemptConfig, AttemptService};
 use remora_contracts::{EventFrameDto, ResyncReasonDto};
 use remora_coordination::{
     ActivateOptions, CoordinationStore, LeaseOwner, ProgressValue, StoreConfig,
 };
-use remora_credentials::{
+use remora_credentials::{CredentialRegistry, VaultClient, VaultConfig};
-    CredentialRegistry, VaultClient, VaultConfig, migrate as credentials_migrate,
+use remora_deletion::{DeletionConfig, DeletionService, TargetRegistry};
-};
-use remora_deletion::{
-    DeletionConfig, DeletionService, TargetRegistry, migrate as deletion_migrate,
-};
 use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
-use remora_durable::{PoolConfig, connect, enqueue, migrate as durable_migrate};
+use remora_durable::{PoolConfig, connect, enqueue};
-use remora_identity::{DefaultTarget, migrate as identity_migrate};
+use remora_identity::DefaultTarget;
-use remora_result::migrate as result_migrate;
+use remora_schema::migrate as migrate_schema;
 use reqwest::{Client, StatusCode};
 use serde_json::{Value, json};
 use sqlx::PgPool;
@@ -73,14 +69,7 @@ impl Harness {
         )
         .await
         .unwrap();
-        durable_migrate(&pool).await.unwrap();
+        migrate_schema(&pool).await.unwrap();
-        identity_migrate(&pool).await.unwrap();
-        credentials_migrate(&pool).await.unwrap();
-        admission_migrate(&pool).await.unwrap();
-        attempt_migrate(&pool).await.unwrap();
-        result_migrate(&pool).await.unwrap();
-        deletion_migrate(&pool).await.unwrap();
-        api_migrate(&pool).await.unwrap();
         sqlx::query("TRUNCATE TABLE jobs CASCADE")
             .execute(&pool)
             .await

crates/remora-api/tests/public_api.rs

@@ -1,12 +1,9 @@
 use std::{net::TcpListener, time::Duration};
 
 use actix_web::{App, HttpServer, web};
-use remora_admission::{AdmissionConfig, AdmissionService, migrate as admission_migrate};
+use remora_admission::{AdmissionConfig, AdmissionService};
-use remora_api::{
+use remora_api::{PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api};
-    PublicApiConfig, PublicApiServices, PublicApiState, configure_public_api,
+use remora_attempt::{AttemptConfig, AttemptService};
-    migrate as api_migrate,
-};
-use remora_attempt::{AttemptConfig, AttemptService, migrate as attempt_migrate};
 use remora_contracts::{
     AttemptPageDto, DeletionOperationDto, JobPageDto, JobResourceDto, ResultDescriptorDto,
     ResultMemberPageDto, SourceCredentialDto,
@@ -14,16 +11,12 @@ use remora_contracts::{
 use remora_coordination::{
     ActivateOptions, CoordinationStore, LeaseOwner, ProgressValue, StoreConfig,
 };
-use remora_credentials::{
+use remora_credentials::{CredentialRegistry, VaultClient, VaultConfig};
-    CredentialRegistry, VaultClient, VaultConfig, migrate as credentials_migrate,
+use remora_deletion::{DeletionConfig, DeletionService, TargetRegistry};
-};
-use remora_deletion::{
-    DeletionConfig, DeletionService, TargetRegistry, migrate as deletion_migrate,
-};
 use remora_domain::{BucketName, OpaqueId, TargetPrefix, TargetStoreId};
-use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
+use remora_durable::{PoolConfig, connect};
-use remora_identity::{DefaultTarget, migrate as identity_migrate};
+use remora_identity::DefaultTarget;
-use remora_result::migrate as result_migrate;
+use remora_schema::migrate as migrate_schema;
 use remora_target::{
     CreateObjectRequest, TargetStore, TargetStoreConfig, attempt_manifest_key_in,
     attempt_object_key_in,
@@ -64,14 +57,7 @@ impl Harness {
         )
         .await
         .unwrap();
-        durable_migrate(&pool).await.unwrap();
+        migrate_schema(&pool).await.unwrap();
-        identity_migrate(&pool).await.unwrap();
-        credentials_migrate(&pool).await.unwrap();
-        admission_migrate(&pool).await.unwrap();
-        attempt_migrate(&pool).await.unwrap();
-        result_migrate(&pool).await.unwrap();
-        deletion_migrate(&pool).await.unwrap();
-        api_migrate(&pool).await.unwrap();
         sqlx::query("TRUNCATE TABLE jobs CASCADE")
             .execute(&pool)
             .await

crates/remora-api/tests/worker_api.rs

@@ -5,13 +5,11 @@ use rcgen::{
     BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer,
     KeyPair, KeyUsagePurpose,
 };
-use remora_admission::{
+use remora_admission::{AdmissionConfig, AdmissionDecision, AdmissionService};
-    AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
-};
 use remora_api::{
     AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
 };
-use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig, migrate as attempt_migrate};
+use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig};
 use remora_contracts::{
     ApiErrorDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto, AttemptFailureOutcomeDto,
     AttemptOutcomeDto, AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
@@ -24,13 +22,12 @@ use remora_contracts::{
     StagingMultipartIdentityDto, StagingObjectIdentityDto, StagingWriteIntentDto,
 };
 use remora_coordination::{CoordinationStore, StoreConfig};
-use remora_credentials::migrate as credentials_migrate;
 use remora_domain::{
     BucketName, JobPhase, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
 };
-use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
+use remora_durable::{PoolConfig, connect};
 use remora_identity::{DefaultTarget, Principal};
-use remora_result::migrate as result_migrate;
+use remora_schema::migrate as migrate_schema;
 use remora_target::attempt_manifest_key_in;
 use reqwest::{Certificate as ClientCertificate, Client, Identity};
 use rustls::{
@@ -59,11 +56,7 @@ impl Harness {
         )
         .await
         .unwrap();
-        durable_migrate(&pool).await.unwrap();
+        migrate_schema(&pool).await.unwrap();
-        credentials_migrate(&pool).await.unwrap();
-        admission_migrate(&pool).await.unwrap();
-        attempt_migrate(&pool).await.unwrap();
-        result_migrate(&pool).await.unwrap();
         sqlx::query("TRUNCATE TABLE jobs CASCADE")
             .execute(&pool)
             .await

crates/remora-attempt/Cargo.toml

@@ -23,5 +23,6 @@ remora-admission = { path = "../remora-admission" }
 remora-contracts = { path = "../remora-contracts" }
 remora-credentials = { path = "../remora-credentials" }
 remora-identity = { path = "../remora-identity" }
+remora-schema.workspace = true
 remora-target = { path = "../remora-target", features = ["test-utils"] }
 tokio = { workspace = true, features = ["io-util", "net"] }

crates/remora-attempt/src/lib.rs

@@ -1,5 +1,4 @@
 mod error;
-mod migrate;
 mod mutation_types;
 mod mutations;
 mod recovery;
@@ -8,7 +7,6 @@ mod service;
 mod types;
 
 pub use error::AttemptError;
-pub use migrate::migrate;
 pub use mutation_types::{
     AttemptMutationOutcome, CheckpointMultipart, CheckpointProjection, CheckpointRecord,
     CommittedManifestRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,

crates/remora-attempt/src/migrate.rs

@@ -1,12 +0,0 @@
-use sqlx::PgPool;
-
-use crate::AttemptError;
-
-pub async fn migrate(pool: &PgPool) -> Result<(), AttemptError> {
-    let mut migrator = sqlx::migrate!("./migrations");
-    migrator.dangerous_set_table_name("_remora_attempt_migrations");
-    migrator
-        .run(pool)
-        .await
-        .map_err(|error| AttemptError::Durable(remora_durable::DurableError::Migrate(error)))
-}

crates/remora-attempt/tests/authority.rs

@@ -2,19 +2,17 @@ use std::collections::HashSet;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use remora_admission::{
+use remora_admission::{AdmissionConfig, AdmissionDecision, AdmissionService};
-    AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
-};
 use remora_attempt::{
     ActivationOutcome, AttemptConfig, AttemptFaultInjector, AttemptFaultPoint, AttemptService,
-    AttemptState, ClaimOutcome, RecoveryOutcome, RenewalOutcome, migrate as attempt_migrate,
+    AttemptState, ClaimOutcome, RecoveryOutcome, RenewalOutcome,
 };
 use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
 use remora_coordination::{ActivateOptions, CoordinationStore, LeaseOwner, StoreConfig};
-use remora_credentials::migrate as credentials_migrate;
 use remora_domain::{BucketName, OpaqueId, StorageScope, TargetPrefix, TargetStoreId};
-use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
+use remora_durable::{MutationDisposition, PoolConfig, connect};
 use remora_identity::{DefaultTarget, Principal};
+use remora_schema::migrate as migrate_schema;
 use sqlx::PgPool;
 use sqlx::postgres::PgPoolOptions;
 use tokio::time::sleep;
@@ -38,12 +36,7 @@ impl Harness {
         )
         .await
         .expect("connect postgres");
-        durable_migrate(&pool).await.expect("durable migrate");
+        migrate_schema(&pool).await.expect("schema migrate");
-        credentials_migrate(&pool)
-            .await
-            .expect("credentials migrate");
-        admission_migrate(&pool).await.expect("admission migrate");
-        attempt_migrate(&pool).await.expect("attempt migrate");
         let namespace = format!("{}wp15:{name}:{}:", coordination_namespace(), nonce());
         let coordination = CoordinationStore::connect(
             &coordination_url,

crates/remora-attempt/tests/mutations.rs

@@ -1,23 +1,21 @@
 use std::time::Duration;
 
-use remora_admission::{
+use remora_admission::{AdmissionConfig, AdmissionDecision, AdmissionService};
-    AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
-};
 use remora_attempt::{
     ActivationOutcome, AttemptConfig, AttemptMutationOutcome, AttemptService, AttemptState,
     CheckpointProjection, CheckpointRecord, CommittedResultRecord, DurableDescriptor,
-    FailureRecord, MutationRejection, ProgressRecord, migrate as attempt_migrate,
+    FailureRecord, MutationRejection, ProgressRecord,
 };
 use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
 use remora_coordination::{CoordinationStore, StoreConfig};
-use remora_credentials::migrate as credentials_migrate;
 use remora_domain::{
     BucketName, FailureDisposition, JobPhase, OpaqueId, RuntimeFailureClass, StorageScope,
     TargetPrefix, TargetStoreId,
 };
-use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
+use remora_durable::{MutationDisposition, PoolConfig, connect};
 use remora_identity::{DefaultTarget, Principal};
-use remora_result::{PublicationInput, migrate as result_migrate};
+use remora_result::PublicationInput;
+use remora_schema::migrate as migrate_schema;
 use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_manifest_key_in};
 use sqlx::PgPool;
 
@@ -40,13 +38,7 @@ impl Harness {
         )
         .await
         .expect("connect postgres");
-        durable_migrate(&pool).await.expect("durable migrate");
+        migrate_schema(&pool).await.expect("schema migrate");
-        credentials_migrate(&pool)
-            .await
-            .expect("credentials migrate");
-        admission_migrate(&pool).await.expect("admission migrate");
-        attempt_migrate(&pool).await.expect("attempt migrate");
-        result_migrate(&pool).await.expect("result migrate");
         let coordination = CoordinationStore::connect(
             &coordination_url,
             format!("{}wp16:{name}:{}:", coordination_namespace(), nonce()),

crates/remora-attempt/tests/recovery.rs

@@ -2,23 +2,21 @@ use std::net::SocketAddr;
 use std::sync::{Arc, Barrier, Mutex};
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
-use remora_admission::{
+use remora_admission::{AdmissionConfig, AdmissionDecision, AdmissionService};
-    AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
-};
 use remora_attempt::{
     ActivationOutcome, AttemptConfig, AttemptService, CancelOutcome, CleanupCompletion,
     CleanupIdentity, CooperativeStopOutcome, RecoveryConfig, StagingMultipart, StagingObject,
-    StagingWriteIntent, migrate as attempt_migrate,
+    StagingWriteIntent,
 };
 use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
 use remora_coordination::{CoordinationStore, StoreConfig};
-use remora_credentials::migrate as credentials_migrate;
 use remora_domain::{
     BucketName, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
 };
-use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
+use remora_durable::{PoolConfig, connect};
 use remora_identity::{DefaultTarget, Principal};
-use remora_result::{PublicationInput, migrate as result_migrate};
+use remora_result::PublicationInput;
+use remora_schema::migrate as migrate_schema;
 use remora_target::{
     CreateObjectRequest, MultipartIdentity, ObjectIdentity, StoredObject, TargetStore,
     TargetStoreConfig, attempt_manifest_key_in, attempt_object_key_in,
@@ -127,13 +125,7 @@ impl Harness {
         )
         .await
         .expect("connect postgres");
-        durable_migrate(&pool).await.expect("durable migrate");
+        migrate_schema(&pool).await.expect("schema migrate");
-        credentials_migrate(&pool)
-            .await
-            .expect("credentials migrate");
-        admission_migrate(&pool).await.expect("admission migrate");
-        attempt_migrate(&pool).await.expect("attempt migrate");
-        result_migrate(&pool).await.expect("result migrate");
         sqlx::query("TRUNCATE TABLE jobs CASCADE")
             .execute(&pool)
             .await

crates/remora-credentials/Cargo.toml

@@ -22,4 +22,5 @@ time.workspace = true
 tokio.workspace = true
 
 [dev-dependencies]
+remora-schema.workspace = true
 tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] }