ms2k/remora
SHA256
Watch
0
Code Releases Activity

Centralize durable schema migration ownership

Browse source
This commit is contained in:
ms2k 2026-07-19 01:20:18 +09:00
commit 069f3813ae
12 changed files with 354 additions and 350 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/remora-attempt/Cargo.toml
View file

@ -23,5 +23,6 @@ remora-admission = { path = "../remora-admission" }
remora-contracts = { path = "../remora-contracts" }
remora-credentials = { path = "../remora-credentials" }
remora-identity = { path = "../remora-identity" }
remora-schema.workspace = true
remora-target = { path = "../remora-target", features = ["test-utils"] }
tokio = { workspace = true, features = ["io-util", "net"] }

38
crates/remora-attempt/migrations/0001_attempt_authority.sql
View file

@ -1,38 +0,0 @@
ALTER TABLE jobs
ADD COLUMN current_fence BIGINT NOT NULL DEFAULT 0,
ADD COLUMN authority_attempt_id TEXT,
ADD CONSTRAINT jobs_current_fence_check CHECK (
current_fence >= 0 AND current_fence <= 9007199254740991
);
CREATE TABLE attempts (
attempt_id TEXT PRIMARY KEY,
job_id TEXT NOT NULL REFERENCES jobs (job_id),
fence BIGINT NOT NULL CHECK (fence BETWEEN 1 AND 9007199254740991),
worker_incarnation TEXT NOT NULL,
state TEXT NOT NULL CHECK (state IN ('authorized', 'executing', 'reaped')),
initial_progress BIGINT NOT NULL DEFAULT 0 CHECK (initial_progress >= 0),
authorized_at TIMESTAMPTZ NOT NULL,
executing_at TIMESTAMPTZ,
reaped_at TIMESTAMPTZ,
CONSTRAINT attempts_state_timestamps_check CHECK (
(state = 'authorized' AND executing_at IS NULL AND reaped_at IS NULL)
OR (state = 'executing' AND executing_at IS NOT NULL AND reaped_at IS NULL)
OR (state = 'reaped' AND executing_at IS NULL AND reaped_at IS NOT NULL)
),
CONSTRAINT attempts_job_fence_unique UNIQUE (job_id, fence),
CONSTRAINT attempts_job_authority_unique UNIQUE (attempt_id, job_id, fence),
CONSTRAINT attempts_authority_tuple_unique UNIQUE (
attempt_id, job_id, fence, worker_incarnation
)
);
ALTER TABLE jobs
ADD CONSTRAINT jobs_authority_attempt_fk
FOREIGN KEY (authority_attempt_id, job_id, current_fence)
REFERENCES attempts (attempt_id, job_id, fence)
DEFERRABLE INITIALLY DEFERRED;
CREATE INDEX attempts_job_authorized_idx
ON attempts (job_id, authorized_at)
WHERE state = 'authorized';

35
crates/remora-attempt/migrations/0002_attempt_mutations.sql
View file

@ -1,35 +0,0 @@
ALTER TABLE attempts
DROP CONSTRAINT attempts_state_check,
DROP CONSTRAINT attempts_state_timestamps_check,
ADD COLUMN stopped_at TIMESTAMPTZ,
ADD COLUMN committed_at TIMESTAMPTZ,
ADD COLUMN failure JSONB,
ADD CONSTRAINT attempts_state_check CHECK (
state IN ('authorized', 'executing', 'reaped', 'stopped', 'committed')
),
ADD CONSTRAINT attempts_state_timestamps_check CHECK (
(state = 'authorized' AND executing_at IS NULL AND reaped_at IS NULL
AND stopped_at IS NULL AND committed_at IS NULL)
OR (state = 'executing' AND executing_at IS NOT NULL AND reaped_at IS NULL
AND stopped_at IS NULL AND committed_at IS NULL)
OR (state = 'reaped' AND executing_at IS NULL AND reaped_at IS NOT NULL
AND stopped_at IS NULL AND committed_at IS NULL)
OR (state = 'stopped' AND executing_at IS NOT NULL AND reaped_at IS NULL
AND stopped_at IS NOT NULL AND committed_at IS NULL)
OR (state = 'committed' AND executing_at IS NOT NULL AND reaped_at IS NULL
AND stopped_at IS NULL AND committed_at IS NOT NULL)
);
ALTER TABLE jobs
ADD COLUMN cancel_requested BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN resolved_source JSONB,
ADD COLUMN checkpoint JSONB,
ADD COLUMN committed_result JSONB,
ADD COLUMN failure JSONB,
ADD COLUMN terminal_at TIMESTAMPTZ,
ADD CONSTRAINT jobs_terminal_projection_check CHECK (
((phase = 'succeeded') = (committed_result IS NOT NULL))
AND ((phase = 'failed') = (failure IS NOT NULL))
AND ((phase IN ('succeeded', 'failed', 'cancelled')) = (terminal_at IS NOT NULL))
AND (committed_result IS NULL OR failure IS NULL)
);

63
crates/remora-attempt/migrations/0003_recovery.sql
View file

@ -1,63 +0,0 @@
ALTER TABLE jobs
ADD COLUMN retry_not_before TIMESTAMPTZ,
ADD COLUMN coordination_cleaned_at TIMESTAMPTZ,
ADD COLUMN recovery_checked_at TIMESTAMPTZ;
ALTER TABLE attempts
DROP CONSTRAINT attempts_state_timestamps_check,
ADD COLUMN worker_down_at TIMESTAMPTZ,
ADD CONSTRAINT attempts_state_timestamps_check CHECK (
(state = 'authorized' AND executing_at IS NULL AND reaped_at IS NULL
AND stopped_at IS NULL AND committed_at IS NULL)
OR (state = 'executing' AND executing_at IS NOT NULL AND reaped_at IS NULL
AND stopped_at IS NULL AND committed_at IS NULL)
OR (state = 'reaped' AND executing_at IS NULL AND reaped_at IS NOT NULL
AND stopped_at IS NULL AND committed_at IS NULL)
OR (state = 'stopped' AND reaped_at IS NULL
AND stopped_at IS NOT NULL AND committed_at IS NULL)
OR (state = 'committed' AND executing_at IS NOT NULL AND reaped_at IS NULL
AND stopped_at IS NULL AND committed_at IS NOT NULL)
);
CREATE TABLE attempt_staging_cleanup (
cleanup_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
attempt_id TEXT NOT NULL REFERENCES attempts (attempt_id),
kind TEXT NOT NULL CHECK (kind IN ('object', 'multipart')),
target_store TEXT NOT NULL,
bucket TEXT NOT NULL,
object_key TEXT NOT NULL,
version_id TEXT,
upload_id TEXT,
state TEXT NOT NULL DEFAULT 'pending'
CHECK (state IN ('pending', 'claimed', 'complete')),
claim_token TEXT,
claimed_until TIMESTAMPTZ,
failure_count BIGINT NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
last_error TEXT,
created_at TIMESTAMPTZ NOT NULL,
completed_at TIMESTAMPTZ,
CONSTRAINT attempt_staging_cleanup_identity_shape_check CHECK (
(kind = 'object' AND upload_id IS NULL)
OR (kind = 'multipart' AND version_id IS NULL AND upload_id IS NOT NULL)
),
CONSTRAINT attempt_staging_cleanup_claim_check CHECK (
(state = 'pending' AND claim_token IS NULL AND claimed_until IS NULL
AND completed_at IS NULL)
OR (state = 'claimed' AND claim_token IS NOT NULL AND claimed_until IS NOT NULL
AND completed_at IS NULL)
OR (state = 'complete' AND claim_token IS NULL AND claimed_until IS NULL
AND completed_at IS NOT NULL)
),
UNIQUE NULLS NOT DISTINCT (
attempt_id, kind, target_store, bucket, object_key, version_id, upload_id
)
);
CREATE INDEX attempt_staging_cleanup_claimable_idx
ON attempt_staging_cleanup (cleanup_id)
WHERE state IN ('pending', 'claimed');
CREATE INDEX jobs_recovery_idx
ON jobs (updated_at, job_id)
WHERE phase NOT IN ('succeeded', 'failed', 'cancelled')
OR coordination_cleaned_at IS NULL;

2
crates/remora-attempt/migrations/0004_worker_api.sql
View file

@ -1,2 +0,0 @@
ALTER TABLE attempts
ADD COLUMN execution_envelope JSONB;

39
crates/remora-attempt/migrations/0005_checkpoint_staging_ownership.sql
View file

@ -1,39 +0,0 @@
ALTER TABLE attempts
ADD COLUMN staging_origin_attempt_id TEXT;
UPDATE attempts
SET staging_origin_attempt_id = attempt_id;
ALTER TABLE attempts
ALTER COLUMN staging_origin_attempt_id SET NOT NULL,
ADD CONSTRAINT attempts_staging_origin_fk
FOREIGN KEY (staging_origin_attempt_id)
REFERENCES attempts (attempt_id)
DEFERRABLE INITIALLY DEFERRED,
ADD CONSTRAINT attempts_id_job_unique UNIQUE (attempt_id, job_id);
ALTER TABLE jobs
ADD COLUMN checkpoint_owner_attempt_id TEXT;
UPDATE jobs j
SET checkpoint_owner_attempt_id = (
SELECT a.attempt_id
FROM attempts a
WHERE a.job_id = j.job_id
ORDER BY a.fence DESC
LIMIT 1
)
WHERE j.checkpoint IS NOT NULL;
ALTER TABLE jobs
ADD CONSTRAINT jobs_checkpoint_owner_fk
FOREIGN KEY (checkpoint_owner_attempt_id, job_id)
REFERENCES attempts (attempt_id, job_id)
DEFERRABLE INITIALLY DEFERRED,
ADD CONSTRAINT jobs_checkpoint_owner_check CHECK (
(checkpoint IS NULL) = (checkpoint_owner_attempt_id IS NULL)
);
CREATE INDEX jobs_checkpoint_owner_idx
ON jobs (checkpoint_owner_attempt_id)
WHERE checkpoint_owner_attempt_id IS NOT NULL;

9
crates/remora-attempt/migrations/0006_staging_write_intents.sql
View file

@ -1,9 +0,0 @@
ALTER TABLE attempt_staging_cleanup
DROP CONSTRAINT attempt_staging_cleanup_kind_check,
DROP CONSTRAINT attempt_staging_cleanup_identity_shape_check,
ADD CONSTRAINT attempt_staging_cleanup_kind_check
CHECK (kind IN ('write_intent', 'object', 'multipart')),
ADD CONSTRAINT attempt_staging_cleanup_identity_shape_check CHECK (
(kind IN ('write_intent', 'object') AND upload_id IS NULL)
OR (kind = 'multipart' AND version_id IS NULL AND upload_id IS NOT NULL)
);

2
crates/remora-attempt/src/lib.rs
View file

@ -1,5 +1,4 @@
mod error;
mod migrate;
mod mutation_types;
mod mutations;
mod recovery;
@ -8,7 +7,6 @@ mod service;
mod types;
pub use error::AttemptError;
pub use migrate::migrate;
pub use mutation_types::{
AttemptMutationOutcome, CheckpointMultipart, CheckpointProjection, CheckpointRecord,
CommittedManifestRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,

12
crates/remora-attempt/src/migrate.rs
View file

@ -1,12 +0,0 @@
use sqlx::PgPool;
use crate::AttemptError;
pub async fn migrate(pool: &PgPool) -> Result<(), AttemptError> {
let mut migrator = sqlx::migrate!("./migrations");
migrator.dangerous_set_table_name("_remora_attempt_migrations");
migrator
.run(pool)
.await
.map_err(|error| AttemptError::Durable(remora_durable::DurableError::Migrate(error)))
}

17
crates/remora-attempt/tests/authority.rs
View file

@ -2,19 +2,17 @@ use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use remora_admission::{
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
};
use remora_admission::{AdmissionConfig, AdmissionDecision, AdmissionService};
use remora_attempt::{
ActivationOutcome, AttemptConfig, AttemptFaultInjector, AttemptFaultPoint, AttemptService,
AttemptState, ClaimOutcome, RecoveryOutcome, RenewalOutcome, migrate as attempt_migrate,
AttemptState, ClaimOutcome, RecoveryOutcome, RenewalOutcome,
};
use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
use remora_coordination::{ActivateOptions, CoordinationStore, LeaseOwner, StoreConfig};
use remora_credentials::migrate as credentials_migrate;
use remora_domain::{BucketName, OpaqueId, StorageScope, TargetPrefix, TargetStoreId};
use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
use remora_durable::{MutationDisposition, PoolConfig, connect};
use remora_identity::{DefaultTarget, Principal};
use remora_schema::migrate as migrate_schema;
use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
use tokio::time::sleep;
@ -38,12 +36,7 @@ impl Harness {
)
.await
.expect("connect postgres");
durable_migrate(&pool).await.expect("durable migrate");
credentials_migrate(&pool)
.await
.expect("credentials migrate");
admission_migrate(&pool).await.expect("admission migrate");
attempt_migrate(&pool).await.expect("attempt migrate");
migrate_schema(&pool).await.expect("schema migrate");
let namespace = format!("{}wp15:{name}:{}:", coordination_namespace(), nonce());
let coordination = CoordinationStore::connect(
&coordination_url,

20
crates/remora-attempt/tests/mutations.rs
View file

@ -1,23 +1,21 @@
use std::time::Duration;
use remora_admission::{
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
};
use remora_admission::{AdmissionConfig, AdmissionDecision, AdmissionService};
use remora_attempt::{
ActivationOutcome, AttemptConfig, AttemptMutationOutcome, AttemptService, AttemptState,
CheckpointProjection, CheckpointRecord, CommittedResultRecord, DurableDescriptor,
FailureRecord, MutationRejection, ProgressRecord, migrate as attempt_migrate,
FailureRecord, MutationRejection, ProgressRecord,
};
use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
use remora_coordination::{CoordinationStore, StoreConfig};
use remora_credentials::migrate as credentials_migrate;
use remora_domain::{
BucketName, FailureDisposition, JobPhase, OpaqueId, RuntimeFailureClass, StorageScope,
TargetPrefix, TargetStoreId,
};
use remora_durable::{MutationDisposition, PoolConfig, connect, migrate as durable_migrate};
use remora_durable::{MutationDisposition, PoolConfig, connect};
use remora_identity::{DefaultTarget, Principal};
use remora_result::{PublicationInput, migrate as result_migrate};
use remora_result::PublicationInput;
use remora_schema::migrate as migrate_schema;
use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_manifest_key_in};
use sqlx::PgPool;
@ -40,13 +38,7 @@ impl Harness {
)
.await
.expect("connect postgres");
durable_migrate(&pool).await.expect("durable migrate");
credentials_migrate(&pool)
.await
.expect("credentials migrate");
admission_migrate(&pool).await.expect("admission migrate");
attempt_migrate(&pool).await.expect("attempt migrate");
result_migrate(&pool).await.expect("result migrate");
migrate_schema(&pool).await.expect("schema migrate");
let coordination = CoordinationStore::connect(
&coordination_url,
format!("{}wp16:{name}:{}:", coordination_namespace(), nonce()),

20
crates/remora-attempt/tests/recovery.rs
View file

@ -2,23 +2,21 @@ use std::net::SocketAddr;
use std::sync::{Arc, Barrier, Mutex};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use remora_admission::{
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
};
use remora_admission::{AdmissionConfig, AdmissionDecision, AdmissionService};
use remora_attempt::{
ActivationOutcome, AttemptConfig, AttemptService, CancelOutcome, CleanupCompletion,
CleanupIdentity, CooperativeStopOutcome, RecoveryConfig, StagingMultipart, StagingObject,
StagingWriteIntent, migrate as attempt_migrate,
StagingWriteIntent,
};
use remora_contracts::{HttpSourceSubmissionDto, JobSubmissionDto, SourceSubmissionDto};
use remora_coordination::{CoordinationStore, StoreConfig};
use remora_credentials::migrate as credentials_migrate;
use remora_domain::{
BucketName, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
};
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
use remora_durable::{PoolConfig, connect};
use remora_identity::{DefaultTarget, Principal};
use remora_result::{PublicationInput, migrate as result_migrate};
use remora_result::PublicationInput;
use remora_schema::migrate as migrate_schema;
use remora_target::{
CreateObjectRequest, MultipartIdentity, ObjectIdentity, StoredObject, TargetStore,
TargetStoreConfig, attempt_manifest_key_in, attempt_object_key_in,
@ -127,13 +125,7 @@ impl Harness {
)
.await
.expect("connect postgres");
durable_migrate(&pool).await.expect("durable migrate");
credentials_migrate(&pool)
.await
.expect("credentials migrate");
admission_migrate(&pool).await.expect("admission migrate");
attempt_migrate(&pool).await.expect("attempt migrate");
result_migrate(&pool).await.expect("result migrate");
migrate_schema(&pool).await.expect("schema migrate");
sqlx::query("TRUNCATE TABLE jobs CASCADE")
.execute(&pool)
.await