ms2k/remora
SHA256
Watch
0
Code Releases Activity

Centralize durable schema migration ownership

Browse source
This commit is contained in:
ms2k 2026-07-19 01:20:18 +09:00
commit 069f3813ae
5 changed files with 354 additions and 350 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/remora-deletion/Cargo.toml
View file

@ -23,5 +23,6 @@ remora-admission = { path = "../remora-admission" }
remora-attempt = { path = "../remora-attempt" }
remora-credentials = { path = "../remora-credentials" }
remora-result = { path = "../remora-result" }
remora-schema.workspace = true
remora-stream = { path = "../remora-stream" }
tokio.workspace = true

96
crates/remora-deletion/migrations/0001_deletion_lifecycle.sql
View file

@ -1,96 +0,0 @@
CREATE TABLE deletion_operations (
operation_id TEXT PRIMARY KEY,
job_id TEXT NOT NULL UNIQUE REFERENCES jobs (job_id),
state TEXT NOT NULL CHECK (
state IN ('queued', 'running', 'retry_wait', 'blocked', 'completed')
),
problem_class TEXT CHECK (
problem_class IS NULL
OR problem_class IN ('authorization', 'target_unavailable', 'configuration', 'internal')
),
problem_message TEXT,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL,
completed_at TIMESTAMPTZ,
CONSTRAINT deletion_operations_problem_check CHECK (
(state = 'blocked') = (problem_class IS NOT NULL AND problem_message IS NOT NULL)
),
CONSTRAINT deletion_operations_completion_check CHECK (
(state = 'completed') = (completed_at IS NOT NULL)
)
);
ALTER TABLE jobs
ADD COLUMN deletion_operation_id TEXT,
ADD COLUMN hard_delete_started_at TIMESTAMPTZ,
ADD CONSTRAINT jobs_deletion_operation_fk
FOREIGN KEY (deletion_operation_id)
REFERENCES deletion_operations (operation_id)
DEFERRABLE INITIALLY DEFERRED,
ADD CONSTRAINT jobs_deletion_lifecycle_check CHECK (
(
lifecycle IN ('visible', 'soft_deleted')
AND deletion_operation_id IS NULL
AND hard_delete_started_at IS NULL
)
OR (
lifecycle IN ('hard_deleting', 'hard_deleted')
AND deletion_operation_id IS NOT NULL
AND hard_delete_started_at IS NOT NULL
)
);
CREATE TABLE deletion_items (
item_id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
operation_id TEXT NOT NULL REFERENCES deletion_operations (operation_id),
kind TEXT NOT NULL CHECK (kind IN ('write_intent', 'object', 'multipart', 'manifest')),
priority SMALLINT NOT NULL CHECK (priority IN (0, 10, 100)),
target_store TEXT NOT NULL,
bucket TEXT NOT NULL,
object_key TEXT NOT NULL,
version_id TEXT,
upload_id TEXT,
state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending', 'claimed', 'complete')),
claim_token TEXT,
claimed_until TIMESTAMPTZ,
failure_count BIGINT NOT NULL DEFAULT 0 CHECK (failure_count >= 0),
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT deletion_items_identity_shape_check CHECK (
(kind IN ('write_intent', 'object', 'manifest') AND upload_id IS NULL)
OR (kind = 'multipart' AND version_id IS NULL AND upload_id IS NOT NULL)
),
CONSTRAINT deletion_items_claim_check CHECK (
(
state = 'pending'
AND claim_token IS NULL
AND claimed_until IS NULL
AND completed_at IS NULL
)
OR (
state = 'claimed'
AND claim_token IS NOT NULL
AND claimed_until IS NOT NULL
AND completed_at IS NULL
)
OR (
state = 'complete'
AND claim_token IS NULL
AND claimed_until IS NULL
AND completed_at IS NOT NULL
)
),
UNIQUE NULLS NOT DISTINCT (
operation_id,
kind,
target_store,
bucket,
object_key,
version_id,
upload_id
)
);
CREATE INDEX deletion_items_claimable_idx
ON deletion_items (operation_id, priority, item_id)
WHERE state IN ('pending', 'claimed');

2
crates/remora-deletion/src/lib.rs
View file

@ -1,13 +1,11 @@
//! Durable soft-delete visibility and recoverable exact hard deletion.
mod error;
mod migrate;
mod registry;
mod service;
mod types;
pub use error::DeletionError;
pub use migrate::{migrate, migrator};
pub use registry::TargetRegistry;
pub use service::DeletionService;
pub use types::{

17
crates/remora-deletion/src/migrate.rs
View file

@ -1,17 +0,0 @@
use remora_durable::DurableError;
use sqlx::PgPool;
const MIGRATION_TABLE: &str = "_remora_deletion_migrations";
pub async fn migrate(pool: &PgPool) -> Result<(), DurableError> {
let mut migrator = sqlx::migrate!("./migrations");
migrator.dangerous_set_table_name(MIGRATION_TABLE);
migrator.run(pool).await?;
Ok(())
}
pub fn migrator() -> sqlx::migrate::Migrator {
let mut migrator = sqlx::migrate!("./migrations");
migrator.dangerous_set_table_name(MIGRATION_TABLE);
migrator
}

19
crates/remora-deletion/tests/lifecycle.rs
View file

@ -1,17 +1,15 @@
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use remora_admission::migrate as admission_migrate;
use remora_attempt::migrate as attempt_migrate;
use remora_credentials::migrate as credentials_migrate;
use remora_deletion::{
DeleteIdentity, DeletionConfig, DeletionError, DeletionProblemClass, DeletionService,
DeletionState, DrainOutcome, LifecycleMutation, TargetRegistry, migrate as deletion_migrate,
DeletionState, DrainOutcome, LifecycleMutation, TargetRegistry,
};
use remora_domain::{BucketName, OpaqueId, StorageScope, TargetPrefix, TargetStoreId};
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
use remora_durable::{PoolConfig, connect};
use remora_identity::Principal;
use remora_result::{PublicationInput, ResultInventory, ResultMember, migrate as result_migrate};
use remora_result::{PublicationInput, ResultInventory, ResultMember};
use remora_schema::migrate as migrate_schema;
use remora_target::{
CreateObjectRequest, ObjectIdentity, TargetError, TargetStore, TargetStoreConfig, hash_bytes,
};
@ -56,14 +54,7 @@ impl Harness {
)
.await
.expect("connect postgres");
durable_migrate(&pool).await.expect("durable migration");
credentials_migrate(&pool)
.await
.expect("credentials migration");
admission_migrate(&pool).await.expect("admission migration");
attempt_migrate(&pool).await.expect("attempt migration");
result_migrate(&pool).await.expect("result migration");
deletion_migrate(&pool).await.expect("deletion migration");
migrate_schema(&pool).await.expect("schema migration");
sqlx::query("TRUNCATE TABLE jobs CASCADE")
.execute(&pool)
.await