ms2k/remora
SHA256
Watch
0
Code Releases Activity

Split orchestration by durable use case

Browse source
This commit is contained in:
ms2k 2026-07-19 01:33:14 +09:00
commit fa81cb7f21
12 changed files with 609 additions and 513 deletions
Download patch file Download diff file Expand all files Collapse all files

56
crates/remora-api/src/public.rs
View file

@ -40,6 +40,7 @@ use sqlx::{PgPool, types::Json};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
mod events;
mod routes;
const DEFAULT_LIMIT: u32 = 50;
const MAX_LIMIT: u32 = 200;
@ -170,56 +171,11 @@ pub fn configure_public_api(config: &mut web::ServiceConfig) {
.limit(1024 * 1024)
.error_handler(|_, _| PublicApiError::invalid_request().into()),
)
.route("/jobs", web::post().to(submit_job))
.route("/jobs", web::get().to(list_jobs))
.route("/jobs/{job_id}", web::get().to(get_job))
.route("/jobs/{job_id}/result", web::get().to(get_job_result))
.route(
"/jobs/{job_id}/result/members",
web::get().to(list_result_members),
)
.route(
"/jobs/{job_id}/result/members/{member_id}/content",
web::get().to(access_result_member),
)
.route("/jobs/{job_id}/cancel", web::post().to(cancel_job))
.route(
"/jobs/{job_id}/soft-delete",
web::post().to(soft_delete_job),
)
.route("/jobs/{job_id}/restore", web::post().to(restore_job))
.route(
"/jobs/{job_id}/hard-delete",
web::post().to(hard_delete_job),
)
.route("/jobs/{job_id}/revoke", web::post().to(revoke_job))
.route("/jobs/{job_id}/attempts", web::get().to(list_job_attempts))
.route(
"/source-credentials",
web::post().to(create_source_credential),
)
.route(
"/source-credentials",
web::get().to(list_source_credentials),
)
.route(
"/source-credentials/{credential_id}",
web::get().to(get_source_credential),
)
.route(
"/source-credentials/{credential_id}/versions",
web::post().to(rotate_source_credential),
)
.route(
"/source-credentials/{credential_id}/revoke",
web::post().to(revoke_source_credential),
)
.route(
"/deletions/{operation_id}",
web::get().to(get_deletion_operation),
)
.route("/event-tickets", web::post().to(events::issue_ticket))
.route("/events", web::get().to(events::connect)),
.configure(routes::jobs)
.configure(routes::results)
.configure(routes::credentials)
.configure(routes::deletions)
.configure(routes::events),
);
}

80
crates/remora-api/src/public/routes.rs Normal file
View file

@ -0,0 +1,80 @@
use actix_web::web;
use super::{
access_result_member, cancel_job, create_source_credential, events as event_handlers,
get_deletion_operation, get_job, get_job_result, get_source_credential, hard_delete_job,
list_job_attempts, list_jobs, list_result_members, list_source_credentials, restore_job,
revoke_job, revoke_source_credential, rotate_source_credential, soft_delete_job, submit_job,
};
pub(super) fn jobs(config: &mut web::ServiceConfig) {
config
.route("/jobs", web::post().to(submit_job))
.route("/jobs", web::get().to(list_jobs))
.route("/jobs/{job_id}", web::get().to(get_job))
.route("/jobs/{job_id}/cancel", web::post().to(cancel_job))
.route(
"/jobs/{job_id}/soft-delete",
web::post().to(soft_delete_job),
)
.route("/jobs/{job_id}/restore", web::post().to(restore_job))
.route(
"/jobs/{job_id}/hard-delete",
web::post().to(hard_delete_job),
)
.route("/jobs/{job_id}/revoke", web::post().to(revoke_job))
.route("/jobs/{job_id}/attempts", web::get().to(list_job_attempts));
}
pub(super) fn results(config: &mut web::ServiceConfig) {
config
.route("/jobs/{job_id}/result", web::get().to(get_job_result))
.route(
"/jobs/{job_id}/result/members",
web::get().to(list_result_members),
)
.route(
"/jobs/{job_id}/result/members/{member_id}/content",
web::get().to(access_result_member),
);
}
pub(super) fn credentials(config: &mut web::ServiceConfig) {
config
.route(
"/source-credentials",
web::post().to(create_source_credential),
)
.route(
"/source-credentials",
web::get().to(list_source_credentials),
)
.route(
"/source-credentials/{credential_id}",
web::get().to(get_source_credential),
)
.route(
"/source-credentials/{credential_id}/versions",
web::post().to(rotate_source_credential),
)
.route(
"/source-credentials/{credential_id}/revoke",
web::post().to(revoke_source_credential),
);
}
pub(super) fn deletions(config: &mut web::ServiceConfig) {
config.route(
"/deletions/{operation_id}",
web::get().to(get_deletion_operation),
);
}
pub(super) fn events(config: &mut web::ServiceConfig) {
config
.route(
"/event-tickets",
web::post().to(event_handlers::issue_ticket),
)
.route("/events", web::get().to(event_handlers::connect));
}

38
crates/remora-api/src/worker.rs
View file

@ -30,6 +30,8 @@ use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_staging
use sha2::{Digest, Sha256};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
mod routes;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AuthenticatedWorker {
pub identity: OpaqueId,
@ -96,39 +98,9 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
.limit(1024 * 1024)
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
)
.route("/health/ready", web::get().to(worker_ready))
.route("/jobs/{job_id}/claim", web::post().to(claim))
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
.route("/attempts/{attempt_id}/stop", web::post().to(stop))
.route(
"/attempts/{attempt_id}/staging/write-intent",
web::post().to(register_staging_write_intent),
)
.route(
"/attempts/{attempt_id}/staging/object",
web::post().to(register_staging_object),
)
.route(
"/attempts/{attempt_id}/staging/multipart",
web::post().to(register_staging_multipart),
)
.route(
"/attempts/{attempt_id}/resolved-source",
web::post().to(pin_resolved_source),
)
.route(
"/attempts/{attempt_id}/checkpoint",
web::post().to(persist_checkpoint),
)
.route(
"/attempts/{attempt_id}/progress",
web::post().to(report_progress),
)
.route(
"/attempts/{attempt_id}/outcome",
web::post().to(report_outcome),
),
.configure(routes::authority)
.configure(routes::staging)
.configure(routes::execution),
);
}

52
crates/remora-api/src/worker/routes.rs Normal file
View file

@ -0,0 +1,52 @@
use actix_web::web;
use super::{
activate, claim, persist_checkpoint, pin_resolved_source, register_staging_multipart,
register_staging_object, register_staging_write_intent, renew, report_outcome, report_progress,
stop, worker_ready,
};
pub(super) fn authority(config: &mut web::ServiceConfig) {
config
.route("/health/ready", web::get().to(worker_ready))
.route("/jobs/{job_id}/claim", web::post().to(claim))
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
.route("/attempts/{attempt_id}/stop", web::post().to(stop));
}
pub(super) fn staging(config: &mut web::ServiceConfig) {
config
.route(
"/attempts/{attempt_id}/staging/write-intent",
web::post().to(register_staging_write_intent),
)
.route(
"/attempts/{attempt_id}/staging/object",
web::post().to(register_staging_object),
)
.route(
"/attempts/{attempt_id}/staging/multipart",
web::post().to(register_staging_multipart),
);
}
pub(super) fn execution(config: &mut web::ServiceConfig) {
config
.route(
"/attempts/{attempt_id}/resolved-source",
web::post().to(pin_resolved_source),
)
.route(
"/attempts/{attempt_id}/checkpoint",
web::post().to(persist_checkpoint),
)
.route(
"/attempts/{attempt_id}/progress",
web::post().to(report_progress),
)
.route(
"/attempts/{attempt_id}/outcome",
web::post().to(report_outcome),
);
}

152
crates/remora-attempt/src/recovery.rs
View file

@ -21,157 +21,9 @@ use crate::{
StagingMultipart, StagingObject, StagingWriteIntent,
};
mod cancellation;
impl AttemptService {
pub async fn request_cancel(&self, job_id: &OpaqueId) -> Result<CancelOutcome, AttemptError> {
with_transaction(&self.pool, |tx| {
let job_id = job_id.clone();
Box::pin(async move { Self::request_cancel_in_tx(tx, &job_id).await })
})
.await
.map_err(Into::into)
}
pub async fn request_cancel_in_tx(
tx: &mut PgTx<'_>,
job_id: &OpaqueId,
) -> Result<CancelOutcome, DurableError> {
let Some(row) = sqlx::query_as::<_, CancelRow>(
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
)
.bind(job_id.as_str())
.fetch_optional(tx.as_mut())
.await?
else {
return Ok(CancelOutcome::NotFound);
};
let phase = parse_phase(&row.phase)?;
if phase.is_terminal() {
return Ok(CancelOutcome::AlreadyTerminal);
}
if row.cancel_requested {
return Ok(CancelOutcome::AlreadyRequested);
}
let now = OffsetDateTime::now_utc();
if phase == JobPhase::Running {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
} else {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
authority_attempt_id = NULL, dispatch_needed = FALSE,
terminal_at = $2, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
}
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(CancelOutcome::Requested)
}
pub async fn cooperative_stop(
&self,
authority: &AttemptAuthority,
) -> Result<CooperativeStopOutcome, AttemptError> {
let authority_for_effect = authority.clone();
let coordination = self.coordination.clone();
let outcome = with_transaction(&self.pool, move |tx| {
Box::pin(async move {
cooperative_stop_transaction(tx, &coordination, &authority_for_effect).await
})
})
.await?;
if outcome == CooperativeStopOutcome::Stopped {
let _ = self
.coordination
.release(authority.job_id.as_str(), &lease_owner(authority))
.await;
}
Ok(outcome)
}
pub async fn cooperative_stop_operation(
&self,
authority: &AttemptAuthority,
operation_id: &OpaqueId,
) -> Result<CooperativeStopOutcome, AttemptError> {
let request_fingerprint = fingerprint(&StopRequest {
job_id: &authority.job_id,
attempt_id: &authority.attempt_id,
fence: authority.fence,
worker_incarnation: &authority.worker_incarnation,
})
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
let authority_for_effect = authority.clone();
let coordination = self.coordination.clone();
let outcome = execute_mutation(
&self.pool,
MutationScope {
subject: &self.operation_subject,
operation: "attempt.mutation",
key: operation_id.as_str(),
},
"cooperative_stop",
&[],
request_fingerprint,
move |tx| {
Box::pin(async move {
let decision =
cooperative_stop_transaction(tx, &coordination, &authority_for_effect)
.await?;
let status = if decision == CooperativeStopOutcome::AuthorityMismatch {
MutationStatus::Rejected
} else {
MutationStatus::Accepted
};
Ok((
status,
serde_json::to_value(decision).map_err(DurableError::effect)?,
))
})
},
)
.await?;
if outcome.disposition == MutationDisposition::Conflict {
return Err(AttemptError::OperationConflict);
}
let decision = serde_json::from_value(outcome.response)
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
if decision == CooperativeStopOutcome::Stopped {
let _ = self
.coordination
.release(authority.job_id.as_str(), &lease_owner(authority))
.await;
}
Ok(decision)
}
pub async fn mark_worker_down(
&self,
worker_incarnation: &OpaqueId,
) -> Result<u64, AttemptError> {
let changed = sqlx::query(
"UPDATE attempts SET worker_down_at = COALESCE(worker_down_at, $2)
WHERE worker_incarnation = $1 AND state IN ('authorized', 'executing')",
)
.bind(worker_incarnation.as_str())
.bind(OffsetDateTime::now_utc())
.execute(&self.pool)
.await?;
Ok(changed.rows_affected())
}
pub async fn register_staging_object(
&self,
attempt_id: &OpaqueId,

153
crates/remora-attempt/src/recovery/cancellation.rs Normal file
View file

@ -0,0 +1,153 @@
use super::*;
impl AttemptService {
pub async fn request_cancel(&self, job_id: &OpaqueId) -> Result<CancelOutcome, AttemptError> {
with_transaction(&self.pool, |tx| {
let job_id = job_id.clone();
Box::pin(async move { Self::request_cancel_in_tx(tx, &job_id).await })
})
.await
.map_err(Into::into)
}
pub async fn request_cancel_in_tx(
tx: &mut PgTx<'_>,
job_id: &OpaqueId,
) -> Result<CancelOutcome, DurableError> {
let Some(row) = sqlx::query_as::<_, CancelRow>(
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
)
.bind(job_id.as_str())
.fetch_optional(tx.as_mut())
.await?
else {
return Ok(CancelOutcome::NotFound);
};
let phase = parse_phase(&row.phase)?;
if phase.is_terminal() {
return Ok(CancelOutcome::AlreadyTerminal);
}
if row.cancel_requested {
return Ok(CancelOutcome::AlreadyRequested);
}
let now = OffsetDateTime::now_utc();
if phase == JobPhase::Running {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
} else {
sqlx::query(
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
authority_attempt_id = NULL, dispatch_needed = FALSE,
terminal_at = $2, coordination_cleaned_at = NULL,
revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
}
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(CancelOutcome::Requested)
}
pub async fn cooperative_stop(
&self,
authority: &AttemptAuthority,
) -> Result<CooperativeStopOutcome, AttemptError> {
let authority_for_effect = authority.clone();
let coordination = self.coordination.clone();
let outcome = with_transaction(&self.pool, move |tx| {
Box::pin(async move {
cooperative_stop_transaction(tx, &coordination, &authority_for_effect).await
})
})
.await?;
if outcome == CooperativeStopOutcome::Stopped {
let _ = self
.coordination
.release(authority.job_id.as_str(), &lease_owner(authority))
.await;
}
Ok(outcome)
}
pub async fn cooperative_stop_operation(
&self,
authority: &AttemptAuthority,
operation_id: &OpaqueId,
) -> Result<CooperativeStopOutcome, AttemptError> {
let request_fingerprint = fingerprint(&StopRequest {
job_id: &authority.job_id,
attempt_id: &authority.attempt_id,
fence: authority.fence,
worker_incarnation: &authority.worker_incarnation,
})
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
let authority_for_effect = authority.clone();
let coordination = self.coordination.clone();
let outcome = execute_mutation(
&self.pool,
MutationScope {
subject: &self.operation_subject,
operation: "attempt.mutation",
key: operation_id.as_str(),
},
"cooperative_stop",
&[],
request_fingerprint,
move |tx| {
Box::pin(async move {
let decision =
cooperative_stop_transaction(tx, &coordination, &authority_for_effect)
.await?;
let status = if decision == CooperativeStopOutcome::AuthorityMismatch {
MutationStatus::Rejected
} else {
MutationStatus::Accepted
};
Ok((
status,
serde_json::to_value(decision).map_err(DurableError::effect)?,
))
})
},
)
.await?;
if outcome.disposition == MutationDisposition::Conflict {
return Err(AttemptError::OperationConflict);
}
let decision = serde_json::from_value(outcome.response)
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
if decision == CooperativeStopOutcome::Stopped {
let _ = self
.coordination
.release(authority.job_id.as_str(), &lease_owner(authority))
.await;
}
Ok(decision)
}
pub async fn mark_worker_down(
&self,
worker_incarnation: &OpaqueId,
) -> Result<u64, AttemptError> {
let changed = sqlx::query(
"UPDATE attempts SET worker_down_at = COALESCE(worker_down_at, $2)
WHERE worker_incarnation = $1 AND state IN ('authorized', 'executing')",
)
.bind(worker_incarnation.as_str())
.bind(OffsetDateTime::now_utc())
.execute(&self.pool)
.await?;
Ok(changed.rows_affected())
}
}

118
crates/remora-credentials/src/registry.rs
View file

@ -16,6 +16,8 @@ use crate::types::{
};
use crate::vault::{EmptyPathFence, VaultClient};
mod query;
/// Durable source-credential registry with Vault-backed secret versions.
#[derive(Clone, Debug)]
pub struct CredentialRegistry {
@ -417,122 +419,6 @@ impl CredentialRegistry {
.ok_or_else(|| DurableError::Effect("revoked credential is missing".into()))
}
pub async fn get(
&self,
caller: &CredentialCaller,
credential_id: &OpaqueId,
) -> Result<Option<SourceCredential>, CredentialError> {
let record = self.get_internal(credential_id).await?;
match require_manage(caller, record.as_ref()) {
Ok(visible) => Ok(Some(visible)),
Err(CredentialError::NotFound) => Ok(None),
Err(error) => Err(error),
}
}
pub async fn list(
&self,
caller: &CredentialCaller,
limit: u32,
cursor: Option<&str>,
) -> Result<SourceCredentialPage, CredentialError> {
let limit = limit.clamp(1, 100) as i64;
let after = cursor.map(decode_cursor).transpose()?;
let owner_filter = if caller.administrator {
None
} else {
Some(caller.principal_id.as_str().to_owned())
};
let rows = if let Some(owner) = owner_filter.as_deref() {
if let Some((created_at, id)) = after {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE c.owner_id = $1
AND (c.created_at, c.credential_id) < ($2, $3)
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $4",
)
.bind(owner)
.bind(created_at)
.bind(id)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
} else {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE c.owner_id = $1
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $2",
)
.bind(owner)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
}
} else if let Some((created_at, id)) = after {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE (c.created_at, c.credential_id) < ($1, $2)
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $3",
)
.bind(created_at)
.bind(id)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
} else {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $1",
)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
};
let mut items: Vec<SourceCredential> = Vec::with_capacity(rows.len().min(limit as usize));
let mut next_cursor = None;
for (index, row) in rows.into_iter().enumerate() {
if index as i64 >= limit {
let last = items
.last()
.ok_or_else(|| CredentialError::Config("page overflow without items".into()))?;
next_cursor = Some(encode_cursor(last.created_at, last.id.as_str()));
break;
}
items.push(row.into_record()?);
}
Ok(SourceCredentialPage { items, next_cursor })
}
/// Pin the current authorized version for a new execution grant.
///
/// Serializes with revoke/rotate on the credential advisory lock so a

119
crates/remora-credentials/src/registry/query.rs Normal file
View file

@ -0,0 +1,119 @@
use super::*;
impl CredentialRegistry {
pub async fn get(
&self,
caller: &CredentialCaller,
credential_id: &OpaqueId,
) -> Result<Option<SourceCredential>, CredentialError> {
let record = self.get_internal(credential_id).await?;
match require_manage(caller, record.as_ref()) {
Ok(visible) => Ok(Some(visible)),
Err(CredentialError::NotFound) => Ok(None),
Err(error) => Err(error),
}
}
pub async fn list(
&self,
caller: &CredentialCaller,
limit: u32,
cursor: Option<&str>,
) -> Result<SourceCredentialPage, CredentialError> {
let limit = limit.clamp(1, 100) as i64;
let after = cursor.map(decode_cursor).transpose()?;
let owner_filter = if caller.administrator {
None
} else {
Some(caller.principal_id.as_str().to_owned())
};
let rows = if let Some(owner) = owner_filter.as_deref() {
if let Some((created_at, id)) = after {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE c.owner_id = $1
AND (c.created_at, c.credential_id) < ($2, $3)
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $4",
)
.bind(owner)
.bind(created_at)
.bind(id)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
} else {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE c.owner_id = $1
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $2",
)
.bind(owner)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
}
} else if let Some((created_at, id)) = after {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
WHERE (c.created_at, c.credential_id) < ($1, $2)
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $3",
)
.bind(created_at)
.bind(id)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
} else {
sqlx::query_as::<_, CredentialRow>(
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
c.created_at, c.updated_at,
v.version_id AS active_version_id, v.created_at AS active_version_created_at
FROM source_credentials c
LEFT JOIN source_credential_versions v
ON v.version_id = c.active_version_id
ORDER BY c.created_at DESC, c.credential_id DESC
LIMIT $1",
)
.bind(limit + 1)
.fetch_all(&self.pool)
.await
.map_err(DurableError::from)?
};
let mut items: Vec<SourceCredential> = Vec::with_capacity(rows.len().min(limit as usize));
let mut next_cursor = None;
for (index, row) in rows.into_iter().enumerate() {
if index as i64 >= limit {
let last = items
.last()
.ok_or_else(|| CredentialError::Config("page overflow without items".into()))?;
next_cursor = Some(encode_cursor(last.created_at, last.id.as_str()));
break;
}
items.push(row.into_record()?);
}
Ok(SourceCredentialPage { items, next_cursor })
}
}

152
crates/remora-deletion/src/service.rs
View file

@ -21,6 +21,8 @@ use crate::{
TargetRegistry,
};
mod lifecycle;
#[derive(Clone)]
pub struct DeletionService {
pool: PgPool,
@ -46,156 +48,6 @@ impl DeletionService {
})
}
pub async fn soft_delete(
&self,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let mut tx = self.pool.begin().await?;
let outcome = Self::soft_delete_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn soft_delete_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::SoftDelete)?;
require_terminal(&row.phase)?;
if row.deletion_operation_id.is_some() {
return Err(DeletionError::HardDeleteStarted);
}
if row.lifecycle == "soft_deleted" {
return Ok(LifecycleMutation::Unchanged);
}
let now = OffsetDateTime::now_utc();
sqlx::query(
"UPDATE jobs
SET lifecycle = 'soft_deleted', revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(LifecycleMutation::Changed)
}
pub async fn restore(
&self,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let mut tx = self.pool.begin().await?;
let outcome = Self::restore_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn restore_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::Restore)?;
require_terminal(&row.phase)?;
if row.deletion_operation_id.is_some() {
return Err(DeletionError::HardDeleteStarted);
}
if row.lifecycle == "visible" {
return Ok(LifecycleMutation::Unchanged);
}
let now = OffsetDateTime::now_utc();
sqlx::query(
"UPDATE jobs
SET lifecycle = 'visible', revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(LifecycleMutation::Changed)
}
pub async fn hard_delete(
&self,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<HardDeleteOutcome, DeletionError> {
let mut tx = self.pool.begin().await?;
let outcome = Self::hard_delete_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn hard_delete_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<HardDeleteOutcome, DeletionError> {
let operation_id = new_id('d')?;
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::HardDelete)?;
require_terminal(&row.phase)?;
if let Some(existing) = row.deletion_operation_id {
let operation = load_operation_in_tx(tx, &existing).await?;
return Ok(HardDeleteOutcome {
mutation: LifecycleMutation::Unchanged,
operation,
});
}
validate_committed_inventory(tx, &row).await?;
validate_deletion_identities(tx, &row).await?;
let now = OffsetDateTime::now_utc();
sqlx::query(
"INSERT INTO deletion_operations (
operation_id, job_id, state, created_at, updated_at
) VALUES ($1, $2, 'queued', $3, $3)",
)
.bind(operation_id.as_str())
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
sqlx::query(
"UPDATE jobs
SET lifecycle = 'hard_deleting', deletion_operation_id = $2,
hard_delete_started_at = $3, revision = revision + 1, updated_at = $3
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(operation_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
materialize_items(tx, job_id, &operation_id, now).await?;
finalize_if_empty(tx, job_id, &operation_id, now).await?;
enqueue(
tx,
"job.updated",
json!({ "job_id": job_id, "deletion_operation_id": operation_id }),
)
.await?;
enqueue(
tx,
"deletion.updated",
json!({ "job_id": job_id, "operation_id": operation_id }),
)
.await?;
let operation = load_operation_in_tx(tx, operation_id.as_str()).await?;
Ok(HardDeleteOutcome {
mutation: LifecycleMutation::Changed,
operation,
})
}
pub async fn operation(
&self,
principal: &Principal,

153
crates/remora-deletion/src/service/lifecycle.rs Normal file
View file

@ -0,0 +1,153 @@
use super::*;
impl DeletionService {
pub async fn soft_delete(
&self,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let mut tx = self.pool.begin().await?;
let outcome = Self::soft_delete_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn soft_delete_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::SoftDelete)?;
require_terminal(&row.phase)?;
if row.deletion_operation_id.is_some() {
return Err(DeletionError::HardDeleteStarted);
}
if row.lifecycle == "soft_deleted" {
return Ok(LifecycleMutation::Unchanged);
}
let now = OffsetDateTime::now_utc();
sqlx::query(
"UPDATE jobs
SET lifecycle = 'soft_deleted', revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(LifecycleMutation::Changed)
}
pub async fn restore(
&self,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let mut tx = self.pool.begin().await?;
let outcome = Self::restore_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn restore_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<LifecycleMutation, DeletionError> {
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::Restore)?;
require_terminal(&row.phase)?;
if row.deletion_operation_id.is_some() {
return Err(DeletionError::HardDeleteStarted);
}
if row.lifecycle == "visible" {
return Ok(LifecycleMutation::Unchanged);
}
let now = OffsetDateTime::now_utc();
sqlx::query(
"UPDATE jobs
SET lifecycle = 'visible', revision = revision + 1, updated_at = $2
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
Ok(LifecycleMutation::Changed)
}
pub async fn hard_delete(
&self,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<HardDeleteOutcome, DeletionError> {
let mut tx = self.pool.begin().await?;
let outcome = Self::hard_delete_in_tx(&mut tx, principal, job_id).await?;
tx.commit().await?;
Ok(outcome)
}
pub async fn hard_delete_in_tx(
tx: &mut PgTx<'_>,
principal: &Principal,
job_id: &OpaqueId,
) -> Result<HardDeleteOutcome, DeletionError> {
let operation_id = new_id('d')?;
let row = lock_job(tx, job_id).await?;
authorize(principal, &row, ManagementOperation::HardDelete)?;
require_terminal(&row.phase)?;
if let Some(existing) = row.deletion_operation_id {
let operation = load_operation_in_tx(tx, &existing).await?;
return Ok(HardDeleteOutcome {
mutation: LifecycleMutation::Unchanged,
operation,
});
}
validate_committed_inventory(tx, &row).await?;
validate_deletion_identities(tx, &row).await?;
let now = OffsetDateTime::now_utc();
sqlx::query(
"INSERT INTO deletion_operations (
operation_id, job_id, state, created_at, updated_at
) VALUES ($1, $2, 'queued', $3, $3)",
)
.bind(operation_id.as_str())
.bind(job_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
sqlx::query(
"UPDATE jobs
SET lifecycle = 'hard_deleting', deletion_operation_id = $2,
hard_delete_started_at = $3, revision = revision + 1, updated_at = $3
WHERE job_id = $1",
)
.bind(job_id.as_str())
.bind(operation_id.as_str())
.bind(now)
.execute(tx.as_mut())
.await?;
materialize_items(tx, job_id, &operation_id, now).await?;
finalize_if_empty(tx, job_id, &operation_id, now).await?;
enqueue(
tx,
"job.updated",
json!({ "job_id": job_id, "deletion_operation_id": operation_id }),
)
.await?;
enqueue(
tx,
"deletion.updated",
json!({ "job_id": job_id, "operation_id": operation_id }),
)
.await?;
let operation = load_operation_in_tx(tx, operation_id.as_str()).await?;
Ok(HardDeleteOutcome {
mutation: LifecycleMutation::Changed,
operation,
})
}
}

18
crates/remora-worker/src/executor.rs
View file

@ -46,6 +46,8 @@ use crate::{
WorkerError,
};
mod source_execution;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectSourceEndpoint {
pub endpoint: String,
@ -159,20 +161,8 @@ impl AdapterExecutor {
resume: &resume,
};
match source {
SourceSubmissionDto::Http(source) => {
self.execute_http(context, envelope, source, secret, &resources)
.await?;
}
SourceSubmissionDto::ObjectStore(source) => {
self.execute_object(context, envelope, source, secret, &resources)
.await?;
}
SourceSubmissionDto::HuggingFace(source) => {
self.execute_hugging_face(context, envelope, source, secret, &resources)
.await?;
}
}
self.execute_source(context, envelope, source, secret, &resources)
.await?;
context
.require_live_authority()

31
crates/remora-worker/src/executor/source_execution.rs Normal file
View file

@ -0,0 +1,31 @@
use remora_contracts::{ExecutionEnvelopeDto, SourceSubmissionDto};
use remora_credentials::SecretMaterial;
use super::{AdapterExecutor, AdapterFailure, ExecutionResources};
use crate::AttemptContext;
impl AdapterExecutor {
pub(super) async fn execute_source(
&self,
context: &AttemptContext,
envelope: &ExecutionEnvelopeDto,
source: SourceSubmissionDto,
secret: Option<SecretMaterial>,
resources: &ExecutionResources<'_>,
) -> Result<(), AdapterFailure> {
match source {
SourceSubmissionDto::Http(source) => {
self.execute_http(context, envelope, source, secret, resources)
.await
}
SourceSubmissionDto::ObjectStore(source) => {
self.execute_object(context, envelope, source, secret, resources)
.await
}
SourceSubmissionDto::HuggingFace(source) => {
self.execute_hugging_face(context, envelope, source, secret, resources)
.await
}
}
}
}