Split orchestration by durable use case
This commit is contained in:
parent
069f3813ae
commit
fa81cb7f21
12 changed files with 609 additions and 513 deletions
|
|
@ -40,6 +40,7 @@ use sqlx::{PgPool, types::Json};
|
||||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||||
|
|
||||||
mod events;
|
mod events;
|
||||||
|
mod routes;
|
||||||
|
|
||||||
const DEFAULT_LIMIT: u32 = 50;
|
const DEFAULT_LIMIT: u32 = 50;
|
||||||
const MAX_LIMIT: u32 = 200;
|
const MAX_LIMIT: u32 = 200;
|
||||||
|
|
@ -170,56 +171,11 @@ pub fn configure_public_api(config: &mut web::ServiceConfig) {
|
||||||
.limit(1024 * 1024)
|
.limit(1024 * 1024)
|
||||||
.error_handler(|_, _| PublicApiError::invalid_request().into()),
|
.error_handler(|_, _| PublicApiError::invalid_request().into()),
|
||||||
)
|
)
|
||||||
.route("/jobs", web::post().to(submit_job))
|
.configure(routes::jobs)
|
||||||
.route("/jobs", web::get().to(list_jobs))
|
.configure(routes::results)
|
||||||
.route("/jobs/{job_id}", web::get().to(get_job))
|
.configure(routes::credentials)
|
||||||
.route("/jobs/{job_id}/result", web::get().to(get_job_result))
|
.configure(routes::deletions)
|
||||||
.route(
|
.configure(routes::events),
|
||||||
"/jobs/{job_id}/result/members",
|
|
||||||
web::get().to(list_result_members),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/jobs/{job_id}/result/members/{member_id}/content",
|
|
||||||
web::get().to(access_result_member),
|
|
||||||
)
|
|
||||||
.route("/jobs/{job_id}/cancel", web::post().to(cancel_job))
|
|
||||||
.route(
|
|
||||||
"/jobs/{job_id}/soft-delete",
|
|
||||||
web::post().to(soft_delete_job),
|
|
||||||
)
|
|
||||||
.route("/jobs/{job_id}/restore", web::post().to(restore_job))
|
|
||||||
.route(
|
|
||||||
"/jobs/{job_id}/hard-delete",
|
|
||||||
web::post().to(hard_delete_job),
|
|
||||||
)
|
|
||||||
.route("/jobs/{job_id}/revoke", web::post().to(revoke_job))
|
|
||||||
.route("/jobs/{job_id}/attempts", web::get().to(list_job_attempts))
|
|
||||||
.route(
|
|
||||||
"/source-credentials",
|
|
||||||
web::post().to(create_source_credential),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/source-credentials",
|
|
||||||
web::get().to(list_source_credentials),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/source-credentials/{credential_id}",
|
|
||||||
web::get().to(get_source_credential),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/source-credentials/{credential_id}/versions",
|
|
||||||
web::post().to(rotate_source_credential),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/source-credentials/{credential_id}/revoke",
|
|
||||||
web::post().to(revoke_source_credential),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/deletions/{operation_id}",
|
|
||||||
web::get().to(get_deletion_operation),
|
|
||||||
)
|
|
||||||
.route("/event-tickets", web::post().to(events::issue_ticket))
|
|
||||||
.route("/events", web::get().to(events::connect)),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
80
crates/remora-api/src/public/routes.rs
Normal file
80
crates/remora-api/src/public/routes.rs
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
use actix_web::web;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
access_result_member, cancel_job, create_source_credential, events as event_handlers,
|
||||||
|
get_deletion_operation, get_job, get_job_result, get_source_credential, hard_delete_job,
|
||||||
|
list_job_attempts, list_jobs, list_result_members, list_source_credentials, restore_job,
|
||||||
|
revoke_job, revoke_source_credential, rotate_source_credential, soft_delete_job, submit_job,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(super) fn jobs(config: &mut web::ServiceConfig) {
|
||||||
|
config
|
||||||
|
.route("/jobs", web::post().to(submit_job))
|
||||||
|
.route("/jobs", web::get().to(list_jobs))
|
||||||
|
.route("/jobs/{job_id}", web::get().to(get_job))
|
||||||
|
.route("/jobs/{job_id}/cancel", web::post().to(cancel_job))
|
||||||
|
.route(
|
||||||
|
"/jobs/{job_id}/soft-delete",
|
||||||
|
web::post().to(soft_delete_job),
|
||||||
|
)
|
||||||
|
.route("/jobs/{job_id}/restore", web::post().to(restore_job))
|
||||||
|
.route(
|
||||||
|
"/jobs/{job_id}/hard-delete",
|
||||||
|
web::post().to(hard_delete_job),
|
||||||
|
)
|
||||||
|
.route("/jobs/{job_id}/revoke", web::post().to(revoke_job))
|
||||||
|
.route("/jobs/{job_id}/attempts", web::get().to(list_job_attempts));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn results(config: &mut web::ServiceConfig) {
|
||||||
|
config
|
||||||
|
.route("/jobs/{job_id}/result", web::get().to(get_job_result))
|
||||||
|
.route(
|
||||||
|
"/jobs/{job_id}/result/members",
|
||||||
|
web::get().to(list_result_members),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/jobs/{job_id}/result/members/{member_id}/content",
|
||||||
|
web::get().to(access_result_member),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn credentials(config: &mut web::ServiceConfig) {
|
||||||
|
config
|
||||||
|
.route(
|
||||||
|
"/source-credentials",
|
||||||
|
web::post().to(create_source_credential),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/source-credentials",
|
||||||
|
web::get().to(list_source_credentials),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/source-credentials/{credential_id}",
|
||||||
|
web::get().to(get_source_credential),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/source-credentials/{credential_id}/versions",
|
||||||
|
web::post().to(rotate_source_credential),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/source-credentials/{credential_id}/revoke",
|
||||||
|
web::post().to(revoke_source_credential),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn deletions(config: &mut web::ServiceConfig) {
|
||||||
|
config.route(
|
||||||
|
"/deletions/{operation_id}",
|
||||||
|
web::get().to(get_deletion_operation),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn events(config: &mut web::ServiceConfig) {
|
||||||
|
config
|
||||||
|
.route(
|
||||||
|
"/event-tickets",
|
||||||
|
web::post().to(event_handlers::issue_ticket),
|
||||||
|
)
|
||||||
|
.route("/events", web::get().to(event_handlers::connect));
|
||||||
|
}
|
||||||
|
|
@ -30,6 +30,8 @@ use remora_target::{ContentSha256, ObjectIdentity, StoredObject, attempt_staging
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||||
|
|
||||||
|
mod routes;
|
||||||
|
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct AuthenticatedWorker {
|
pub struct AuthenticatedWorker {
|
||||||
pub identity: OpaqueId,
|
pub identity: OpaqueId,
|
||||||
|
|
@ -96,39 +98,9 @@ pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
||||||
.limit(1024 * 1024)
|
.limit(1024 * 1024)
|
||||||
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
|
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
|
||||||
)
|
)
|
||||||
.route("/health/ready", web::get().to(worker_ready))
|
.configure(routes::authority)
|
||||||
.route("/jobs/{job_id}/claim", web::post().to(claim))
|
.configure(routes::staging)
|
||||||
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
|
.configure(routes::execution),
|
||||||
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
|
|
||||||
.route("/attempts/{attempt_id}/stop", web::post().to(stop))
|
|
||||||
.route(
|
|
||||||
"/attempts/{attempt_id}/staging/write-intent",
|
|
||||||
web::post().to(register_staging_write_intent),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/attempts/{attempt_id}/staging/object",
|
|
||||||
web::post().to(register_staging_object),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/attempts/{attempt_id}/staging/multipart",
|
|
||||||
web::post().to(register_staging_multipart),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/attempts/{attempt_id}/resolved-source",
|
|
||||||
web::post().to(pin_resolved_source),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/attempts/{attempt_id}/checkpoint",
|
|
||||||
web::post().to(persist_checkpoint),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/attempts/{attempt_id}/progress",
|
|
||||||
web::post().to(report_progress),
|
|
||||||
)
|
|
||||||
.route(
|
|
||||||
"/attempts/{attempt_id}/outcome",
|
|
||||||
web::post().to(report_outcome),
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
52
crates/remora-api/src/worker/routes.rs
Normal file
52
crates/remora-api/src/worker/routes.rs
Normal file
|
|
@ -0,0 +1,52 @@
|
||||||
|
use actix_web::web;
|
||||||
|
|
||||||
|
use super::{
|
||||||
|
activate, claim, persist_checkpoint, pin_resolved_source, register_staging_multipart,
|
||||||
|
register_staging_object, register_staging_write_intent, renew, report_outcome, report_progress,
|
||||||
|
stop, worker_ready,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub(super) fn authority(config: &mut web::ServiceConfig) {
|
||||||
|
config
|
||||||
|
.route("/health/ready", web::get().to(worker_ready))
|
||||||
|
.route("/jobs/{job_id}/claim", web::post().to(claim))
|
||||||
|
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
|
||||||
|
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
|
||||||
|
.route("/attempts/{attempt_id}/stop", web::post().to(stop));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn staging(config: &mut web::ServiceConfig) {
|
||||||
|
config
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/staging/write-intent",
|
||||||
|
web::post().to(register_staging_write_intent),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/staging/object",
|
||||||
|
web::post().to(register_staging_object),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/staging/multipart",
|
||||||
|
web::post().to(register_staging_multipart),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn execution(config: &mut web::ServiceConfig) {
|
||||||
|
config
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/resolved-source",
|
||||||
|
web::post().to(pin_resolved_source),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/checkpoint",
|
||||||
|
web::post().to(persist_checkpoint),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/progress",
|
||||||
|
web::post().to(report_progress),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/outcome",
|
||||||
|
web::post().to(report_outcome),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -21,157 +21,9 @@ use crate::{
|
||||||
StagingMultipart, StagingObject, StagingWriteIntent,
|
StagingMultipart, StagingObject, StagingWriteIntent,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod cancellation;
|
||||||
|
|
||||||
impl AttemptService {
|
impl AttemptService {
|
||||||
pub async fn request_cancel(&self, job_id: &OpaqueId) -> Result<CancelOutcome, AttemptError> {
|
|
||||||
with_transaction(&self.pool, |tx| {
|
|
||||||
let job_id = job_id.clone();
|
|
||||||
Box::pin(async move { Self::request_cancel_in_tx(tx, &job_id).await })
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(Into::into)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn request_cancel_in_tx(
|
|
||||||
tx: &mut PgTx<'_>,
|
|
||||||
job_id: &OpaqueId,
|
|
||||||
) -> Result<CancelOutcome, DurableError> {
|
|
||||||
let Some(row) = sqlx::query_as::<_, CancelRow>(
|
|
||||||
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
|
|
||||||
)
|
|
||||||
.bind(job_id.as_str())
|
|
||||||
.fetch_optional(tx.as_mut())
|
|
||||||
.await?
|
|
||||||
else {
|
|
||||||
return Ok(CancelOutcome::NotFound);
|
|
||||||
};
|
|
||||||
let phase = parse_phase(&row.phase)?;
|
|
||||||
if phase.is_terminal() {
|
|
||||||
return Ok(CancelOutcome::AlreadyTerminal);
|
|
||||||
}
|
|
||||||
if row.cancel_requested {
|
|
||||||
return Ok(CancelOutcome::AlreadyRequested);
|
|
||||||
}
|
|
||||||
let now = OffsetDateTime::now_utc();
|
|
||||||
if phase == JobPhase::Running {
|
|
||||||
sqlx::query(
|
|
||||||
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
|
|
||||||
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
|
|
||||||
revision = revision + 1, updated_at = $2
|
|
||||||
WHERE job_id = $1",
|
|
||||||
)
|
|
||||||
.bind(job_id.as_str())
|
|
||||||
.bind(now)
|
|
||||||
.execute(tx.as_mut())
|
|
||||||
.await?;
|
|
||||||
} else {
|
|
||||||
sqlx::query(
|
|
||||||
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
|
|
||||||
authority_attempt_id = NULL, dispatch_needed = FALSE,
|
|
||||||
terminal_at = $2, coordination_cleaned_at = NULL,
|
|
||||||
revision = revision + 1, updated_at = $2
|
|
||||||
WHERE job_id = $1",
|
|
||||||
)
|
|
||||||
.bind(job_id.as_str())
|
|
||||||
.bind(now)
|
|
||||||
.execute(tx.as_mut())
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
|
|
||||||
Ok(CancelOutcome::Requested)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn cooperative_stop(
|
|
||||||
&self,
|
|
||||||
authority: &AttemptAuthority,
|
|
||||||
) -> Result<CooperativeStopOutcome, AttemptError> {
|
|
||||||
let authority_for_effect = authority.clone();
|
|
||||||
let coordination = self.coordination.clone();
|
|
||||||
let outcome = with_transaction(&self.pool, move |tx| {
|
|
||||||
Box::pin(async move {
|
|
||||||
cooperative_stop_transaction(tx, &coordination, &authority_for_effect).await
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
if outcome == CooperativeStopOutcome::Stopped {
|
|
||||||
let _ = self
|
|
||||||
.coordination
|
|
||||||
.release(authority.job_id.as_str(), &lease_owner(authority))
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
Ok(outcome)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn cooperative_stop_operation(
|
|
||||||
&self,
|
|
||||||
authority: &AttemptAuthority,
|
|
||||||
operation_id: &OpaqueId,
|
|
||||||
) -> Result<CooperativeStopOutcome, AttemptError> {
|
|
||||||
let request_fingerprint = fingerprint(&StopRequest {
|
|
||||||
job_id: &authority.job_id,
|
|
||||||
attempt_id: &authority.attempt_id,
|
|
||||||
fence: authority.fence,
|
|
||||||
worker_incarnation: &authority.worker_incarnation,
|
|
||||||
})
|
|
||||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
|
||||||
let authority_for_effect = authority.clone();
|
|
||||||
let coordination = self.coordination.clone();
|
|
||||||
let outcome = execute_mutation(
|
|
||||||
&self.pool,
|
|
||||||
MutationScope {
|
|
||||||
subject: &self.operation_subject,
|
|
||||||
operation: "attempt.mutation",
|
|
||||||
key: operation_id.as_str(),
|
|
||||||
},
|
|
||||||
"cooperative_stop",
|
|
||||||
&[],
|
|
||||||
request_fingerprint,
|
|
||||||
move |tx| {
|
|
||||||
Box::pin(async move {
|
|
||||||
let decision =
|
|
||||||
cooperative_stop_transaction(tx, &coordination, &authority_for_effect)
|
|
||||||
.await?;
|
|
||||||
let status = if decision == CooperativeStopOutcome::AuthorityMismatch {
|
|
||||||
MutationStatus::Rejected
|
|
||||||
} else {
|
|
||||||
MutationStatus::Accepted
|
|
||||||
};
|
|
||||||
Ok((
|
|
||||||
status,
|
|
||||||
serde_json::to_value(decision).map_err(DurableError::effect)?,
|
|
||||||
))
|
|
||||||
})
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
if outcome.disposition == MutationDisposition::Conflict {
|
|
||||||
return Err(AttemptError::OperationConflict);
|
|
||||||
}
|
|
||||||
let decision = serde_json::from_value(outcome.response)
|
|
||||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
|
||||||
if decision == CooperativeStopOutcome::Stopped {
|
|
||||||
let _ = self
|
|
||||||
.coordination
|
|
||||||
.release(authority.job_id.as_str(), &lease_owner(authority))
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
Ok(decision)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn mark_worker_down(
|
|
||||||
&self,
|
|
||||||
worker_incarnation: &OpaqueId,
|
|
||||||
) -> Result<u64, AttemptError> {
|
|
||||||
let changed = sqlx::query(
|
|
||||||
"UPDATE attempts SET worker_down_at = COALESCE(worker_down_at, $2)
|
|
||||||
WHERE worker_incarnation = $1 AND state IN ('authorized', 'executing')",
|
|
||||||
)
|
|
||||||
.bind(worker_incarnation.as_str())
|
|
||||||
.bind(OffsetDateTime::now_utc())
|
|
||||||
.execute(&self.pool)
|
|
||||||
.await?;
|
|
||||||
Ok(changed.rows_affected())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn register_staging_object(
|
pub async fn register_staging_object(
|
||||||
&self,
|
&self,
|
||||||
attempt_id: &OpaqueId,
|
attempt_id: &OpaqueId,
|
||||||
|
|
|
||||||
153
crates/remora-attempt/src/recovery/cancellation.rs
Normal file
153
crates/remora-attempt/src/recovery/cancellation.rs
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl AttemptService {
|
||||||
|
pub async fn request_cancel(&self, job_id: &OpaqueId) -> Result<CancelOutcome, AttemptError> {
|
||||||
|
with_transaction(&self.pool, |tx| {
|
||||||
|
let job_id = job_id.clone();
|
||||||
|
Box::pin(async move { Self::request_cancel_in_tx(tx, &job_id).await })
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn request_cancel_in_tx(
|
||||||
|
tx: &mut PgTx<'_>,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
) -> Result<CancelOutcome, DurableError> {
|
||||||
|
let Some(row) = sqlx::query_as::<_, CancelRow>(
|
||||||
|
"SELECT phase, cancel_requested FROM jobs WHERE job_id = $1 FOR UPDATE",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.fetch_optional(tx.as_mut())
|
||||||
|
.await?
|
||||||
|
else {
|
||||||
|
return Ok(CancelOutcome::NotFound);
|
||||||
|
};
|
||||||
|
let phase = parse_phase(&row.phase)?;
|
||||||
|
if phase.is_terminal() {
|
||||||
|
return Ok(CancelOutcome::AlreadyTerminal);
|
||||||
|
}
|
||||||
|
if row.cancel_requested {
|
||||||
|
return Ok(CancelOutcome::AlreadyRequested);
|
||||||
|
}
|
||||||
|
let now = OffsetDateTime::now_utc();
|
||||||
|
if phase == JobPhase::Running {
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelling',
|
||||||
|
dispatch_needed = FALSE, coordination_cleaned_at = NULL,
|
||||||
|
revision = revision + 1, updated_at = $2
|
||||||
|
WHERE job_id = $1",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.bind(now)
|
||||||
|
.execute(tx.as_mut())
|
||||||
|
.await?;
|
||||||
|
} else {
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE jobs SET cancel_requested = TRUE, phase = 'cancelled',
|
||||||
|
authority_attempt_id = NULL, dispatch_needed = FALSE,
|
||||||
|
terminal_at = $2, coordination_cleaned_at = NULL,
|
||||||
|
revision = revision + 1, updated_at = $2
|
||||||
|
WHERE job_id = $1",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.bind(now)
|
||||||
|
.execute(tx.as_mut())
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
|
||||||
|
Ok(CancelOutcome::Requested)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn cooperative_stop(
|
||||||
|
&self,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
) -> Result<CooperativeStopOutcome, AttemptError> {
|
||||||
|
let authority_for_effect = authority.clone();
|
||||||
|
let coordination = self.coordination.clone();
|
||||||
|
let outcome = with_transaction(&self.pool, move |tx| {
|
||||||
|
Box::pin(async move {
|
||||||
|
cooperative_stop_transaction(tx, &coordination, &authority_for_effect).await
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
if outcome == CooperativeStopOutcome::Stopped {
|
||||||
|
let _ = self
|
||||||
|
.coordination
|
||||||
|
.release(authority.job_id.as_str(), &lease_owner(authority))
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
Ok(outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn cooperative_stop_operation(
|
||||||
|
&self,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
operation_id: &OpaqueId,
|
||||||
|
) -> Result<CooperativeStopOutcome, AttemptError> {
|
||||||
|
let request_fingerprint = fingerprint(&StopRequest {
|
||||||
|
job_id: &authority.job_id,
|
||||||
|
attempt_id: &authority.attempt_id,
|
||||||
|
fence: authority.fence,
|
||||||
|
worker_incarnation: &authority.worker_incarnation,
|
||||||
|
})
|
||||||
|
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||||
|
let authority_for_effect = authority.clone();
|
||||||
|
let coordination = self.coordination.clone();
|
||||||
|
let outcome = execute_mutation(
|
||||||
|
&self.pool,
|
||||||
|
MutationScope {
|
||||||
|
subject: &self.operation_subject,
|
||||||
|
operation: "attempt.mutation",
|
||||||
|
key: operation_id.as_str(),
|
||||||
|
},
|
||||||
|
"cooperative_stop",
|
||||||
|
&[],
|
||||||
|
request_fingerprint,
|
||||||
|
move |tx| {
|
||||||
|
Box::pin(async move {
|
||||||
|
let decision =
|
||||||
|
cooperative_stop_transaction(tx, &coordination, &authority_for_effect)
|
||||||
|
.await?;
|
||||||
|
let status = if decision == CooperativeStopOutcome::AuthorityMismatch {
|
||||||
|
MutationStatus::Rejected
|
||||||
|
} else {
|
||||||
|
MutationStatus::Accepted
|
||||||
|
};
|
||||||
|
Ok((
|
||||||
|
status,
|
||||||
|
serde_json::to_value(decision).map_err(DurableError::effect)?,
|
||||||
|
))
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if outcome.disposition == MutationDisposition::Conflict {
|
||||||
|
return Err(AttemptError::OperationConflict);
|
||||||
|
}
|
||||||
|
let decision = serde_json::from_value(outcome.response)
|
||||||
|
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||||
|
if decision == CooperativeStopOutcome::Stopped {
|
||||||
|
let _ = self
|
||||||
|
.coordination
|
||||||
|
.release(authority.job_id.as_str(), &lease_owner(authority))
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
Ok(decision)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn mark_worker_down(
|
||||||
|
&self,
|
||||||
|
worker_incarnation: &OpaqueId,
|
||||||
|
) -> Result<u64, AttemptError> {
|
||||||
|
let changed = sqlx::query(
|
||||||
|
"UPDATE attempts SET worker_down_at = COALESCE(worker_down_at, $2)
|
||||||
|
WHERE worker_incarnation = $1 AND state IN ('authorized', 'executing')",
|
||||||
|
)
|
||||||
|
.bind(worker_incarnation.as_str())
|
||||||
|
.bind(OffsetDateTime::now_utc())
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await?;
|
||||||
|
Ok(changed.rows_affected())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -16,6 +16,8 @@ use crate::types::{
|
||||||
};
|
};
|
||||||
use crate::vault::{EmptyPathFence, VaultClient};
|
use crate::vault::{EmptyPathFence, VaultClient};
|
||||||
|
|
||||||
|
mod query;
|
||||||
|
|
||||||
/// Durable source-credential registry with Vault-backed secret versions.
|
/// Durable source-credential registry with Vault-backed secret versions.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct CredentialRegistry {
|
pub struct CredentialRegistry {
|
||||||
|
|
@ -417,122 +419,6 @@ impl CredentialRegistry {
|
||||||
.ok_or_else(|| DurableError::Effect("revoked credential is missing".into()))
|
.ok_or_else(|| DurableError::Effect("revoked credential is missing".into()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get(
|
|
||||||
&self,
|
|
||||||
caller: &CredentialCaller,
|
|
||||||
credential_id: &OpaqueId,
|
|
||||||
) -> Result<Option<SourceCredential>, CredentialError> {
|
|
||||||
let record = self.get_internal(credential_id).await?;
|
|
||||||
match require_manage(caller, record.as_ref()) {
|
|
||||||
Ok(visible) => Ok(Some(visible)),
|
|
||||||
Err(CredentialError::NotFound) => Ok(None),
|
|
||||||
Err(error) => Err(error),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list(
|
|
||||||
&self,
|
|
||||||
caller: &CredentialCaller,
|
|
||||||
limit: u32,
|
|
||||||
cursor: Option<&str>,
|
|
||||||
) -> Result<SourceCredentialPage, CredentialError> {
|
|
||||||
let limit = limit.clamp(1, 100) as i64;
|
|
||||||
let after = cursor.map(decode_cursor).transpose()?;
|
|
||||||
let owner_filter = if caller.administrator {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some(caller.principal_id.as_str().to_owned())
|
|
||||||
};
|
|
||||||
|
|
||||||
let rows = if let Some(owner) = owner_filter.as_deref() {
|
|
||||||
if let Some((created_at, id)) = after {
|
|
||||||
sqlx::query_as::<_, CredentialRow>(
|
|
||||||
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
|
||||||
c.created_at, c.updated_at,
|
|
||||||
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
|
||||||
FROM source_credentials c
|
|
||||||
LEFT JOIN source_credential_versions v
|
|
||||||
ON v.version_id = c.active_version_id
|
|
||||||
WHERE c.owner_id = $1
|
|
||||||
AND (c.created_at, c.credential_id) < ($2, $3)
|
|
||||||
ORDER BY c.created_at DESC, c.credential_id DESC
|
|
||||||
LIMIT $4",
|
|
||||||
)
|
|
||||||
.bind(owner)
|
|
||||||
.bind(created_at)
|
|
||||||
.bind(id)
|
|
||||||
.bind(limit + 1)
|
|
||||||
.fetch_all(&self.pool)
|
|
||||||
.await
|
|
||||||
.map_err(DurableError::from)?
|
|
||||||
} else {
|
|
||||||
sqlx::query_as::<_, CredentialRow>(
|
|
||||||
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
|
||||||
c.created_at, c.updated_at,
|
|
||||||
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
|
||||||
FROM source_credentials c
|
|
||||||
LEFT JOIN source_credential_versions v
|
|
||||||
ON v.version_id = c.active_version_id
|
|
||||||
WHERE c.owner_id = $1
|
|
||||||
ORDER BY c.created_at DESC, c.credential_id DESC
|
|
||||||
LIMIT $2",
|
|
||||||
)
|
|
||||||
.bind(owner)
|
|
||||||
.bind(limit + 1)
|
|
||||||
.fetch_all(&self.pool)
|
|
||||||
.await
|
|
||||||
.map_err(DurableError::from)?
|
|
||||||
}
|
|
||||||
} else if let Some((created_at, id)) = after {
|
|
||||||
sqlx::query_as::<_, CredentialRow>(
|
|
||||||
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
|
||||||
c.created_at, c.updated_at,
|
|
||||||
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
|
||||||
FROM source_credentials c
|
|
||||||
LEFT JOIN source_credential_versions v
|
|
||||||
ON v.version_id = c.active_version_id
|
|
||||||
WHERE (c.created_at, c.credential_id) < ($1, $2)
|
|
||||||
ORDER BY c.created_at DESC, c.credential_id DESC
|
|
||||||
LIMIT $3",
|
|
||||||
)
|
|
||||||
.bind(created_at)
|
|
||||||
.bind(id)
|
|
||||||
.bind(limit + 1)
|
|
||||||
.fetch_all(&self.pool)
|
|
||||||
.await
|
|
||||||
.map_err(DurableError::from)?
|
|
||||||
} else {
|
|
||||||
sqlx::query_as::<_, CredentialRow>(
|
|
||||||
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
|
||||||
c.created_at, c.updated_at,
|
|
||||||
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
|
||||||
FROM source_credentials c
|
|
||||||
LEFT JOIN source_credential_versions v
|
|
||||||
ON v.version_id = c.active_version_id
|
|
||||||
ORDER BY c.created_at DESC, c.credential_id DESC
|
|
||||||
LIMIT $1",
|
|
||||||
)
|
|
||||||
.bind(limit + 1)
|
|
||||||
.fetch_all(&self.pool)
|
|
||||||
.await
|
|
||||||
.map_err(DurableError::from)?
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut items: Vec<SourceCredential> = Vec::with_capacity(rows.len().min(limit as usize));
|
|
||||||
let mut next_cursor = None;
|
|
||||||
for (index, row) in rows.into_iter().enumerate() {
|
|
||||||
if index as i64 >= limit {
|
|
||||||
let last = items
|
|
||||||
.last()
|
|
||||||
.ok_or_else(|| CredentialError::Config("page overflow without items".into()))?;
|
|
||||||
next_cursor = Some(encode_cursor(last.created_at, last.id.as_str()));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
items.push(row.into_record()?);
|
|
||||||
}
|
|
||||||
Ok(SourceCredentialPage { items, next_cursor })
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Pin the current authorized version for a new execution grant.
|
/// Pin the current authorized version for a new execution grant.
|
||||||
///
|
///
|
||||||
/// Serializes with revoke/rotate on the credential advisory lock so a
|
/// Serializes with revoke/rotate on the credential advisory lock so a
|
||||||
|
|
|
||||||
119
crates/remora-credentials/src/registry/query.rs
Normal file
119
crates/remora-credentials/src/registry/query.rs
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl CredentialRegistry {
|
||||||
|
pub async fn get(
|
||||||
|
&self,
|
||||||
|
caller: &CredentialCaller,
|
||||||
|
credential_id: &OpaqueId,
|
||||||
|
) -> Result<Option<SourceCredential>, CredentialError> {
|
||||||
|
let record = self.get_internal(credential_id).await?;
|
||||||
|
match require_manage(caller, record.as_ref()) {
|
||||||
|
Ok(visible) => Ok(Some(visible)),
|
||||||
|
Err(CredentialError::NotFound) => Ok(None),
|
||||||
|
Err(error) => Err(error),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list(
|
||||||
|
&self,
|
||||||
|
caller: &CredentialCaller,
|
||||||
|
limit: u32,
|
||||||
|
cursor: Option<&str>,
|
||||||
|
) -> Result<SourceCredentialPage, CredentialError> {
|
||||||
|
let limit = limit.clamp(1, 100) as i64;
|
||||||
|
let after = cursor.map(decode_cursor).transpose()?;
|
||||||
|
let owner_filter = if caller.administrator {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(caller.principal_id.as_str().to_owned())
|
||||||
|
};
|
||||||
|
|
||||||
|
let rows = if let Some(owner) = owner_filter.as_deref() {
|
||||||
|
if let Some((created_at, id)) = after {
|
||||||
|
sqlx::query_as::<_, CredentialRow>(
|
||||||
|
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
||||||
|
c.created_at, c.updated_at,
|
||||||
|
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
||||||
|
FROM source_credentials c
|
||||||
|
LEFT JOIN source_credential_versions v
|
||||||
|
ON v.version_id = c.active_version_id
|
||||||
|
WHERE c.owner_id = $1
|
||||||
|
AND (c.created_at, c.credential_id) < ($2, $3)
|
||||||
|
ORDER BY c.created_at DESC, c.credential_id DESC
|
||||||
|
LIMIT $4",
|
||||||
|
)
|
||||||
|
.bind(owner)
|
||||||
|
.bind(created_at)
|
||||||
|
.bind(id)
|
||||||
|
.bind(limit + 1)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::from)?
|
||||||
|
} else {
|
||||||
|
sqlx::query_as::<_, CredentialRow>(
|
||||||
|
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
||||||
|
c.created_at, c.updated_at,
|
||||||
|
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
||||||
|
FROM source_credentials c
|
||||||
|
LEFT JOIN source_credential_versions v
|
||||||
|
ON v.version_id = c.active_version_id
|
||||||
|
WHERE c.owner_id = $1
|
||||||
|
ORDER BY c.created_at DESC, c.credential_id DESC
|
||||||
|
LIMIT $2",
|
||||||
|
)
|
||||||
|
.bind(owner)
|
||||||
|
.bind(limit + 1)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::from)?
|
||||||
|
}
|
||||||
|
} else if let Some((created_at, id)) = after {
|
||||||
|
sqlx::query_as::<_, CredentialRow>(
|
||||||
|
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
||||||
|
c.created_at, c.updated_at,
|
||||||
|
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
||||||
|
FROM source_credentials c
|
||||||
|
LEFT JOIN source_credential_versions v
|
||||||
|
ON v.version_id = c.active_version_id
|
||||||
|
WHERE (c.created_at, c.credential_id) < ($1, $2)
|
||||||
|
ORDER BY c.created_at DESC, c.credential_id DESC
|
||||||
|
LIMIT $3",
|
||||||
|
)
|
||||||
|
.bind(created_at)
|
||||||
|
.bind(id)
|
||||||
|
.bind(limit + 1)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::from)?
|
||||||
|
} else {
|
||||||
|
sqlx::query_as::<_, CredentialRow>(
|
||||||
|
"SELECT c.credential_id, c.owner_id, c.label, c.kind, c.status,
|
||||||
|
c.created_at, c.updated_at,
|
||||||
|
v.version_id AS active_version_id, v.created_at AS active_version_created_at
|
||||||
|
FROM source_credentials c
|
||||||
|
LEFT JOIN source_credential_versions v
|
||||||
|
ON v.version_id = c.active_version_id
|
||||||
|
ORDER BY c.created_at DESC, c.credential_id DESC
|
||||||
|
LIMIT $1",
|
||||||
|
)
|
||||||
|
.bind(limit + 1)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::from)?
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut items: Vec<SourceCredential> = Vec::with_capacity(rows.len().min(limit as usize));
|
||||||
|
let mut next_cursor = None;
|
||||||
|
for (index, row) in rows.into_iter().enumerate() {
|
||||||
|
if index as i64 >= limit {
|
||||||
|
let last = items
|
||||||
|
.last()
|
||||||
|
.ok_or_else(|| CredentialError::Config("page overflow without items".into()))?;
|
||||||
|
next_cursor = Some(encode_cursor(last.created_at, last.id.as_str()));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
items.push(row.into_record()?);
|
||||||
|
}
|
||||||
|
Ok(SourceCredentialPage { items, next_cursor })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -21,6 +21,8 @@ use crate::{
|
||||||
TargetRegistry,
|
TargetRegistry,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod lifecycle;
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct DeletionService {
|
pub struct DeletionService {
|
||||||
pool: PgPool,
|
pool: PgPool,
|
||||||
|
|
@ -46,156 +48,6 @@ impl DeletionService {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn soft_delete(
|
|
||||||
&self,
|
|
||||||
principal: &Principal,
|
|
||||||
job_id: &OpaqueId,
|
|
||||||
) -> Result<LifecycleMutation, DeletionError> {
|
|
||||||
let mut tx = self.pool.begin().await?;
|
|
||||||
let outcome = Self::soft_delete_in_tx(&mut tx, principal, job_id).await?;
|
|
||||||
tx.commit().await?;
|
|
||||||
Ok(outcome)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn soft_delete_in_tx(
|
|
||||||
tx: &mut PgTx<'_>,
|
|
||||||
principal: &Principal,
|
|
||||||
job_id: &OpaqueId,
|
|
||||||
) -> Result<LifecycleMutation, DeletionError> {
|
|
||||||
let row = lock_job(tx, job_id).await?;
|
|
||||||
authorize(principal, &row, ManagementOperation::SoftDelete)?;
|
|
||||||
require_terminal(&row.phase)?;
|
|
||||||
if row.deletion_operation_id.is_some() {
|
|
||||||
return Err(DeletionError::HardDeleteStarted);
|
|
||||||
}
|
|
||||||
if row.lifecycle == "soft_deleted" {
|
|
||||||
return Ok(LifecycleMutation::Unchanged);
|
|
||||||
}
|
|
||||||
let now = OffsetDateTime::now_utc();
|
|
||||||
sqlx::query(
|
|
||||||
"UPDATE jobs
|
|
||||||
SET lifecycle = 'soft_deleted', revision = revision + 1, updated_at = $2
|
|
||||||
WHERE job_id = $1",
|
|
||||||
)
|
|
||||||
.bind(job_id.as_str())
|
|
||||||
.bind(now)
|
|
||||||
.execute(tx.as_mut())
|
|
||||||
.await?;
|
|
||||||
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
|
|
||||||
Ok(LifecycleMutation::Changed)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn restore(
|
|
||||||
&self,
|
|
||||||
principal: &Principal,
|
|
||||||
job_id: &OpaqueId,
|
|
||||||
) -> Result<LifecycleMutation, DeletionError> {
|
|
||||||
let mut tx = self.pool.begin().await?;
|
|
||||||
let outcome = Self::restore_in_tx(&mut tx, principal, job_id).await?;
|
|
||||||
tx.commit().await?;
|
|
||||||
Ok(outcome)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn restore_in_tx(
|
|
||||||
tx: &mut PgTx<'_>,
|
|
||||||
principal: &Principal,
|
|
||||||
job_id: &OpaqueId,
|
|
||||||
) -> Result<LifecycleMutation, DeletionError> {
|
|
||||||
let row = lock_job(tx, job_id).await?;
|
|
||||||
authorize(principal, &row, ManagementOperation::Restore)?;
|
|
||||||
require_terminal(&row.phase)?;
|
|
||||||
if row.deletion_operation_id.is_some() {
|
|
||||||
return Err(DeletionError::HardDeleteStarted);
|
|
||||||
}
|
|
||||||
if row.lifecycle == "visible" {
|
|
||||||
return Ok(LifecycleMutation::Unchanged);
|
|
||||||
}
|
|
||||||
let now = OffsetDateTime::now_utc();
|
|
||||||
sqlx::query(
|
|
||||||
"UPDATE jobs
|
|
||||||
SET lifecycle = 'visible', revision = revision + 1, updated_at = $2
|
|
||||||
WHERE job_id = $1",
|
|
||||||
)
|
|
||||||
.bind(job_id.as_str())
|
|
||||||
.bind(now)
|
|
||||||
.execute(tx.as_mut())
|
|
||||||
.await?;
|
|
||||||
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
|
|
||||||
Ok(LifecycleMutation::Changed)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn hard_delete(
|
|
||||||
&self,
|
|
||||||
principal: &Principal,
|
|
||||||
job_id: &OpaqueId,
|
|
||||||
) -> Result<HardDeleteOutcome, DeletionError> {
|
|
||||||
let mut tx = self.pool.begin().await?;
|
|
||||||
let outcome = Self::hard_delete_in_tx(&mut tx, principal, job_id).await?;
|
|
||||||
tx.commit().await?;
|
|
||||||
Ok(outcome)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn hard_delete_in_tx(
|
|
||||||
tx: &mut PgTx<'_>,
|
|
||||||
principal: &Principal,
|
|
||||||
job_id: &OpaqueId,
|
|
||||||
) -> Result<HardDeleteOutcome, DeletionError> {
|
|
||||||
let operation_id = new_id('d')?;
|
|
||||||
let row = lock_job(tx, job_id).await?;
|
|
||||||
authorize(principal, &row, ManagementOperation::HardDelete)?;
|
|
||||||
require_terminal(&row.phase)?;
|
|
||||||
if let Some(existing) = row.deletion_operation_id {
|
|
||||||
let operation = load_operation_in_tx(tx, &existing).await?;
|
|
||||||
return Ok(HardDeleteOutcome {
|
|
||||||
mutation: LifecycleMutation::Unchanged,
|
|
||||||
operation,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
validate_committed_inventory(tx, &row).await?;
|
|
||||||
validate_deletion_identities(tx, &row).await?;
|
|
||||||
let now = OffsetDateTime::now_utc();
|
|
||||||
sqlx::query(
|
|
||||||
"INSERT INTO deletion_operations (
|
|
||||||
operation_id, job_id, state, created_at, updated_at
|
|
||||||
) VALUES ($1, $2, 'queued', $3, $3)",
|
|
||||||
)
|
|
||||||
.bind(operation_id.as_str())
|
|
||||||
.bind(job_id.as_str())
|
|
||||||
.bind(now)
|
|
||||||
.execute(tx.as_mut())
|
|
||||||
.await?;
|
|
||||||
sqlx::query(
|
|
||||||
"UPDATE jobs
|
|
||||||
SET lifecycle = 'hard_deleting', deletion_operation_id = $2,
|
|
||||||
hard_delete_started_at = $3, revision = revision + 1, updated_at = $3
|
|
||||||
WHERE job_id = $1",
|
|
||||||
)
|
|
||||||
.bind(job_id.as_str())
|
|
||||||
.bind(operation_id.as_str())
|
|
||||||
.bind(now)
|
|
||||||
.execute(tx.as_mut())
|
|
||||||
.await?;
|
|
||||||
materialize_items(tx, job_id, &operation_id, now).await?;
|
|
||||||
finalize_if_empty(tx, job_id, &operation_id, now).await?;
|
|
||||||
enqueue(
|
|
||||||
tx,
|
|
||||||
"job.updated",
|
|
||||||
json!({ "job_id": job_id, "deletion_operation_id": operation_id }),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
enqueue(
|
|
||||||
tx,
|
|
||||||
"deletion.updated",
|
|
||||||
json!({ "job_id": job_id, "operation_id": operation_id }),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
let operation = load_operation_in_tx(tx, operation_id.as_str()).await?;
|
|
||||||
Ok(HardDeleteOutcome {
|
|
||||||
mutation: LifecycleMutation::Changed,
|
|
||||||
operation,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn operation(
|
pub async fn operation(
|
||||||
&self,
|
&self,
|
||||||
principal: &Principal,
|
principal: &Principal,
|
||||||
|
|
|
||||||
153
crates/remora-deletion/src/service/lifecycle.rs
Normal file
153
crates/remora-deletion/src/service/lifecycle.rs
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl DeletionService {
|
||||||
|
pub async fn soft_delete(
|
||||||
|
&self,
|
||||||
|
principal: &Principal,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
) -> Result<LifecycleMutation, DeletionError> {
|
||||||
|
let mut tx = self.pool.begin().await?;
|
||||||
|
let outcome = Self::soft_delete_in_tx(&mut tx, principal, job_id).await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn soft_delete_in_tx(
|
||||||
|
tx: &mut PgTx<'_>,
|
||||||
|
principal: &Principal,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
) -> Result<LifecycleMutation, DeletionError> {
|
||||||
|
let row = lock_job(tx, job_id).await?;
|
||||||
|
authorize(principal, &row, ManagementOperation::SoftDelete)?;
|
||||||
|
require_terminal(&row.phase)?;
|
||||||
|
if row.deletion_operation_id.is_some() {
|
||||||
|
return Err(DeletionError::HardDeleteStarted);
|
||||||
|
}
|
||||||
|
if row.lifecycle == "soft_deleted" {
|
||||||
|
return Ok(LifecycleMutation::Unchanged);
|
||||||
|
}
|
||||||
|
let now = OffsetDateTime::now_utc();
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE jobs
|
||||||
|
SET lifecycle = 'soft_deleted', revision = revision + 1, updated_at = $2
|
||||||
|
WHERE job_id = $1",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.bind(now)
|
||||||
|
.execute(tx.as_mut())
|
||||||
|
.await?;
|
||||||
|
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
|
||||||
|
Ok(LifecycleMutation::Changed)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn restore(
|
||||||
|
&self,
|
||||||
|
principal: &Principal,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
) -> Result<LifecycleMutation, DeletionError> {
|
||||||
|
let mut tx = self.pool.begin().await?;
|
||||||
|
let outcome = Self::restore_in_tx(&mut tx, principal, job_id).await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn restore_in_tx(
|
||||||
|
tx: &mut PgTx<'_>,
|
||||||
|
principal: &Principal,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
) -> Result<LifecycleMutation, DeletionError> {
|
||||||
|
let row = lock_job(tx, job_id).await?;
|
||||||
|
authorize(principal, &row, ManagementOperation::Restore)?;
|
||||||
|
require_terminal(&row.phase)?;
|
||||||
|
if row.deletion_operation_id.is_some() {
|
||||||
|
return Err(DeletionError::HardDeleteStarted);
|
||||||
|
}
|
||||||
|
if row.lifecycle == "visible" {
|
||||||
|
return Ok(LifecycleMutation::Unchanged);
|
||||||
|
}
|
||||||
|
let now = OffsetDateTime::now_utc();
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE jobs
|
||||||
|
SET lifecycle = 'visible', revision = revision + 1, updated_at = $2
|
||||||
|
WHERE job_id = $1",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.bind(now)
|
||||||
|
.execute(tx.as_mut())
|
||||||
|
.await?;
|
||||||
|
enqueue(tx, "job.updated", json!({ "job_id": job_id })).await?;
|
||||||
|
Ok(LifecycleMutation::Changed)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn hard_delete(
|
||||||
|
&self,
|
||||||
|
principal: &Principal,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
) -> Result<HardDeleteOutcome, DeletionError> {
|
||||||
|
let mut tx = self.pool.begin().await?;
|
||||||
|
let outcome = Self::hard_delete_in_tx(&mut tx, principal, job_id).await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(outcome)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn hard_delete_in_tx(
|
||||||
|
tx: &mut PgTx<'_>,
|
||||||
|
principal: &Principal,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
) -> Result<HardDeleteOutcome, DeletionError> {
|
||||||
|
let operation_id = new_id('d')?;
|
||||||
|
let row = lock_job(tx, job_id).await?;
|
||||||
|
authorize(principal, &row, ManagementOperation::HardDelete)?;
|
||||||
|
require_terminal(&row.phase)?;
|
||||||
|
if let Some(existing) = row.deletion_operation_id {
|
||||||
|
let operation = load_operation_in_tx(tx, &existing).await?;
|
||||||
|
return Ok(HardDeleteOutcome {
|
||||||
|
mutation: LifecycleMutation::Unchanged,
|
||||||
|
operation,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
validate_committed_inventory(tx, &row).await?;
|
||||||
|
validate_deletion_identities(tx, &row).await?;
|
||||||
|
let now = OffsetDateTime::now_utc();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO deletion_operations (
|
||||||
|
operation_id, job_id, state, created_at, updated_at
|
||||||
|
) VALUES ($1, $2, 'queued', $3, $3)",
|
||||||
|
)
|
||||||
|
.bind(operation_id.as_str())
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.bind(now)
|
||||||
|
.execute(tx.as_mut())
|
||||||
|
.await?;
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE jobs
|
||||||
|
SET lifecycle = 'hard_deleting', deletion_operation_id = $2,
|
||||||
|
hard_delete_started_at = $3, revision = revision + 1, updated_at = $3
|
||||||
|
WHERE job_id = $1",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.bind(operation_id.as_str())
|
||||||
|
.bind(now)
|
||||||
|
.execute(tx.as_mut())
|
||||||
|
.await?;
|
||||||
|
materialize_items(tx, job_id, &operation_id, now).await?;
|
||||||
|
finalize_if_empty(tx, job_id, &operation_id, now).await?;
|
||||||
|
enqueue(
|
||||||
|
tx,
|
||||||
|
"job.updated",
|
||||||
|
json!({ "job_id": job_id, "deletion_operation_id": operation_id }),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
enqueue(
|
||||||
|
tx,
|
||||||
|
"deletion.updated",
|
||||||
|
json!({ "job_id": job_id, "operation_id": operation_id }),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let operation = load_operation_in_tx(tx, operation_id.as_str()).await?;
|
||||||
|
Ok(HardDeleteOutcome {
|
||||||
|
mutation: LifecycleMutation::Changed,
|
||||||
|
operation,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -46,6 +46,8 @@ use crate::{
|
||||||
WorkerError,
|
WorkerError,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
mod source_execution;
|
||||||
|
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct ObjectSourceEndpoint {
|
pub struct ObjectSourceEndpoint {
|
||||||
pub endpoint: String,
|
pub endpoint: String,
|
||||||
|
|
@ -159,20 +161,8 @@ impl AdapterExecutor {
|
||||||
resume: &resume,
|
resume: &resume,
|
||||||
};
|
};
|
||||||
|
|
||||||
match source {
|
self.execute_source(context, envelope, source, secret, &resources)
|
||||||
SourceSubmissionDto::Http(source) => {
|
|
||||||
self.execute_http(context, envelope, source, secret, &resources)
|
|
||||||
.await?;
|
.await?;
|
||||||
}
|
|
||||||
SourceSubmissionDto::ObjectStore(source) => {
|
|
||||||
self.execute_object(context, envelope, source, secret, &resources)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
SourceSubmissionDto::HuggingFace(source) => {
|
|
||||||
self.execute_hugging_face(context, envelope, source, secret, &resources)
|
|
||||||
.await?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
context
|
context
|
||||||
.require_live_authority()
|
.require_live_authority()
|
||||||
|
|
|
||||||
31
crates/remora-worker/src/executor/source_execution.rs
Normal file
31
crates/remora-worker/src/executor/source_execution.rs
Normal file
|
|
@ -0,0 +1,31 @@
|
||||||
|
use remora_contracts::{ExecutionEnvelopeDto, SourceSubmissionDto};
|
||||||
|
use remora_credentials::SecretMaterial;
|
||||||
|
|
||||||
|
use super::{AdapterExecutor, AdapterFailure, ExecutionResources};
|
||||||
|
use crate::AttemptContext;
|
||||||
|
|
||||||
|
impl AdapterExecutor {
|
||||||
|
pub(super) async fn execute_source(
|
||||||
|
&self,
|
||||||
|
context: &AttemptContext,
|
||||||
|
envelope: &ExecutionEnvelopeDto,
|
||||||
|
source: SourceSubmissionDto,
|
||||||
|
secret: Option<SecretMaterial>,
|
||||||
|
resources: &ExecutionResources<'_>,
|
||||||
|
) -> Result<(), AdapterFailure> {
|
||||||
|
match source {
|
||||||
|
SourceSubmissionDto::Http(source) => {
|
||||||
|
self.execute_http(context, envelope, source, secret, resources)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
SourceSubmissionDto::ObjectStore(source) => {
|
||||||
|
self.execute_object(context, envelope, source, secret, resources)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
SourceSubmissionDto::HuggingFace(source) => {
|
||||||
|
self.execute_hugging_face(context, envelope, source, secret, resources)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue