Add Rust domain kernel and wire contracts

This commit is contained in:
ms2k 2026-07-15 21:00:33 +09:00
commit 4da5a980ed
7 changed files with 3757 additions and 1 deletions

View file

@ -0,0 +1,149 @@
use std::collections::BTreeMap;
use remora_domain::OpaqueId;
use serde::{Deserialize, Serialize};
use crate::{DateTimeDto, NullableDto};
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceCredentialKindDto {
BasicAuth,
BearerToken,
HttpHeaders,
ObjectStore,
HuggingFace,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BasicAuthMaterialDto {
pub username: String,
pub password: String,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct TokenMaterialDto {
pub token: String,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HttpHeadersMaterialDto {
pub headers: BTreeMap<String, String>,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ObjectStoreCredentialMaterialDto {
pub access_key_id: String,
pub secret_access_key: String,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub session_token: Option<String>,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(untagged)]
pub enum CredentialMaterialDto {
BasicAuth(BasicAuthMaterialDto),
Token(TokenMaterialDto),
HttpHeaders(HttpHeadersMaterialDto),
ObjectStore(ObjectStoreCredentialMaterialDto),
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum SourceCredentialCreateDto {
BasicAuth(BasicAuthCredentialCreateDto),
BearerToken(BearerTokenCredentialCreateDto),
HttpHeaders(HttpHeadersCredentialCreateDto),
ObjectStore(ObjectStoreCredentialCreateDto),
HuggingFace(HuggingFaceCredentialCreateDto),
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BasicAuthCredentialCreateDto {
pub label: String,
pub material: BasicAuthMaterialDto,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BearerTokenCredentialCreateDto {
pub label: String,
pub material: TokenMaterialDto,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HttpHeadersCredentialCreateDto {
pub label: String,
pub material: HttpHeadersMaterialDto,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ObjectStoreCredentialCreateDto {
pub label: String,
pub material: ObjectStoreCredentialMaterialDto,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HuggingFaceCredentialCreateDto {
pub label: String,
pub material: TokenMaterialDto,
}
#[derive(Clone, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SourceCredentialRotateDto {
pub material: CredentialMaterialDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SourceCredentialVersionDto {
pub id: OpaqueId,
pub created_at: DateTimeDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CredentialGrantVersionDto {
pub credential_id: OpaqueId,
pub version_id: OpaqueId,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceCredentialStatusDto {
Active,
Revoked,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SourceCredentialDto {
pub id: OpaqueId,
pub owner: OpaqueId,
pub label: String,
pub kind: SourceCredentialKindDto,
pub status: SourceCredentialStatusDto,
pub active_version: NullableDto<SourceCredentialVersionDto>,
pub created_at: DateTimeDto,
pub updated_at: DateTimeDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SourceCredentialPageDto {
pub items: Vec<SourceCredentialDto>,
pub next_cursor: NullableDto<String>,
}

View file

@ -0,0 +1,356 @@
use remora_domain::{JobPhase, OpaqueId, RuntimeFailureClass};
use serde::{Deserialize, Serialize};
use crate::{DateTimeDto, PositiveIntegerDto, SafeIntegerDto, SignedSafeIntegerDto};
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct TargetDto {
pub store: String,
pub bucket: String,
pub prefix: String,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct TargetOverrideDto {
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub bucket: Option<String>,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub prefix: Option<String>,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum SourceSubmissionDto {
Http(HttpSourceSubmissionDto),
ObjectStore(ObjectStoreSourceSubmissionDto),
HuggingFace(HuggingFaceSourceSubmissionDto),
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HttpSourceSubmissionDto {
pub url: String,
pub result_path: String,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub credential_id: Option<OpaqueId>,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ObjectStoreSourceSubmissionDto {
pub store: String,
pub bucket: String,
pub key: String,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub version_id: Option<String>,
pub result_path: String,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub credential_id: Option<OpaqueId>,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HuggingFaceSelectionDto {
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub paths: Option<Vec<String>>,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub prefixes: Option<Vec<String>>,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HuggingFaceSourceSubmissionDto {
pub repository_type: HuggingFaceRepositoryTypeDto,
pub repository_id: String,
pub revision: String,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub selection: Option<HuggingFaceSelectionDto>,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub credential_id: Option<OpaqueId>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum HuggingFaceRepositoryTypeDto {
Model,
Dataset,
Space,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct JobSubmissionDto {
pub source: SourceSubmissionDto,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub target: Option<TargetOverrideDto>,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub label: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptAuthorityDto {
pub attempt_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ProgressDto {
pub attempt_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub completed_bytes: SafeIntegerDto,
pub total_bytes: Option<SafeIntegerDto>,
pub completed_members: SafeIntegerDto,
pub total_members: Option<SafeIntegerDto>,
pub observed_at: DateTimeDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(untagged)]
pub enum JobFailureDto {
Permanent(PermanentJobFailureDto),
Retryable(RetryableJobFailureDto),
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct PermanentJobFailureDto {
pub class: PermanentFailureClassDto,
pub message: String,
pub retryable: PermanentDispositionDto,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub safe_details: Option<std::collections::BTreeMap<String, SafeDetailValueDto>>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RetryableJobFailureDto {
pub class: RetryableFailureClassDto,
pub message: String,
pub retryable: RetryableDispositionDto,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub safe_details: Option<std::collections::BTreeMap<String, SafeDetailValueDto>>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum PermanentFailureClassDto {
Validation,
Authorization,
SourceNotFound,
SourceChanged,
Cancelled,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum RetryableFailureClassDto {
TransientSource,
TransientTarget,
Integrity,
Capacity,
Internal,
LostAttempt,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PermanentDispositionDto;
impl Serialize for PermanentDispositionDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_bool(false)
}
}
impl<'de> Deserialize<'de> for PermanentDispositionDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
if bool::deserialize(deserializer)? {
return Err(serde::de::Error::custom(
"permanent failure must not be retryable",
));
}
Ok(Self)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct RetryableDispositionDto;
impl Serialize for RetryableDispositionDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_bool(true)
}
}
impl<'de> Deserialize<'de> for RetryableDispositionDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
if !bool::deserialize(deserializer)? {
return Err(serde::de::Error::custom(
"retryable failure must be retryable",
));
}
Ok(Self)
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptOutcomeRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub outcome: AttemptOutcomeDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum AttemptOutcomeDto {
Success(AttemptSuccessOutcomeDto),
Failure(AttemptFailureOutcomeDto),
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptSuccessOutcomeDto {
pub result: ResultDescriptorDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptFailureOutcomeDto {
pub failure: AttemptRuntimeFailureDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptRuntimeFailureDto {
pub class: RuntimeFailureClass,
pub message: String,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub safe_details: Option<std::collections::BTreeMap<String, SafeDetailValueDto>>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(untagged)]
pub enum SafeDetailValueDto {
String(String),
Integer(SignedSafeIntegerDto),
Boolean(bool),
Null,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ResultDescriptorDto {
pub kind: ResultKindDto,
pub manifest: ObjectIdentityDto,
pub member_count: PositiveIntegerDto,
pub total_bytes: SafeIntegerDto,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ResultKindDto {
Manifest,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ObjectIdentityDto {
pub store: String,
pub bucket: String,
pub key: String,
#[serde(
default,
deserialize_with = "optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub version_id: Option<String>,
pub checksum: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct JobPhaseProjectionDto {
pub phase: JobPhase,
}
pub(crate) fn optional_non_null<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>
where
D: serde::Deserializer<'de>,
T: Deserialize<'de>,
{
T::deserialize(deserializer).map(Some)
}

View file

@ -0,0 +1,114 @@
use remora_domain::{JobLifecycle, JobPhase, OpaqueId};
use serde::{Deserialize, Serialize};
use crate::{
CursorDto, DateTimeDto, NullableDto, PositiveIntegerDto, SafeIntegerDto, SchemaVersionDto,
wire::optional_nullable,
};
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum EventFrameDto {
JobChanged(JobChangedDto),
DeletionChanged(DeletionChangedDto),
ResyncRequired(ResyncRequiredDto),
Heartbeat(HeartbeatDto),
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct JobChangedDto {
pub schema_version: SchemaVersionDto,
pub event_id: OpaqueId,
pub sequence: PositiveIntegerDto,
pub cursor: CursorDto,
pub occurred_at: DateTimeDto,
pub job_id: OpaqueId,
pub job_revision: PositiveIntegerDto,
pub phase: JobPhase,
pub lifecycle: JobLifecycle,
pub authoritative_attempt: NullableDto<EventAttemptDto>,
pub progress: NullableDto<EventProgressDto>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EventAttemptDto {
pub attempt_id: OpaqueId,
pub fence: PositiveIntegerDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EventProgressDto {
pub attempt_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub completed_bytes: SafeIntegerDto,
#[serde(
default,
deserialize_with = "optional_nullable",
skip_serializing_if = "Option::is_none"
)]
pub total_bytes: Option<NullableDto<SafeIntegerDto>>,
pub completed_members: SafeIntegerDto,
#[serde(
default,
deserialize_with = "optional_nullable",
skip_serializing_if = "Option::is_none"
)]
pub total_members: Option<NullableDto<SafeIntegerDto>>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct DeletionChangedDto {
pub schema_version: SchemaVersionDto,
pub event_id: OpaqueId,
pub sequence: PositiveIntegerDto,
pub cursor: CursorDto,
pub occurred_at: DateTimeDto,
pub job_id: OpaqueId,
pub operation_id: OpaqueId,
pub state: DeletionStateDto,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DeletionStateDto {
Queued,
Running,
RetryWait,
Blocked,
Completed,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ResyncRequiredDto {
pub schema_version: SchemaVersionDto,
pub event_id: OpaqueId,
pub sequence: PositiveIntegerDto,
pub cursor: CursorDto,
pub occurred_at: DateTimeDto,
pub reason: ResyncReasonDto,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ResyncReasonDto {
CursorAbsent,
CursorInvalid,
CursorExpired,
AuthorizationChanged,
HistoryUnavailable,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HeartbeatDto {
pub schema_version: SchemaVersionDto,
pub event_id: OpaqueId,
pub sequence: PositiveIntegerDto,
pub cursor: CursorDto,
pub occurred_at: DateTimeDto,
}

View file

@ -0,0 +1,169 @@
use remora_domain::OpaqueId;
use serde::{Deserialize, Serialize};
use crate::{
AttemptAuthorityDto, CredentialGrantVersionDto, DurableDescriptorKindDto,
DurableDescriptorRefDto, LeaseProjectionDto, NullableDto, PositiveIntegerDto, ProgressValueDto,
SafeIntegerDto, SourceProjectionDto, StagingTargetDto, TargetDto,
};
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ClaimRequestDto {
pub operation_id: OpaqueId,
pub worker_incarnation: OpaqueId,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ExecutionEnvelopeDto {
pub id: OpaqueId,
pub job_id: OpaqueId,
pub attempt_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub source: SourceProjectionDto,
pub sensitive_source_ref: OpaqueId,
pub credential_version: NullableDto<CredentialGrantVersionDto>,
pub resolved_source_ref: NullableDto<DurableDescriptorRefDto>,
pub target: TargetDto,
pub staging_target: StagingTargetDto,
pub execution_grant_ref: OpaqueId,
pub checkpoint_ref: NullableDto<DurableDescriptorRefDto>,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ClaimResponseDto {
pub authority: AttemptAuthorityDto,
pub execution: ExecutionEnvelopeDto,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ActivationResponseDto {
pub authority: AttemptAuthorityDto,
pub execution: ExecutionEnvelopeDto,
pub lease: LeaseProjectionDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptMutationDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptRenewRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptActivateRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ResolvedSourceDescriptorRefDto {
pub id: OpaqueId,
pub kind: ResolvedSourceDescriptorKindDto,
pub schema_version: PositiveIntegerDto,
pub byte_length: SafeIntegerDto,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub item_count: Option<SafeIntegerDto>,
pub checksum: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ResolvedSourceDescriptorKindDto {
ResolvedSource,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CheckpointDescriptorRefDto {
pub id: OpaqueId,
pub kind: CheckpointDescriptorKindDto,
pub schema_version: PositiveIntegerDto,
pub byte_length: SafeIntegerDto,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub item_count: Option<SafeIntegerDto>,
pub checksum: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum CheckpointDescriptorKindDto {
Checkpoint,
}
impl From<ResolvedSourceDescriptorRefDto> for DurableDescriptorRefDto {
fn from(value: ResolvedSourceDescriptorRefDto) -> Self {
Self {
id: value.id,
kind: DurableDescriptorKindDto::ResolvedSource,
schema_version: value.schema_version,
byte_length: value.byte_length,
item_count: value.item_count,
checksum: value.checksum,
}
}
}
impl From<CheckpointDescriptorRefDto> for DurableDescriptorRefDto {
fn from(value: CheckpointDescriptorRefDto) -> Self {
Self {
id: value.id,
kind: DurableDescriptorKindDto::Checkpoint,
schema_version: value.schema_version,
byte_length: value.byte_length,
item_count: value.item_count,
checksum: value.checksum,
}
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptResolvedSourceRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub resolved_source_ref: ResolvedSourceDescriptorRefDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptCheckpointRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub checkpoint_ref: CheckpointDescriptorRefDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptProgressRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub progress: ProgressValueDto,
}

View file

@ -0,0 +1,49 @@
mod credentials;
mod dto;
mod events;
mod internal;
mod public;
mod wire;
pub use credentials::{
BasicAuthCredentialCreateDto, BasicAuthMaterialDto, BearerTokenCredentialCreateDto,
CredentialGrantVersionDto, CredentialMaterialDto, HttpHeadersCredentialCreateDto,
HttpHeadersMaterialDto, HuggingFaceCredentialCreateDto, ObjectStoreCredentialCreateDto,
ObjectStoreCredentialMaterialDto, SourceCredentialCreateDto, SourceCredentialDto,
SourceCredentialKindDto, SourceCredentialPageDto, SourceCredentialRotateDto,
SourceCredentialStatusDto, SourceCredentialVersionDto, TokenMaterialDto,
};
pub use dto::{
AttemptAuthorityDto, AttemptFailureOutcomeDto, AttemptOutcomeDto, AttemptOutcomeRequestDto,
AttemptRuntimeFailureDto, AttemptSuccessOutcomeDto, HttpSourceSubmissionDto,
HuggingFaceRepositoryTypeDto, HuggingFaceSelectionDto, HuggingFaceSourceSubmissionDto,
JobFailureDto, JobPhaseProjectionDto, JobSubmissionDto, ObjectIdentityDto,
ObjectStoreSourceSubmissionDto, PermanentDispositionDto, PermanentFailureClassDto,
PermanentJobFailureDto, ProgressDto, ResultDescriptorDto, ResultKindDto,
RetryableDispositionDto, RetryableFailureClassDto, RetryableJobFailureDto, SafeDetailValueDto,
SourceSubmissionDto, TargetDto, TargetOverrideDto,
};
pub use events::{
DeletionChangedDto, DeletionStateDto, EventAttemptDto, EventFrameDto, EventProgressDto,
HeartbeatDto, JobChangedDto, ResyncReasonDto, ResyncRequiredDto,
};
pub use internal::{
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
AttemptMutationDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
AttemptResolvedSourceRequestDto, CheckpointDescriptorKindDto, CheckpointDescriptorRefDto,
ClaimRequestDto, ClaimResponseDto, ExecutionEnvelopeDto, ResolvedSourceDescriptorKindDto,
ResolvedSourceDescriptorRefDto,
};
pub use public::{
ApiErrorCodeDto, ApiErrorDto, ApiErrorFieldDto, AttemptPageDto, AttemptRecordDto,
AttemptRecordStateDto, CheckpointProjectionDto, DeletionOperationDto, DeletionProblemClassDto,
DeletionProblemDto, DeletionStateDto as HttpDeletionStateDto, DurableDescriptorKindDto,
DurableDescriptorRefDto, HttpSourceProjectionDto, HuggingFaceSourceProjectionDto, JobDto,
JobKindDto, JobPageDto, JobResourceDto, JobTombstoneDto, LeaseProjectionDto,
ObjectStoreSourceProjectionDto, ProgressValueDto, ResultMemberDto, ResultMemberPageDto,
SourceProjectionDto, StagingTargetDto, TerminalJobPhaseDto, TombstoneKindDto,
};
pub use wire::{
CursorDto, DateTimeDto, NullableDto, PositiveIntegerDto, SafeIntegerDto, SchemaVersionDto,
SignedSafeIntegerDto,
};

View file

@ -0,0 +1,326 @@
use remora_domain::{AttemptState, JobLifecycle, JobPhase, OpaqueId};
use serde::{Deserialize, Serialize};
use crate::{
AttemptAuthorityDto, CursorDto, DateTimeDto, HuggingFaceRepositoryTypeDto,
HuggingFaceSelectionDto, JobFailureDto, NullableDto, ObjectIdentityDto, PositiveIntegerDto,
ProgressDto, ResultDescriptorDto, SafeIntegerDto, TargetDto,
};
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum SourceProjectionDto {
Http(HttpSourceProjectionDto),
ObjectStore(ObjectStoreSourceProjectionDto),
HuggingFace(HuggingFaceSourceProjectionDto),
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HttpSourceProjectionDto {
pub display_url: String,
pub result_path: String,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub credential_id: Option<OpaqueId>,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ObjectStoreSourceProjectionDto {
pub store: String,
pub bucket: String,
pub key: String,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub version_id: Option<String>,
pub result_path: String,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub credential_id: Option<OpaqueId>,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HuggingFaceSourceProjectionDto {
pub repository_type: HuggingFaceRepositoryTypeDto,
pub repository_id: String,
pub revision: String,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub selection: Option<HuggingFaceSelectionDto>,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub credential_id: Option<OpaqueId>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct LeaseProjectionDto {
pub expires_at: DateTimeDto,
pub renew_after: DateTimeDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ProgressValueDto {
pub completed_bytes: SafeIntegerDto,
pub total_bytes: Option<SafeIntegerDto>,
pub completed_members: SafeIntegerDto,
pub total_members: Option<SafeIntegerDto>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CheckpointProjectionDto {
pub reusable: bool,
pub validated_bytes: SafeIntegerDto,
pub validated_members: SafeIntegerDto,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DurableDescriptorKindDto {
ResolvedSource,
Checkpoint,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct DurableDescriptorRefDto {
pub id: OpaqueId,
pub kind: DurableDescriptorKindDto,
pub schema_version: PositiveIntegerDto,
pub byte_length: SafeIntegerDto,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub item_count: Option<SafeIntegerDto>,
pub checksum: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ResultMemberDto {
pub id: OpaqueId,
pub path: String,
pub size: SafeIntegerDto,
pub object: ObjectIdentityDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ResultMemberPageDto {
pub items: Vec<ResultMemberDto>,
pub next_cursor: NullableDto<String>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum JobKindDto {
Job,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TombstoneKindDto {
Tombstone,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct JobDto {
pub resource_kind: JobKindDto,
pub id: OpaqueId,
pub owner: OpaqueId,
pub label: NullableDto<String>,
pub revision: PositiveIntegerDto,
pub phase: JobPhase,
pub lifecycle: JobLifecycle,
pub source: SourceProjectionDto,
pub target: TargetDto,
pub authoritative_attempt: NullableDto<AttemptAuthorityDto>,
pub progress: NullableDto<ProgressDto>,
pub checkpoint: NullableDto<CheckpointProjectionDto>,
pub result: NullableDto<ResultDescriptorDto>,
pub failure: NullableDto<JobFailureDto>,
pub created_at: DateTimeDto,
pub updated_at: DateTimeDto,
pub event_cursor: CursorDto,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum TerminalJobPhaseDto {
Succeeded,
Failed,
Cancelled,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct JobTombstoneDto {
pub resource_kind: TombstoneKindDto,
pub id: OpaqueId,
pub owner: OpaqueId,
pub target: TargetDto,
pub terminal_phase: TerminalJobPhaseDto,
pub deletion_operation_id: OpaqueId,
pub created_at: DateTimeDto,
pub hard_delete_started_at: DateTimeDto,
pub updated_at: DateTimeDto,
pub event_cursor: CursorDto,
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(untagged)]
pub enum JobResourceDto {
Job(Box<JobDto>),
Tombstone(Box<JobTombstoneDto>),
}
#[derive(Clone, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct JobPageDto {
pub items: Vec<JobResourceDto>,
pub next_cursor: NullableDto<String>,
pub event_cursor: CursorDto,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum AttemptRecordStateDto {
AttemptAuthorized,
AttemptExecuting,
AttemptStopped,
AttemptCommitted,
}
impl From<AttemptRecordStateDto> for AttemptState {
fn from(value: AttemptRecordStateDto) -> Self {
match value {
AttemptRecordStateDto::AttemptAuthorized => Self::AttemptAuthorized,
AttemptRecordStateDto::AttemptExecuting => Self::AttemptExecuting,
AttemptRecordStateDto::AttemptStopped => Self::AttemptStopped,
AttemptRecordStateDto::AttemptCommitted => Self::AttemptCommitted,
}
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptRecordDto {
pub id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub state: AttemptRecordStateDto,
pub progress: NullableDto<ProgressDto>,
pub result: NullableDto<ResultDescriptorDto>,
pub failure: NullableDto<JobFailureDto>,
pub created_at: DateTimeDto,
pub updated_at: DateTimeDto,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptPageDto {
pub items: Vec<AttemptRecordDto>,
pub next_cursor: NullableDto<String>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DeletionProblemClassDto {
Authorization,
TargetUnavailable,
Configuration,
Internal,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct DeletionProblemDto {
pub class: DeletionProblemClassDto,
pub message: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum DeletionStateDto {
Queued,
Running,
RetryWait,
Blocked,
Completed,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct DeletionOperationDto {
pub id: OpaqueId,
pub job_id: OpaqueId,
pub state: DeletionStateDto,
pub problem: NullableDto<DeletionProblemDto>,
pub created_at: DateTimeDto,
pub updated_at: DateTimeDto,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ApiErrorCodeDto {
InvalidRequest,
Unauthenticated,
Forbidden,
NotFound,
IdempotencyConflict,
PreconditionFailed,
CursorInvalid,
CursorExpired,
RateLimited,
Unavailable,
Internal,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ApiErrorFieldDto {
pub pointer: String,
pub code: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ApiErrorDto {
pub code: ApiErrorCodeDto,
pub message: String,
pub request_id: OpaqueId,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub fields: Option<Vec<ApiErrorFieldDto>>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct StagingTargetDto {
pub store: String,
pub bucket: String,
pub prefix: String,
}

View file

@ -0,0 +1,242 @@
use std::fmt;
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
pub const MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(transparent)]
pub struct NullableDto<T>(pub Option<T>);
pub(crate) fn optional_nullable<'de, D, T>(
deserializer: D,
) -> Result<Option<NullableDto<T>>, D::Error>
where
D: Deserializer<'de>,
T: Deserialize<'de>,
{
NullableDto::deserialize(deserializer).map(Some)
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct SafeIntegerDto(u64);
impl SafeIntegerDto {
pub fn new(value: u64) -> Result<Self, WireValueError> {
if value > MAX_SAFE_INTEGER {
return Err(WireValueError::SafeInteger);
}
Ok(Self(value))
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct SignedSafeIntegerDto(i64);
impl SignedSafeIntegerDto {
pub fn new(value: i64) -> Result<Self, WireValueError> {
if value.unsigned_abs() > MAX_SAFE_INTEGER {
return Err(WireValueError::SafeInteger);
}
Ok(Self(value))
}
#[must_use]
pub const fn get(self) -> i64 {
self.0
}
}
impl Serialize for SignedSafeIntegerDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_i64(self.0)
}
}
impl<'de> Deserialize<'de> for SignedSafeIntegerDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Self::new(i64::deserialize(deserializer)?).map_err(D::Error::custom)
}
}
impl Serialize for SafeIntegerDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_u64(self.0)
}
}
impl<'de> Deserialize<'de> for SafeIntegerDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Self::new(u64::deserialize(deserializer)?).map_err(D::Error::custom)
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct PositiveIntegerDto(SafeIntegerDto);
impl PositiveIntegerDto {
pub fn new(value: u64) -> Result<Self, WireValueError> {
let value = SafeIntegerDto::new(value)?;
if value.get() == 0 {
return Err(WireValueError::PositiveInteger);
}
Ok(Self(value))
}
#[must_use]
pub const fn get(self) -> u64 {
self.0.get()
}
}
impl Serialize for PositiveIntegerDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_u64(self.get())
}
}
impl<'de> Deserialize<'de> for PositiveIntegerDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Self::new(u64::deserialize(deserializer)?).map_err(D::Error::custom)
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
pub struct SchemaVersionDto;
impl Serialize for SchemaVersionDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_u8(1)
}
}
impl<'de> Deserialize<'de> for SchemaVersionDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
if u8::deserialize(deserializer)? != 1 {
return Err(D::Error::custom("unsupported schema version"));
}
Ok(Self)
}
}
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct CursorDto(String);
impl CursorDto {
pub fn parse(value: impl Into<String>) -> Result<Self, WireValueError> {
let value = value.into();
if !(16..=2048).contains(&value.chars().count()) {
return Err(WireValueError::Cursor);
}
Ok(Self(value))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl Serialize for CursorDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&self.0)
}
}
impl<'de> Deserialize<'de> for CursorDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Self::parse(String::deserialize(deserializer)?).map_err(D::Error::custom)
}
}
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct DateTimeDto(String);
impl DateTimeDto {
pub fn parse(value: impl Into<String>) -> Result<Self, WireValueError> {
let value = value.into();
OffsetDateTime::parse(&value, &Rfc3339).map_err(|_| WireValueError::DateTime)?;
Ok(Self(value))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl Serialize for DateTimeDto {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&self.0)
}
}
impl<'de> Deserialize<'de> for DateTimeDto {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
Self::parse(String::deserialize(deserializer)?).map_err(D::Error::custom)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum WireValueError {
SafeInteger,
PositiveInteger,
Cursor,
DateTime,
}
impl fmt::Display for WireValueError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let message = match self {
Self::SafeInteger => "integer exceeds the interoperable JSON range",
Self::PositiveInteger => "expected a positive integer",
Self::Cursor => "invalid cursor length",
Self::DateTime => "invalid RFC 3339 date-time",
};
formatter.write_str(message)
}
}
impl std::error::Error for WireValueError {}