ms2k/remora
SHA256
Watch
0
Code Releases Activity

Implement crash-safe streaming worker runtime

Browse source
This commit is contained in:
ms2k 2026-07-17 18:51:44 +09:00
commit cdeef3cc1a
3 changed files with 7448 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

78
crates/remora-contracts/src/internal.rs
View file

@ -71,6 +71,78 @@ pub struct AttemptActivateRequestDto {
pub worker_incarnation: OpaqueId,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptStopRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptStagingObjectRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub object: StagingObjectIdentityDto,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub completed_upload_id: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct StagingObjectIdentityDto {
pub store: String,
pub bucket: String,
pub key: String,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub version_id: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptStagingWriteIntentRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub intent: StagingWriteIntentDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct StagingWriteIntentDto {
pub store: String,
pub bucket: String,
pub key: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AttemptStagingMultipartRequestDto {
pub operation_id: OpaqueId,
pub fence: PositiveIntegerDto,
pub worker_incarnation: OpaqueId,
pub multipart: StagingMultipartIdentityDto,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct StagingMultipartIdentityDto {
pub store: String,
pub bucket: String,
pub key: String,
pub upload_id: String,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ResolvedSourceDescriptorRefDto {
@ -160,6 +232,12 @@ pub struct AttemptCheckpointRequestDto {
pub resolved_source_id: OpaqueId,
pub validated_bytes: SafeIntegerDto,
pub validated_members: SafeIntegerDto,
#[serde(
default,
deserialize_with = "crate::dto::optional_non_null",
skip_serializing_if = "Option::is_none"
)]
pub staging_multipart: Option<StagingMultipartIdentityDto>,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]

8
crates/remora-contracts/src/lib.rs
View file

@ -30,9 +30,11 @@ pub use events::{
pub use internal::{
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
AttemptMutationDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
AttemptResolvedSourceRequestDto, CheckpointDescriptorKindDto, CheckpointDescriptorRefDto,
ClaimRequestDto, ClaimResponseDto, ExecutionEnvelopeDto, ResolvedSourceDescriptorKindDto,
ResolvedSourceDescriptorRefDto,
AttemptResolvedSourceRequestDto, AttemptStagingMultipartRequestDto,
AttemptStagingObjectRequestDto, AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto,
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
ExecutionEnvelopeDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
StagingMultipartIdentityDto, StagingObjectIdentityDto, StagingWriteIntentDto,
};
pub use public::{
ApiErrorCodeDto, ApiErrorDto, ApiErrorFieldDto, AttemptPageDto, AttemptRecordDto,

83
crates/remora-contracts/tests/schema_contract.rs
View file

@ -3,8 +3,9 @@ use std::{collections::BTreeSet, fs};
use remora_contracts::{
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
AttemptResolvedSourceRequestDto, ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto,
LeaseProjectionDto,
AttemptResolvedSourceRequestDto, AttemptStagingMultipartRequestDto,
AttemptStagingObjectRequestDto, AttemptStagingWriteIntentRequestDto, AttemptStopRequestDto,
ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto, LeaseProjectionDto,
};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
@ -96,7 +97,7 @@ fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
"staging_target": {
"store": "primary",
"bucket": "remora-results",
"prefix": "_remora/a/attempt_0001/"
"prefix": "users/alice/_remora/a/attempt_0001/"
},
"execution_grant_ref": "grant_0001",
"checkpoint_ref": null
@ -135,6 +136,58 @@ fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
"AttemptActivateRequest",
mutation.clone(),
);
assert_dto_and_schema::<AttemptStopRequestDto>(
&component_schema,
"AttemptStopRequest",
serde_json::json!({
"operation_id": "operation_0001",
"fence": 1,
"worker_incarnation": "worker_0001"
}),
);
assert_dto_and_schema::<AttemptStagingObjectRequestDto>(
&component_schema,
"AttemptStagingObjectRequest",
serde_json::json!({
"operation_id": "operation_0001",
"fence": 1,
"worker_incarnation": "worker_0001",
"object": {
"store": "primary",
"bucket": "remora-results",
"key": "users/alice/_remora/a/attempt_0001/o/object_0001"
}
}),
);
assert_dto_and_schema::<AttemptStagingWriteIntentRequestDto>(
&component_schema,
"AttemptStagingWriteIntentRequest",
serde_json::json!({
"operation_id": "operation_0001",
"fence": 1,
"worker_incarnation": "worker_0001",
"intent": {
"store": "primary",
"bucket": "remora-results",
"key": "users/alice/_remora/a/attempt_0001/o/object_0001"
}
}),
);
assert_dto_and_schema::<AttemptStagingMultipartRequestDto>(
&component_schema,
"AttemptStagingMultipartRequest",
serde_json::json!({
"operation_id": "operation_0001",
"fence": 1,
"worker_incarnation": "worker_0001",
"multipart": {
"store": "primary",
"bucket": "remora-results",
"key": "users/alice/_remora/a/attempt_0001/o/object_0001",
"upload_id": "upload_0001"
}
}),
);
assert_dto_and_schema::<ActivationResponseDto>(
&component_schema,
"ActivationResponse",
@ -228,6 +281,30 @@ fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
"200",
Some("ActivationResponse"),
),
(
"/internal/v1/attempts/{attempt_id}/stop",
"AttemptStopRequest",
"204",
None,
),
(
"/internal/v1/attempts/{attempt_id}/staging/write-intent",
"AttemptStagingWriteIntentRequest",
"204",
None,
),
(
"/internal/v1/attempts/{attempt_id}/staging/object",
"AttemptStagingObjectRequest",
"204",
None,
),
(
"/internal/v1/attempts/{attempt_id}/staging/multipart",
"AttemptStagingMultipartRequest",
"204",
None,
),
(
"/internal/v1/attempts/{attempt_id}/resolved-source",
"AttemptResolvedSourceRequest",