Govern wire schemas from TypeSpec
This commit is contained in:
parent
fa81cb7f21
commit
2020dd4ed3
24 changed files with 10189 additions and 1571 deletions
16
Cargo.lock
generated
16
Cargo.lock
generated
|
|
@ -1340,6 +1340,17 @@ dependencies = [
|
|||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.18.0"
|
||||
|
|
@ -2231,7 +2242,7 @@ dependencies = [
|
|||
"bytecount",
|
||||
"data-encoding",
|
||||
"email_address",
|
||||
"fancy-regex",
|
||||
"fancy-regex 0.18.0",
|
||||
"fraction",
|
||||
"getrandom 0.3.4",
|
||||
"idna",
|
||||
|
|
@ -3340,12 +3351,14 @@ dependencies = [
|
|||
name = "remora-contracts"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"fancy-regex 0.16.2",
|
||||
"jsonschema",
|
||||
"remora-domain",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml_ng",
|
||||
"time",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -5073,6 +5086,7 @@ dependencies = [
|
|||
"idna",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
13
Makefile
13
Makefile
|
|
@ -1,11 +1,16 @@
|
|||
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-live-events check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-deployment check-requirements check-qualification-kind check-qualification check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
||||
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-public-api check-live-events check-worker check-stream check-target check-result check-deletion check-outbound check-http-source check-object-source check-hf-source check-deployment check-requirements check-qualification-kind check-qualification check-fast check-full check-fast-host check-full-host contract-check rust-check web-install web-check web-test-fast web-test-full
|
||||
|
||||
CONTAINER_ENGINE ?= nerdctl
|
||||
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
|
||||
PLAYWRIGHT_BROWSERS_PATH := $(CURDIR)/web/.cache/ms-playwright
|
||||
export PLAYWRIGHT_BROWSERS_PATH
|
||||
|
||||
rust-check:
|
||||
contract-check:
|
||||
npm ci --prefix contracts/typespec
|
||||
npm ci --prefix contracts/typespec-rust-emitter
|
||||
npm run check --prefix contracts/typespec
|
||||
|
||||
rust-check: contract-check
|
||||
cargo fmt --all --check
|
||||
cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
cargo nextest run --workspace --all-features
|
||||
|
|
@ -31,13 +36,13 @@ web-test-full: web-install
|
|||
pnpm --dir web test:component:full
|
||||
pnpm --dir web test:e2e:full
|
||||
|
||||
check-fast: check-fixture-lib
|
||||
check-fast: check-fixture-lib contract-check
|
||||
$(CONTAINER_ENGINE) build --file Dockerfile.test --target rust-check --tag remora-rust-check:local .
|
||||
$(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-rust-check:local
|
||||
$(CONTAINER_ENGINE) build --file Dockerfile.test --target web-fast --tag remora-web-check:local .
|
||||
$(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-web-check:local
|
||||
|
||||
check-full: check-fixture-lib
|
||||
check-full: check-fixture-lib contract-check
|
||||
$(CONTAINER_ENGINE) build --file Dockerfile.test --target rust-check --tag remora-rust-check:local .
|
||||
$(CONTAINER_ENGINE) run --rm $(CONTAINER_RUN_FLAGS) remora-rust-check:local
|
||||
$(CONTAINER_ENGINE) build --file Dockerfile.test --target web-full --tag remora-web-check:local .
|
||||
|
|
|
|||
1
contracts/typespec-rust-emitter/.gitignore
vendored
Normal file
1
contracts/typespec-rust-emitter/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
node_modules/
|
||||
481
contracts/typespec-rust-emitter/index.js
Normal file
481
contracts/typespec-rust-emitter/index.js
Normal file
|
|
@ -0,0 +1,481 @@
|
|||
import {
|
||||
emitFile,
|
||||
getDiscriminatedUnion,
|
||||
getMaxLength,
|
||||
getMaxValue,
|
||||
getMinLength,
|
||||
getMinValue,
|
||||
getPattern,
|
||||
resolvePath,
|
||||
walkPropertiesInherited,
|
||||
} from '@typespec/compiler'
|
||||
|
||||
export const namespace = 'Remora.Rust'
|
||||
|
||||
const roots = ['JobSubmission', 'EventFrame', 'JobFailure']
|
||||
|
||||
export async function $onEmit(context) {
|
||||
const remora = context.program.getGlobalNamespaceType().namespaces.get('RemoraAPI')
|
||||
if (!remora) {
|
||||
throw new Error('RemoraAPI namespace is required')
|
||||
}
|
||||
|
||||
const reachable = new Set()
|
||||
for (const name of roots) {
|
||||
const type = remora.models.get(name) ?? remora.unions.get(name)
|
||||
if (!type) {
|
||||
throw new Error(`Remora.${name} is required`)
|
||||
}
|
||||
collect(type, reachable)
|
||||
}
|
||||
|
||||
const lines = [supportTypes()]
|
||||
for (const scalar of remora.scalars.values()) {
|
||||
if (reachable.has(scalar)) {
|
||||
lines.push(emitScalar(context.program, scalar))
|
||||
}
|
||||
}
|
||||
for (const enumType of remora.enums.values()) {
|
||||
if (reachable.has(enumType)) {
|
||||
lines.push(emitEnum(enumType))
|
||||
}
|
||||
}
|
||||
for (const model of remora.models.values()) {
|
||||
if (reachable.has(model)) {
|
||||
lines.push(emitInlineEnums(model))
|
||||
lines.push(emitModel(context.program, model, reachable))
|
||||
}
|
||||
}
|
||||
for (const union of remora.unions.values()) {
|
||||
if (reachable.has(union)) {
|
||||
lines.push(emitUnion(context.program, union))
|
||||
}
|
||||
}
|
||||
|
||||
await emitFile(context.program, {
|
||||
path: resolvePath(context.emitterOutputDir, 'remora_wire.rs'),
|
||||
content: lines.filter(Boolean).join('\n\n'),
|
||||
newLine: 'lf',
|
||||
})
|
||||
}
|
||||
|
||||
function collect(type, reachable) {
|
||||
if (!type || reachable.has(type)) {
|
||||
return
|
||||
}
|
||||
reachable.add(type)
|
||||
if (type.kind === 'Model') {
|
||||
for (const property of walkPropertiesInherited(type)) {
|
||||
collect(property.type, reachable)
|
||||
}
|
||||
if (type.indexer) {
|
||||
collect(type.indexer.value, reachable)
|
||||
}
|
||||
} else if (type.kind === 'Union') {
|
||||
for (const variant of type.variants.values()) {
|
||||
collect(variant.type, reachable)
|
||||
}
|
||||
} else if (type.kind === 'Scalar' && type.baseScalar) {
|
||||
collect(type.baseScalar, reachable)
|
||||
}
|
||||
}
|
||||
|
||||
function emitScalar(program, scalar) {
|
||||
if (scalar.name === 'OpaqueId') {
|
||||
return `
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct OpaqueId(String);
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for OpaqueId {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
let valid = (8..=128).contains(&value.len())
|
||||
&& value.as_bytes().first().is_some_and(u8::is_ascii_alphanumeric)
|
||||
&& value.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-');
|
||||
if valid {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("invalid opaque ID"))
|
||||
}
|
||||
}
|
||||
}`.trim()
|
||||
}
|
||||
|
||||
if (inheritsFrom(scalar, 'OpaqueId')) {
|
||||
return `pub type ${scalar.name} = OpaqueId;`
|
||||
}
|
||||
if (inheritsFrom(scalar, 'string')) {
|
||||
const pattern = getPattern(program, scalar)
|
||||
if (pattern) {
|
||||
return patternString(scalar.name, pattern)
|
||||
}
|
||||
const min = getMinLength(program, scalar) ?? 0
|
||||
const max = getMaxLength(program, scalar) ?? '18446744073709551615'
|
||||
return `pub type ${scalar.name} = BoundedString<${min}, ${max}>;`
|
||||
}
|
||||
const min = getMinValue(program, scalar) ?? 0
|
||||
const max = getMaxValue(program, scalar) ?? Number.MAX_SAFE_INTEGER
|
||||
return `
|
||||
pub type ${scalar.name} = BoundedU64<${min}, ${max}>;`.trim()
|
||||
}
|
||||
|
||||
function inheritsFrom(scalar, name) {
|
||||
for (let current = scalar.baseScalar; current; current = current.baseScalar) {
|
||||
if (current.name === name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function patternString(name, pattern) {
|
||||
return `
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ${name}(String);
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for ${name} {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
let pattern = fancy_regex::Regex::new(${JSON.stringify(pattern)})
|
||||
.map_err(serde::de::Error::custom)?;
|
||||
match pattern.is_match(&value).map_err(serde::de::Error::custom)? {
|
||||
true => Ok(Self(value)),
|
||||
false => Err(serde::de::Error::custom("string does not match required pattern")),
|
||||
}
|
||||
}
|
||||
}`.trim()
|
||||
}
|
||||
|
||||
function emitEnum(enumType) {
|
||||
const variants = [...enumType.members.values()]
|
||||
.map((member) => {
|
||||
const wire = member.value ?? member.name
|
||||
return ` #[serde(rename = ${JSON.stringify(wire)})]\n ${pascal(member.name)},`
|
||||
})
|
||||
.join('\n')
|
||||
return `
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum ${enumType.name} {
|
||||
${variants}
|
||||
}`.trim()
|
||||
}
|
||||
|
||||
function emitInlineEnums(model) {
|
||||
const definitions = []
|
||||
for (const property of walkPropertiesInherited(model)) {
|
||||
if (
|
||||
property.type.kind !== 'Union' ||
|
||||
![...property.type.variants.values()].every(
|
||||
(variant) => variant.type.kind === 'String',
|
||||
)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const name = `${model.name}${pascal(property.name)}`
|
||||
const variants = [...property.type.variants.values()]
|
||||
.map(
|
||||
(variant) =>
|
||||
` #[serde(rename = ${JSON.stringify(variant.type.value)})]\n ${pascal(variant.type.value)},`,
|
||||
)
|
||||
.join('\n')
|
||||
definitions.push(`
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum ${name} {
|
||||
${variants}
|
||||
}`.trim())
|
||||
}
|
||||
return definitions.join('\n\n')
|
||||
}
|
||||
|
||||
function emitModel(program, model, reachable) {
|
||||
const discriminatorNames = new Set()
|
||||
for (const union of model.namespace.unions.values()) {
|
||||
if (!reachable.has(union)) {
|
||||
continue
|
||||
}
|
||||
const [discriminated] = getDiscriminatedUnion(program, union)
|
||||
if (
|
||||
discriminated &&
|
||||
[...discriminated.variants.values()].includes(model)
|
||||
) {
|
||||
discriminatorNames.add(discriminated.options.discriminatorPropertyName)
|
||||
}
|
||||
}
|
||||
|
||||
const fields = []
|
||||
for (const property of walkPropertiesInherited(model)) {
|
||||
if (discriminatorNames.has(property.name)) {
|
||||
continue
|
||||
}
|
||||
const rustType = propertyType(program, model, property)
|
||||
fields.push(
|
||||
` pub ${rustIdentifier(property.name)}: ${property.optional ? `Option<${rustType}>` : rustType},`,
|
||||
)
|
||||
}
|
||||
|
||||
return `
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ${model.name} {
|
||||
${fields.join('\n')}
|
||||
}`.trim()
|
||||
}
|
||||
|
||||
function emitUnion(program, union) {
|
||||
const [discriminated, diagnostics] = getDiscriminatedUnion(program, union)
|
||||
contextlessReport(program, diagnostics)
|
||||
if (discriminated) {
|
||||
const variants = [...discriminated.variants.entries()]
|
||||
.map(
|
||||
([tag, type]) =>
|
||||
` #[serde(rename = ${JSON.stringify(tag)})]\n ${pascal(tag)}(${type.name}),`,
|
||||
)
|
||||
.join('\n')
|
||||
return `
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(tag = ${JSON.stringify(discriminated.options.discriminatorPropertyName)})]
|
||||
pub enum ${union.name} {
|
||||
${variants}
|
||||
}`.trim()
|
||||
}
|
||||
|
||||
const variants = [...union.variants.values()]
|
||||
.map((variant) => ` ${pascal(String(variant.name))}(${rustType(variant.type)}),`)
|
||||
.join('\n')
|
||||
return `
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ${union.name} {
|
||||
${variants}
|
||||
}`.trim()
|
||||
}
|
||||
|
||||
function contextlessReport(program, diagnostics) {
|
||||
if (diagnostics.length > 0) {
|
||||
program.reportDiagnostics(diagnostics)
|
||||
}
|
||||
}
|
||||
|
||||
function propertyType(program, model, property) {
|
||||
const type = property.type
|
||||
if (type.kind === 'Number') {
|
||||
return `ConstU64<${type.valueAsString}>`
|
||||
}
|
||||
if (type.kind === 'Boolean') {
|
||||
return `ConstBool<${type.value}>`
|
||||
}
|
||||
if (
|
||||
type.kind === 'Union' &&
|
||||
[...type.variants.values()].every((variant) => variant.type.kind === 'String')
|
||||
) {
|
||||
return `${model.name}${pascal(property.name)}`
|
||||
}
|
||||
if (type.kind === 'Scalar') {
|
||||
if (type.name === 'url' || type.name === 'utcDateTime') {
|
||||
return rustType(type)
|
||||
}
|
||||
const minLength = getMinLength(program, property)
|
||||
const maxLength = getMaxLength(program, property)
|
||||
if (minLength !== undefined || maxLength !== undefined) {
|
||||
return `BoundedString<${minLength ?? 0}, ${maxLength ?? '18446744073709551615'}>`
|
||||
}
|
||||
const minValue = getMinValue(program, property)
|
||||
const maxValue = getMaxValue(program, property)
|
||||
if (minValue !== undefined || maxValue !== undefined) {
|
||||
if ((minValue ?? 0) < 0) {
|
||||
throw new Error(`unsupported signed bound on ${model.name}.${property.name}`)
|
||||
}
|
||||
return `BoundedU64<${minValue ?? 0}, ${maxValue ?? Number.MAX_SAFE_INTEGER}>`
|
||||
}
|
||||
}
|
||||
return rustType(type)
|
||||
}
|
||||
|
||||
function rustType(type) {
|
||||
if (type.kind === 'Model') {
|
||||
if (type.name === 'Array' && type.indexer) {
|
||||
return `Vec<${rustType(type.indexer.value)}>`
|
||||
}
|
||||
if (type.name === 'Record' && type.indexer) {
|
||||
return `std::collections::BTreeMap<String, ${rustType(type.indexer.value)}>`
|
||||
}
|
||||
return type.name
|
||||
}
|
||||
if (type.kind === 'Union') {
|
||||
const variants = [...type.variants.values()].map((variant) => variant.type)
|
||||
const nonNull = variants.filter((variant) => variant.kind !== 'Intrinsic' || variant.name !== 'null')
|
||||
if (nonNull.length === 1 && nonNull.length + 1 === variants.length) {
|
||||
return `Option<${rustType(nonNull[0])}>`
|
||||
}
|
||||
return type.name
|
||||
}
|
||||
if (type.kind === 'Enum') {
|
||||
return type.name
|
||||
}
|
||||
if (type.kind === 'Scalar') {
|
||||
if (type.name === 'url') {
|
||||
return 'url::Url'
|
||||
}
|
||||
if (type.name === 'utcDateTime') {
|
||||
return 'Rfc3339DateTime'
|
||||
}
|
||||
if (type.name === 'string') {
|
||||
return 'String'
|
||||
}
|
||||
if (type.name === 'integer' || type.name === 'int64') {
|
||||
return 'i64'
|
||||
}
|
||||
if (type.name === 'boolean') {
|
||||
return 'bool'
|
||||
}
|
||||
if (type.name === 'uint64') {
|
||||
return 'u64'
|
||||
}
|
||||
return type.name
|
||||
}
|
||||
if (type.kind === 'String') {
|
||||
return 'String'
|
||||
}
|
||||
if (type.kind === 'Number') {
|
||||
return type.numericValue?.isInteger ? 'i64' : 'f64'
|
||||
}
|
||||
if (type.kind === 'Boolean') {
|
||||
return 'bool'
|
||||
}
|
||||
if (type.kind === 'Intrinsic' && type.name === 'null') {
|
||||
return '()'
|
||||
}
|
||||
throw new Error(`unsupported TypeSpec type ${type.kind}`)
|
||||
}
|
||||
|
||||
function pascal(value) {
|
||||
return value
|
||||
.split(/[^A-Za-z0-9]+/)
|
||||
.filter(Boolean)
|
||||
.map((part) => part[0].toUpperCase() + part.slice(1))
|
||||
.join('')
|
||||
}
|
||||
|
||||
function rustIdentifier(value) {
|
||||
return ['type', 'ref', 'self', 'crate', 'super'].includes(value)
|
||||
? `r#${value}`
|
||||
: value
|
||||
}
|
||||
|
||||
function supportTypes() {
|
||||
return `
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct Rfc3339DateTime(time::OffsetDateTime);
|
||||
|
||||
impl serde::Serialize for Rfc3339DateTime {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let value = self
|
||||
.0
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(serde::ser::Error::custom)?;
|
||||
serializer.serialize_str(&value)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for Rfc3339DateTime {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
time::OffsetDateTime::parse(
|
||||
&value,
|
||||
&time::format_description::well_known::Rfc3339,
|
||||
)
|
||||
.map(Self)
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ConstU64<const VALUE: u64>(u64);
|
||||
|
||||
impl<'de, const VALUE: u64> serde::Deserialize<'de> for ConstU64<VALUE> {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <u64 as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if value == VALUE {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("incorrect integer literal"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ConstBool<const VALUE: bool>(bool);
|
||||
|
||||
impl<'de, const VALUE: bool> serde::Deserialize<'de> for ConstBool<VALUE> {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <bool as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if value == VALUE {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("incorrect boolean literal"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct BoundedU64<const MIN: u64, const MAX: u64>(u64);
|
||||
|
||||
impl<'de, const MIN: u64, const MAX: u64> serde::Deserialize<'de>
|
||||
for BoundedU64<MIN, MAX>
|
||||
{
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <u64 as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if (MIN..=MAX).contains(&value) {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("integer outside bounds"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct BoundedString<const MIN: usize, const MAX: usize>(String);
|
||||
|
||||
impl<'de, const MIN: usize, const MAX: usize> serde::Deserialize<'de>
|
||||
for BoundedString<MIN, MAX>
|
||||
{
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if (MIN..=MAX).contains(&value.chars().count()) {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("string length outside bounds"))
|
||||
}
|
||||
}
|
||||
}`.trim()
|
||||
}
|
||||
1
contracts/typespec-rust-emitter/lib/main.tsp
Normal file
1
contracts/typespec-rust-emitter/lib/main.tsp
Normal file
|
|
@ -0,0 +1 @@
|
|||
import "../index.js";
|
||||
1003
contracts/typespec-rust-emitter/package-lock.json
generated
Normal file
1003
contracts/typespec-rust-emitter/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
20
contracts/typespec-rust-emitter/package.json
Normal file
20
contracts/typespec-rust-emitter/package.json
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"name": "@remora/typespec-rust",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "index.js",
|
||||
"tspMain": "lib/main.tsp",
|
||||
"exports": {
|
||||
".": {
|
||||
"typespec": "./lib/main.tsp",
|
||||
"default": "./index.js"
|
||||
}
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@typespec/compiler": "1.14.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@typespec/compiler": "1.14.0"
|
||||
}
|
||||
}
|
||||
2
contracts/typespec/.gitignore
vendored
Normal file
2
contracts/typespec/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
generated/
|
||||
node_modules/
|
||||
107
contracts/typespec/generate.mjs
Normal file
107
contracts/typespec/generate.mjs
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parse, stringify } from "yaml";
|
||||
|
||||
const root = dirname(fileURLToPath(import.meta.url));
|
||||
const repository = resolve(root, "../..");
|
||||
const check = process.argv.includes("--check");
|
||||
const compiler = resolve(root, "node_modules/.bin/tsp");
|
||||
const result = spawnSync(compiler, ["compile", root], {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
process.stderr.write(result.stdout);
|
||||
process.stderr.write(result.stderr);
|
||||
process.exit(result.status ?? 1);
|
||||
}
|
||||
const generatedRust = resolve(root, "generated/rust/remora_wire.rs");
|
||||
const rustfmt = spawnSync("rustfmt", ["--edition", "2024", generatedRust], {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
});
|
||||
if (rustfmt.status !== 0) {
|
||||
process.stderr.write(rustfmt.stdout);
|
||||
process.stderr.write(rustfmt.stderr);
|
||||
process.exit(rustfmt.status ?? 1);
|
||||
}
|
||||
|
||||
const openapiDocument = parse(
|
||||
readFileSync(resolve(root, "generated/openapi/openapi.yaml"), "utf8"),
|
||||
);
|
||||
openapiDocument.security = [{ oidc: [] }];
|
||||
openapiDocument.components ??= {};
|
||||
openapiDocument.components.securitySchemes = {
|
||||
oidc: {
|
||||
type: "openIdConnect",
|
||||
openIdConnectUrl: "https://identity.invalid/.well-known/openid-configuration",
|
||||
},
|
||||
serviceAuth: { type: "mutualTLS" },
|
||||
eventTicket: { type: "apiKey", in: "query", name: "ticket" },
|
||||
};
|
||||
for (const [path, item] of Object.entries(openapiDocument.paths)) {
|
||||
for (const operation of Object.values(item)) {
|
||||
if (typeof operation !== "object" || operation === null || !("responses" in operation)) {
|
||||
continue;
|
||||
}
|
||||
if (path.startsWith("/internal/")) {
|
||||
operation.security = [{ serviceAuth: [] }];
|
||||
} else if (path === "/v1/events") {
|
||||
operation.security = [{ oidc: [] }, { eventTicket: [] }];
|
||||
}
|
||||
}
|
||||
}
|
||||
const openapi = stringify(openapiDocument, { lineWidth: 0 });
|
||||
const bundle = JSON.parse(
|
||||
readFileSync(resolve(root, "generated/json-schema/remora-wire.json"), "utf8"),
|
||||
);
|
||||
rewriteBundleReferences(bundle);
|
||||
for (const schema of Object.values(bundle.$defs)) {
|
||||
delete schema.$schema;
|
||||
delete schema.$id;
|
||||
}
|
||||
const events = {
|
||||
...bundle,
|
||||
$id: "https://remora.invalid/api/events.schema.json",
|
||||
title: "Remora WebSocket frame",
|
||||
$ref: "#/$defs/EventFrame",
|
||||
};
|
||||
const outputs = [
|
||||
[resolve(repository, "docs/api/openapi.yaml"), openapi],
|
||||
[resolve(repository, "docs/api/events.schema.json"), `${JSON.stringify(events, null, 2)}\n`],
|
||||
[
|
||||
resolve(repository, "crates/remora-contracts/src/generated_wire.rs"),
|
||||
readFileSync(generatedRust, "utf8"),
|
||||
],
|
||||
];
|
||||
|
||||
let stale = false;
|
||||
for (const [path, content] of outputs) {
|
||||
if (check) {
|
||||
if (readFileSync(path, "utf8") !== content) {
|
||||
console.error(`${path} is not generated from contracts/typespec/main.tsp`);
|
||||
stale = true;
|
||||
}
|
||||
} else {
|
||||
writeFileSync(path, content);
|
||||
}
|
||||
}
|
||||
if (stale) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function rewriteBundleReferences(value) {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(rewriteBundleReferences);
|
||||
return;
|
||||
}
|
||||
if (value === null || typeof value !== "object") {
|
||||
return;
|
||||
}
|
||||
if (typeof value.$ref === "string" && /^[A-Za-z][A-Za-z0-9]*\.json$/.test(value.$ref)) {
|
||||
value.$ref = `#/$defs/${value.$ref.slice(0, -5)}`;
|
||||
}
|
||||
Object.values(value).forEach(rewriteBundleReferences);
|
||||
}
|
||||
1382
contracts/typespec/main.tsp
Normal file
1382
contracts/typespec/main.tsp
Normal file
File diff suppressed because it is too large
Load diff
1265
contracts/typespec/package-lock.json
generated
Normal file
1265
contracts/typespec/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
19
contracts/typespec/package.json
Normal file
19
contracts/typespec/package.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"name": "remora-wire-contract",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"generate": "node generate.mjs",
|
||||
"check": "node generate.mjs --check",
|
||||
"compile": "tsp compile ."
|
||||
},
|
||||
"devDependencies": {
|
||||
"@remora/typespec-rust": "file:../typespec-rust-emitter",
|
||||
"@typespec/compiler": "1.14.0",
|
||||
"@typespec/http": "1.14.0",
|
||||
"@typespec/json-schema": "1.14.0",
|
||||
"@typespec/openapi": "1.14.0",
|
||||
"@typespec/openapi3": "1.14.0",
|
||||
"yaml": "2.9.0"
|
||||
}
|
||||
}
|
||||
22
contracts/typespec/tspconfig.yaml
Normal file
22
contracts/typespec/tspconfig.yaml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
emit:
|
||||
- "@remora/typespec-rust"
|
||||
- "@typespec/openapi3"
|
||||
- "@typespec/json-schema"
|
||||
options:
|
||||
"@remora/typespec-rust":
|
||||
emitter-output-dir: "{output-dir}/rust"
|
||||
"@typespec/openapi3":
|
||||
emitter-output-dir: "{output-dir}/openapi"
|
||||
file-type: yaml
|
||||
openapi-versions:
|
||||
- "3.2.0"
|
||||
safeint-strategy: double-int
|
||||
"@typespec/json-schema":
|
||||
emitter-output-dir: "{output-dir}/json-schema"
|
||||
file-type: json
|
||||
bundleId: remora-wire.json
|
||||
emitAllModels: true
|
||||
emitAllRefs: true
|
||||
int64-strategy: number
|
||||
seal-object-schemas: true
|
||||
output-dir: "{project-root}/generated"
|
||||
|
|
@ -205,6 +205,7 @@ async fn submit_job(
|
|||
http: HttpRequest,
|
||||
request: web::Json<JobSubmissionDto>,
|
||||
) -> Result<HttpResponse, PublicApiError> {
|
||||
validate_request("JobSubmission", &*request)?;
|
||||
let principal = state.principal(&http).await?;
|
||||
let key = idempotency_key(&http)?;
|
||||
let request = request.into_inner();
|
||||
|
|
@ -694,6 +695,7 @@ async fn create_source_credential(
|
|||
http: HttpRequest,
|
||||
request: web::Json<SourceCredentialCreateDto>,
|
||||
) -> Result<HttpResponse, PublicApiError> {
|
||||
validate_request("SourceCredentialCreate", &*request)?;
|
||||
let principal = state.principal(&http).await?;
|
||||
let key = idempotency_key(&http)?;
|
||||
let request = request.into_inner();
|
||||
|
|
@ -828,6 +830,7 @@ async fn rotate_source_credential(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<SourceCredentialRotateDto>,
|
||||
) -> Result<HttpResponse, PublicApiError> {
|
||||
validate_request("SourceCredentialRotate", &*request)?;
|
||||
let principal = state.principal(&http).await?;
|
||||
let credential_id = parse_id(path.into_inner())?;
|
||||
let key = idempotency_key(&http)?;
|
||||
|
|
@ -2038,6 +2041,12 @@ fn request_id() -> OpaqueId {
|
|||
OpaqueId::parse(value).expect("random request id is valid")
|
||||
}
|
||||
|
||||
fn validate_request(schema: &'static str, request: &impl Serialize) -> Result<(), PublicApiError> {
|
||||
remora_contracts::request_is_valid(schema, request)
|
||||
.then_some(())
|
||||
.ok_or_else(PublicApiError::invalid_request)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use remora_domain::StorageScope;
|
||||
|
|
|
|||
|
|
@ -114,6 +114,7 @@ async fn register_staging_object(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingObjectRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptStagingObjectRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -149,6 +150,7 @@ async fn register_staging_write_intent(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingWriteIntentRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptStagingWriteIntentRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -182,6 +184,7 @@ async fn register_staging_multipart(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStagingMultipartRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptStagingMultipartRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -216,6 +219,7 @@ async fn stop(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptStopRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptStopRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -245,6 +249,7 @@ async fn claim(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<ClaimRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("ClaimRequest", &*request)?;
|
||||
let job_id = parse_id(path.into_inner())?;
|
||||
let outcome = state
|
||||
.attempts
|
||||
|
|
@ -267,6 +272,7 @@ async fn renew(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptRenewRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptRenewRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -293,6 +299,7 @@ async fn activate(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptActivateRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptActivateRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -325,6 +332,7 @@ async fn pin_resolved_source(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptResolvedSourceRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptResolvedSourceRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -351,6 +359,7 @@ async fn persist_checkpoint(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptCheckpointRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptCheckpointRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -390,6 +399,7 @@ async fn report_progress(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptProgressRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptProgressRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -421,6 +431,7 @@ async fn report_outcome(
|
|||
path: web::Path<String>,
|
||||
request: web::Json<AttemptOutcomeRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
validate_request("AttemptOutcomeRequest", &*request)?;
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
|
|
@ -652,6 +663,15 @@ fn parse_id(value: impl Into<String>) -> Result<OpaqueId, WorkerApiError> {
|
|||
OpaqueId::parse(value).map_err(|_| WorkerApiError::invalid_request())
|
||||
}
|
||||
|
||||
fn validate_request(
|
||||
schema: &'static str,
|
||||
request: &impl serde::Serialize,
|
||||
) -> Result<(), WorkerApiError> {
|
||||
remora_contracts::request_is_valid(schema, request)
|
||||
.then_some(())
|
||||
.ok_or_else(WorkerApiError::invalid_request)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("worker API request failed")]
|
||||
pub struct WorkerApiError {
|
||||
|
|
|
|||
|
|
@ -6,11 +6,13 @@ rust-version.workspace = true
|
|||
version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
fancy-regex = "0.16.2"
|
||||
jsonschema = { version = "0.47.0", default-features = false }
|
||||
remora-domain.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
time.workspace = true
|
||||
url = { version = "2.5.8", features = ["serde"] }
|
||||
|
||||
[dev-dependencies]
|
||||
jsonschema = { version = "0.47.0", default-features = false }
|
||||
serde_json.workspace = true
|
||||
serde_yaml_ng = "0.10.0"
|
||||
|
|
|
|||
448
crates/remora-contracts/src/generated_wire.rs
Normal file
448
crates/remora-contracts/src/generated_wire.rs
Normal file
|
|
@ -0,0 +1,448 @@
|
|||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct Rfc3339DateTime(time::OffsetDateTime);
|
||||
|
||||
impl serde::Serialize for Rfc3339DateTime {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let value = self
|
||||
.0
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(serde::ser::Error::custom)?;
|
||||
serializer.serialize_str(&value)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for Rfc3339DateTime {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
time::OffsetDateTime::parse(&value, &time::format_description::well_known::Rfc3339)
|
||||
.map(Self)
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ConstU64<const VALUE: u64>(u64);
|
||||
|
||||
impl<'de, const VALUE: u64> serde::Deserialize<'de> for ConstU64<VALUE> {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <u64 as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if value == VALUE {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("incorrect integer literal"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ConstBool<const VALUE: bool>(bool);
|
||||
|
||||
impl<'de, const VALUE: bool> serde::Deserialize<'de> for ConstBool<VALUE> {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <bool as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if value == VALUE {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("incorrect boolean literal"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct BoundedU64<const MIN: u64, const MAX: u64>(u64);
|
||||
|
||||
impl<'de, const MIN: u64, const MAX: u64> serde::Deserialize<'de> for BoundedU64<MIN, MAX> {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <u64 as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if (MIN..=MAX).contains(&value) {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("integer outside bounds"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct BoundedString<const MIN: usize, const MAX: usize>(String);
|
||||
|
||||
impl<'de, const MIN: usize, const MAX: usize> serde::Deserialize<'de> for BoundedString<MIN, MAX> {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
if (MIN..=MAX).contains(&value.chars().count()) {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("string length outside bounds"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct OpaqueId(String);
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for OpaqueId {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
let valid = (8..=128).contains(&value.len())
|
||||
&& value
|
||||
.as_bytes()
|
||||
.first()
|
||||
.is_some_and(u8::is_ascii_alphanumeric)
|
||||
&& value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-');
|
||||
if valid {
|
||||
Ok(Self(value))
|
||||
} else {
|
||||
Err(serde::de::Error::custom("invalid opaque ID"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct BucketName(String);
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for BucketName {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
let pattern = fancy_regex::Regex::new("^(?!.*\\.\\.)[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$")
|
||||
.map_err(serde::de::Error::custom)?;
|
||||
match pattern.is_match(&value).map_err(serde::de::Error::custom)? {
|
||||
true => Ok(Self(value)),
|
||||
false => Err(serde::de::Error::custom(
|
||||
"string does not match required pattern",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct HuggingFaceRepositoryId(String);
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for HuggingFaceRepositoryId {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = <String as serde::Deserialize>::deserialize(deserializer)?;
|
||||
let pattern = fancy_regex::Regex::new("^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$")
|
||||
.map_err(serde::de::Error::custom)?;
|
||||
match pattern.is_match(&value).map_err(serde::de::Error::custom)? {
|
||||
true => Ok(Self(value)),
|
||||
false => Err(serde::de::Error::custom(
|
||||
"string does not match required pattern",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub type CredentialId = OpaqueId;
|
||||
|
||||
pub type ResultPath = BoundedString<1, 4096>;
|
||||
|
||||
pub type EventCursor = BoundedString<16, 2048>;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum JobPhase {
|
||||
#[serde(rename = "Queued")]
|
||||
Queued,
|
||||
#[serde(rename = "Running")]
|
||||
Running,
|
||||
#[serde(rename = "RetryWait")]
|
||||
RetryWait,
|
||||
#[serde(rename = "Cancelling")]
|
||||
Cancelling,
|
||||
#[serde(rename = "Succeeded")]
|
||||
Succeeded,
|
||||
#[serde(rename = "Failed")]
|
||||
Failed,
|
||||
#[serde(rename = "Cancelled")]
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum JobLifecycle {
|
||||
#[serde(rename = "visible")]
|
||||
Visible,
|
||||
#[serde(rename = "soft_deleted")]
|
||||
SoftDeleted,
|
||||
#[serde(rename = "hard_deleting")]
|
||||
HardDeleting,
|
||||
#[serde(rename = "hard_deleted")]
|
||||
HardDeleted,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TargetOverride {
|
||||
pub bucket: Option<BucketName>,
|
||||
pub prefix: Option<BoundedString<0, 1024>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HttpSourceSubmission {
|
||||
pub url: url::Url,
|
||||
pub result_path: ResultPath,
|
||||
pub credential_id: Option<CredentialId>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ObjectStoreSourceSubmission {
|
||||
pub store: BoundedString<1, 128>,
|
||||
pub bucket: BoundedString<3, 63>,
|
||||
pub key: BoundedString<1, 4096>,
|
||||
pub version_id: Option<BoundedString<0, 1024>>,
|
||||
pub result_path: ResultPath,
|
||||
pub credential_id: Option<CredentialId>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HuggingFaceSelection {
|
||||
pub paths: Option<Vec<ResultPath>>,
|
||||
pub prefixes: Option<Vec<ResultPath>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum HuggingFaceSourceSubmissionRepositoryType {
|
||||
#[serde(rename = "model")]
|
||||
Model,
|
||||
#[serde(rename = "dataset")]
|
||||
Dataset,
|
||||
#[serde(rename = "space")]
|
||||
Space,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct HuggingFaceSourceSubmission {
|
||||
pub repository_type: HuggingFaceSourceSubmissionRepositoryType,
|
||||
pub repository_id: HuggingFaceRepositoryId,
|
||||
pub revision: BoundedString<1, 256>,
|
||||
pub selection: Option<HuggingFaceSelection>,
|
||||
pub credential_id: Option<CredentialId>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobSubmission {
|
||||
pub source: SourceSubmission,
|
||||
pub target: Option<TargetOverride>,
|
||||
pub label: Option<BoundedString<1, 200>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum PermanentJobFailureClass {
|
||||
#[serde(rename = "validation")]
|
||||
Validation,
|
||||
#[serde(rename = "authorization")]
|
||||
Authorization,
|
||||
#[serde(rename = "source_not_found")]
|
||||
SourceNotFound,
|
||||
#[serde(rename = "source_changed")]
|
||||
SourceChanged,
|
||||
#[serde(rename = "cancelled")]
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PermanentJobFailure {
|
||||
pub class: PermanentJobFailureClass,
|
||||
pub retryable: ConstBool<false>,
|
||||
pub message: BoundedString<0, 500>,
|
||||
pub safe_details: Option<std::collections::BTreeMap<String, SafeDetailValue>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum RetryableJobFailureClass {
|
||||
#[serde(rename = "transient_source")]
|
||||
TransientSource,
|
||||
#[serde(rename = "transient_target")]
|
||||
TransientTarget,
|
||||
#[serde(rename = "integrity")]
|
||||
Integrity,
|
||||
#[serde(rename = "capacity")]
|
||||
Capacity,
|
||||
#[serde(rename = "internal")]
|
||||
Internal,
|
||||
#[serde(rename = "lost_attempt")]
|
||||
LostAttempt,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RetryableJobFailure {
|
||||
pub class: RetryableJobFailureClass,
|
||||
pub retryable: ConstBool<true>,
|
||||
pub message: BoundedString<0, 500>,
|
||||
pub safe_details: Option<std::collections::BTreeMap<String, SafeDetailValue>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EventAttempt {
|
||||
pub attempt_id: OpaqueId,
|
||||
pub fence: BoundedU64<1, 9007199254740991>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct EventProgress {
|
||||
pub attempt_id: OpaqueId,
|
||||
pub fence: BoundedU64<1, 9007199254740991>,
|
||||
pub completed_bytes: BoundedU64<0, 9007199254740991>,
|
||||
pub total_bytes: Option<Option<i64>>,
|
||||
pub completed_members: BoundedU64<0, 9007199254740991>,
|
||||
pub total_members: Option<Option<i64>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct JobChanged {
|
||||
pub job_id: OpaqueId,
|
||||
pub job_revision: BoundedU64<1, 9007199254740991>,
|
||||
pub phase: JobPhase,
|
||||
pub lifecycle: JobLifecycle,
|
||||
pub authoritative_attempt: Option<EventAttempt>,
|
||||
pub progress: Option<EventProgress>,
|
||||
pub schema_version: ConstU64<1>,
|
||||
pub event_id: OpaqueId,
|
||||
pub sequence: BoundedU64<1, 9007199254740991>,
|
||||
pub cursor: EventCursor,
|
||||
pub occurred_at: Rfc3339DateTime,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum DeletionChangedState {
|
||||
#[serde(rename = "queued")]
|
||||
Queued,
|
||||
#[serde(rename = "running")]
|
||||
Running,
|
||||
#[serde(rename = "retry_wait")]
|
||||
RetryWait,
|
||||
#[serde(rename = "blocked")]
|
||||
Blocked,
|
||||
#[serde(rename = "completed")]
|
||||
Completed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DeletionChanged {
|
||||
pub job_id: OpaqueId,
|
||||
pub operation_id: OpaqueId,
|
||||
pub state: DeletionChangedState,
|
||||
pub schema_version: ConstU64<1>,
|
||||
pub event_id: OpaqueId,
|
||||
pub sequence: BoundedU64<1, 9007199254740991>,
|
||||
pub cursor: EventCursor,
|
||||
pub occurred_at: Rfc3339DateTime,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
pub enum ResyncRequiredReason {
|
||||
#[serde(rename = "cursor_absent")]
|
||||
CursorAbsent,
|
||||
#[serde(rename = "cursor_invalid")]
|
||||
CursorInvalid,
|
||||
#[serde(rename = "cursor_expired")]
|
||||
CursorExpired,
|
||||
#[serde(rename = "authorization_changed")]
|
||||
AuthorizationChanged,
|
||||
#[serde(rename = "history_unavailable")]
|
||||
HistoryUnavailable,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ResyncRequired {
|
||||
pub reason: ResyncRequiredReason,
|
||||
pub schema_version: ConstU64<1>,
|
||||
pub event_id: OpaqueId,
|
||||
pub sequence: BoundedU64<1, 9007199254740991>,
|
||||
pub cursor: EventCursor,
|
||||
pub occurred_at: Rfc3339DateTime,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Heartbeat {
|
||||
pub schema_version: ConstU64<1>,
|
||||
pub event_id: OpaqueId,
|
||||
pub sequence: BoundedU64<1, 9007199254740991>,
|
||||
pub cursor: EventCursor,
|
||||
pub occurred_at: Rfc3339DateTime,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(tag = "kind")]
|
||||
pub enum SourceSubmission {
|
||||
#[serde(rename = "http")]
|
||||
Http(HttpSourceSubmission),
|
||||
#[serde(rename = "object_store")]
|
||||
ObjectStore(ObjectStoreSourceSubmission),
|
||||
#[serde(rename = "hugging_face")]
|
||||
HuggingFace(HuggingFaceSourceSubmission),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum SafeDetailValue {
|
||||
Text(String),
|
||||
Integer(i64),
|
||||
Boolean(bool),
|
||||
SymbolName(()),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum JobFailure {
|
||||
Permanent(PermanentJobFailure),
|
||||
Retryable(RetryableJobFailure),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
|
||||
#[serde(tag = "kind")]
|
||||
pub enum EventFrame {
|
||||
#[serde(rename = "job_changed")]
|
||||
JobChanged(JobChanged),
|
||||
#[serde(rename = "deletion_changed")]
|
||||
DeletionChanged(DeletionChanged),
|
||||
#[serde(rename = "resync_required")]
|
||||
ResyncRequired(ResyncRequired),
|
||||
#[serde(rename = "heartbeat")]
|
||||
Heartbeat(Heartbeat),
|
||||
}
|
||||
|
|
@ -1,8 +1,10 @@
|
|||
mod credentials;
|
||||
mod dto;
|
||||
mod events;
|
||||
pub mod generated_wire;
|
||||
mod internal;
|
||||
mod public;
|
||||
mod validation;
|
||||
mod wire;
|
||||
|
||||
pub use credentials::{
|
||||
|
|
@ -46,6 +48,7 @@ pub use public::{
|
|||
ResultMemberAccessDto, ResultMemberDto, ResultMemberPageDto, SourceProjectionDto,
|
||||
StagingTargetDto, TerminalJobPhaseDto, TombstoneKindDto,
|
||||
};
|
||||
pub use validation::request_is_valid;
|
||||
pub use wire::{
|
||||
CursorDto, DateTimeDto, NullableDto, PositiveIntegerDto, SafeIntegerDto, SchemaVersionDto,
|
||||
SignedSafeIntegerDto,
|
||||
|
|
|
|||
61
crates/remora-contracts/src/validation.rs
Normal file
61
crates/remora-contracts/src/validation.rs
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
use std::{collections::HashMap, sync::LazyLock};
|
||||
|
||||
use jsonschema::Validator;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const REQUEST_SCHEMAS: &[&str] = &[
|
||||
"JobSubmission",
|
||||
"SourceCredentialCreate",
|
||||
"SourceCredentialRotate",
|
||||
"ClaimRequest",
|
||||
"AttemptRenewRequest",
|
||||
"AttemptActivateRequest",
|
||||
"AttemptStopRequest",
|
||||
"AttemptStagingObjectRequest",
|
||||
"AttemptStagingWriteIntentRequest",
|
||||
"AttemptStagingMultipartRequest",
|
||||
"AttemptResolvedSourceRequest",
|
||||
"AttemptCheckpointRequest",
|
||||
"AttemptProgressRequest",
|
||||
"AttemptOutcomeRequest",
|
||||
];
|
||||
|
||||
static REQUEST_VALIDATORS: LazyLock<HashMap<&'static str, Validator>> = LazyLock::new(|| {
|
||||
let bundle: Value =
|
||||
serde_json::from_str(include_str!("../../../docs/api/events.schema.json")).unwrap();
|
||||
REQUEST_SCHEMAS
|
||||
.iter()
|
||||
.map(|name| {
|
||||
let mut schema = bundle.clone();
|
||||
schema
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.insert("$ref".into(), json!(format!("#/$defs/{name}")));
|
||||
let validator = jsonschema::draft202012::options()
|
||||
.should_validate_formats(true)
|
||||
.build(&schema)
|
||||
.unwrap();
|
||||
(*name, validator)
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
|
||||
pub fn request_is_valid(name: &str, value: &impl Serialize) -> bool {
|
||||
let Some(validator) = REQUEST_VALIDATORS.get(name) else {
|
||||
return false;
|
||||
};
|
||||
serde_json::to_value(value)
|
||||
.ok()
|
||||
.is_some_and(|value| validator.is_valid(&value))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::request_is_valid;
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_schema_names() {
|
||||
assert!(!request_is_valid("Unknown", &serde_json::json!({})));
|
||||
}
|
||||
}
|
||||
63
crates/remora-contracts/tests/generated_wire.rs
Normal file
63
crates/remora-contracts/tests/generated_wire.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
use remora_contracts::generated_wire::{EventFrame, JobFailure, JobSubmission};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
#[test]
|
||||
fn generated_roots_accept_the_shared_valid_corpus() {
|
||||
let corpus: Value =
|
||||
serde_json::from_str(include_str!("../../../contracts/golden/domain.json")).unwrap();
|
||||
assert_all::<JobSubmission>(&corpus["job_submissions"]);
|
||||
assert_all::<JobFailure>(&corpus["job_failures"]);
|
||||
assert_all::<EventFrame>(&corpus["event_frames"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_roots_reject_governance_regressions() {
|
||||
assert_invalid::<JobSubmission>(json!({
|
||||
"source": {
|
||||
"kind": "http",
|
||||
"url": "not a URL",
|
||||
"result_path": "file.bin"
|
||||
}
|
||||
}));
|
||||
assert_invalid::<JobSubmission>(json!({
|
||||
"source": {
|
||||
"kind": "hugging_face",
|
||||
"repository_type": "model",
|
||||
"repository_id": "missing-owner",
|
||||
"revision": "main"
|
||||
}
|
||||
}));
|
||||
assert_invalid::<JobFailure>(json!({
|
||||
"class": "source_not_found",
|
||||
"message": "missing",
|
||||
"retryable": true
|
||||
}));
|
||||
assert_invalid::<EventFrame>(json!({
|
||||
"schema_version": 2,
|
||||
"kind": "heartbeat",
|
||||
"event_id": "event_0001",
|
||||
"sequence": 1,
|
||||
"cursor": "cursor-value-0001",
|
||||
"occurred_at": "2026-07-15T00:00:00Z"
|
||||
}));
|
||||
assert_invalid::<EventFrame>(json!({
|
||||
"schema_version": 1,
|
||||
"kind": "heartbeat",
|
||||
"event_id": "event_0001",
|
||||
"sequence": 1,
|
||||
"cursor": "cursor-value-0001",
|
||||
"occurred_at": "2026-07-15T00:00:00Z",
|
||||
"unexpected": true
|
||||
}));
|
||||
}
|
||||
|
||||
fn assert_all<T: DeserializeOwned>(values: &Value) {
|
||||
for value in values.as_array().unwrap() {
|
||||
serde_json::from_value::<T>(value.clone()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_invalid<T: DeserializeOwned>(value: Value) {
|
||||
assert!(serde_json::from_value::<T>(value).is_err());
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue