Govern wire schemas from TypeSpec

contracts/typespec-rust-emitter/.gitignore

@@ -0,0 +1 @@
+node_modules/

contracts/typespec-rust-emitter/index.js

@@ -0,0 +1,481 @@
+import {
+  emitFile,
+  getDiscriminatedUnion,
+  getMaxLength,
+  getMaxValue,
+  getMinLength,
+  getMinValue,
+  getPattern,
+  resolvePath,
+  walkPropertiesInherited,
+} from '@typespec/compiler'
+
+export const namespace = 'Remora.Rust'
+
+const roots = ['JobSubmission', 'EventFrame', 'JobFailure']
+
+export async function $onEmit(context) {
+  const remora = context.program.getGlobalNamespaceType().namespaces.get('RemoraAPI')
+  if (!remora) {
+    throw new Error('RemoraAPI namespace is required')
+  }
+
+  const reachable = new Set()
+  for (const name of roots) {
+    const type = remora.models.get(name) ?? remora.unions.get(name)
+    if (!type) {
+      throw new Error(`Remora.${name} is required`)
+    }
+    collect(type, reachable)
+  }
+
+  const lines = [supportTypes()]
+  for (const scalar of remora.scalars.values()) {
+    if (reachable.has(scalar)) {
+      lines.push(emitScalar(context.program, scalar))
+    }
+  }
+  for (const enumType of remora.enums.values()) {
+    if (reachable.has(enumType)) {
+      lines.push(emitEnum(enumType))
+    }
+  }
+  for (const model of remora.models.values()) {
+    if (reachable.has(model)) {
+      lines.push(emitInlineEnums(model))
+      lines.push(emitModel(context.program, model, reachable))
+    }
+  }
+  for (const union of remora.unions.values()) {
+    if (reachable.has(union)) {
+      lines.push(emitUnion(context.program, union))
+    }
+  }
+
+  await emitFile(context.program, {
+    path: resolvePath(context.emitterOutputDir, 'remora_wire.rs'),
+    content: lines.filter(Boolean).join('\n\n'),
+    newLine: 'lf',
+  })
+}
+
+function collect(type, reachable) {
+  if (!type || reachable.has(type)) {
+    return
+  }
+  reachable.add(type)
+  if (type.kind === 'Model') {
+    for (const property of walkPropertiesInherited(type)) {
+      collect(property.type, reachable)
+    }
+    if (type.indexer) {
+      collect(type.indexer.value, reachable)
+    }
+  } else if (type.kind === 'Union') {
+    for (const variant of type.variants.values()) {
+      collect(variant.type, reachable)
+    }
+  } else if (type.kind === 'Scalar' && type.baseScalar) {
+    collect(type.baseScalar, reachable)
+  }
+}
+
+function emitScalar(program, scalar) {
+  if (scalar.name === 'OpaqueId') {
+    return `
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[serde(transparent)]
+pub struct OpaqueId(String);
+
+impl<'de> serde::Deserialize<'de> for OpaqueId {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = <String as serde::Deserialize>::deserialize(deserializer)?;
+        let valid = (8..=128).contains(&value.len())
+            && value.as_bytes().first().is_some_and(u8::is_ascii_alphanumeric)
+            && value.bytes().all(|byte| byte.is_ascii_alphanumeric() || byte == b'_' || byte == b'-');
+        if valid {
+            Ok(Self(value))
+        } else {
+            Err(serde::de::Error::custom("invalid opaque ID"))
+        }
+    }
+}`.trim()
+  }
+
+  if (inheritsFrom(scalar, 'OpaqueId')) {
+    return `pub type ${scalar.name} = OpaqueId;`
+  }
+  if (inheritsFrom(scalar, 'string')) {
+    const pattern = getPattern(program, scalar)
+    if (pattern) {
+      return patternString(scalar.name, pattern)
+    }
+    const min = getMinLength(program, scalar) ?? 0
+    const max = getMaxLength(program, scalar) ?? '18446744073709551615'
+    return `pub type ${scalar.name} = BoundedString<${min}, ${max}>;`
+  }
+  const min = getMinValue(program, scalar) ?? 0
+  const max = getMaxValue(program, scalar) ?? Number.MAX_SAFE_INTEGER
+  return `
+pub type ${scalar.name} = BoundedU64<${min}, ${max}>;`.trim()
+}
+
+function inheritsFrom(scalar, name) {
+  for (let current = scalar.baseScalar; current; current = current.baseScalar) {
+    if (current.name === name) {
+      return true
+    }
+  }
+  return false
+}
+
+function patternString(name, pattern) {
+  return `
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[serde(transparent)]
+pub struct ${name}(String);
+
+impl<'de> serde::Deserialize<'de> for ${name} {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = <String as serde::Deserialize>::deserialize(deserializer)?;
+        let pattern = fancy_regex::Regex::new(${JSON.stringify(pattern)})
+            .map_err(serde::de::Error::custom)?;
+        match pattern.is_match(&value).map_err(serde::de::Error::custom)? {
+            true => Ok(Self(value)),
+            false => Err(serde::de::Error::custom("string does not match required pattern")),
+        }
+    }
+}`.trim()
+}
+
+function emitEnum(enumType) {
+  const variants = [...enumType.members.values()]
+    .map((member) => {
+      const wire = member.value ?? member.name
+      return `    #[serde(rename = ${JSON.stringify(wire)})]\n    ${pascal(member.name)},`
+    })
+    .join('\n')
+  return `
+#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
+pub enum ${enumType.name} {
+${variants}
+}`.trim()
+}
+
+function emitInlineEnums(model) {
+  const definitions = []
+  for (const property of walkPropertiesInherited(model)) {
+    if (
+      property.type.kind !== 'Union' ||
+      ![...property.type.variants.values()].every(
+        (variant) => variant.type.kind === 'String',
+      )
+    ) {
+      continue
+    }
+    const name = `${model.name}${pascal(property.name)}`
+    const variants = [...property.type.variants.values()]
+      .map(
+        (variant) =>
+          `    #[serde(rename = ${JSON.stringify(variant.type.value)})]\n    ${pascal(variant.type.value)},`,
+      )
+      .join('\n')
+    definitions.push(`
+#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
+pub enum ${name} {
+${variants}
+}`.trim())
+  }
+  return definitions.join('\n\n')
+}
+
+function emitModel(program, model, reachable) {
+  const discriminatorNames = new Set()
+  for (const union of model.namespace.unions.values()) {
+    if (!reachable.has(union)) {
+      continue
+    }
+    const [discriminated] = getDiscriminatedUnion(program, union)
+    if (
+      discriminated &&
+      [...discriminated.variants.values()].includes(model)
+    ) {
+      discriminatorNames.add(discriminated.options.discriminatorPropertyName)
+    }
+  }
+
+  const fields = []
+  for (const property of walkPropertiesInherited(model)) {
+    if (discriminatorNames.has(property.name)) {
+      continue
+    }
+    const rustType = propertyType(program, model, property)
+    fields.push(
+      `    pub ${rustIdentifier(property.name)}: ${property.optional ? `Option<${rustType}>` : rustType},`,
+    )
+  }
+
+  return `
+#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct ${model.name} {
+${fields.join('\n')}
+}`.trim()
+}
+
+function emitUnion(program, union) {
+  const [discriminated, diagnostics] = getDiscriminatedUnion(program, union)
+  contextlessReport(program, diagnostics)
+  if (discriminated) {
+    const variants = [...discriminated.variants.entries()]
+      .map(
+        ([tag, type]) =>
+          `    #[serde(rename = ${JSON.stringify(tag)})]\n    ${pascal(tag)}(${type.name}),`,
+      )
+      .join('\n')
+    return `
+#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
+#[serde(tag = ${JSON.stringify(discriminated.options.discriminatorPropertyName)})]
+pub enum ${union.name} {
+${variants}
+}`.trim()
+  }
+
+  const variants = [...union.variants.values()]
+    .map((variant) => `    ${pascal(String(variant.name))}(${rustType(variant.type)}),`)
+    .join('\n')
+  return `
+#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
+#[serde(untagged)]
+pub enum ${union.name} {
+${variants}
+}`.trim()
+}
+
+function contextlessReport(program, diagnostics) {
+  if (diagnostics.length > 0) {
+    program.reportDiagnostics(diagnostics)
+  }
+}
+
+function propertyType(program, model, property) {
+  const type = property.type
+  if (type.kind === 'Number') {
+    return `ConstU64<${type.valueAsString}>`
+  }
+  if (type.kind === 'Boolean') {
+    return `ConstBool<${type.value}>`
+  }
+  if (
+    type.kind === 'Union' &&
+    [...type.variants.values()].every((variant) => variant.type.kind === 'String')
+  ) {
+    return `${model.name}${pascal(property.name)}`
+  }
+  if (type.kind === 'Scalar') {
+    if (type.name === 'url' || type.name === 'utcDateTime') {
+      return rustType(type)
+    }
+    const minLength = getMinLength(program, property)
+    const maxLength = getMaxLength(program, property)
+    if (minLength !== undefined || maxLength !== undefined) {
+      return `BoundedString<${minLength ?? 0}, ${maxLength ?? '18446744073709551615'}>`
+    }
+    const minValue = getMinValue(program, property)
+    const maxValue = getMaxValue(program, property)
+    if (minValue !== undefined || maxValue !== undefined) {
+      if ((minValue ?? 0) < 0) {
+        throw new Error(`unsupported signed bound on ${model.name}.${property.name}`)
+      }
+      return `BoundedU64<${minValue ?? 0}, ${maxValue ?? Number.MAX_SAFE_INTEGER}>`
+    }
+  }
+  return rustType(type)
+}
+
+function rustType(type) {
+  if (type.kind === 'Model') {
+    if (type.name === 'Array' && type.indexer) {
+      return `Vec<${rustType(type.indexer.value)}>`
+    }
+    if (type.name === 'Record' && type.indexer) {
+      return `std::collections::BTreeMap<String, ${rustType(type.indexer.value)}>`
+    }
+    return type.name
+  }
+  if (type.kind === 'Union') {
+    const variants = [...type.variants.values()].map((variant) => variant.type)
+    const nonNull = variants.filter((variant) => variant.kind !== 'Intrinsic' || variant.name !== 'null')
+    if (nonNull.length === 1 && nonNull.length + 1 === variants.length) {
+      return `Option<${rustType(nonNull[0])}>`
+    }
+    return type.name
+  }
+  if (type.kind === 'Enum') {
+    return type.name
+  }
+  if (type.kind === 'Scalar') {
+    if (type.name === 'url') {
+      return 'url::Url'
+    }
+    if (type.name === 'utcDateTime') {
+      return 'Rfc3339DateTime'
+    }
+    if (type.name === 'string') {
+      return 'String'
+    }
+    if (type.name === 'integer' || type.name === 'int64') {
+      return 'i64'
+    }
+    if (type.name === 'boolean') {
+      return 'bool'
+    }
+    if (type.name === 'uint64') {
+      return 'u64'
+    }
+    return type.name
+  }
+  if (type.kind === 'String') {
+    return 'String'
+  }
+  if (type.kind === 'Number') {
+    return type.numericValue?.isInteger ? 'i64' : 'f64'
+  }
+  if (type.kind === 'Boolean') {
+    return 'bool'
+  }
+  if (type.kind === 'Intrinsic' && type.name === 'null') {
+    return '()'
+  }
+  throw new Error(`unsupported TypeSpec type ${type.kind}`)
+}
+
+function pascal(value) {
+  return value
+    .split(/[^A-Za-z0-9]+/)
+    .filter(Boolean)
+    .map((part) => part[0].toUpperCase() + part.slice(1))
+    .join('')
+}
+
+function rustIdentifier(value) {
+  return ['type', 'ref', 'self', 'crate', 'super'].includes(value)
+    ? `r#${value}`
+    : value
+}
+
+function supportTypes() {
+  return `
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Rfc3339DateTime(time::OffsetDateTime);
+
+impl serde::Serialize for Rfc3339DateTime {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let value = self
+            .0
+            .format(&time::format_description::well_known::Rfc3339)
+            .map_err(serde::ser::Error::custom)?;
+        serializer.serialize_str(&value)
+    }
+}
+
+impl<'de> serde::Deserialize<'de> for Rfc3339DateTime {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = <String as serde::Deserialize>::deserialize(deserializer)?;
+        time::OffsetDateTime::parse(
+            &value,
+            &time::format_description::well_known::Rfc3339,
+        )
+        .map(Self)
+        .map_err(serde::de::Error::custom)
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[serde(transparent)]
+pub struct ConstU64<const VALUE: u64>(u64);
+
+impl<'de, const VALUE: u64> serde::Deserialize<'de> for ConstU64<VALUE> {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = <u64 as serde::Deserialize>::deserialize(deserializer)?;
+        if value == VALUE {
+            Ok(Self(value))
+        } else {
+            Err(serde::de::Error::custom("incorrect integer literal"))
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[serde(transparent)]
+pub struct ConstBool<const VALUE: bool>(bool);
+
+impl<'de, const VALUE: bool> serde::Deserialize<'de> for ConstBool<VALUE> {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = <bool as serde::Deserialize>::deserialize(deserializer)?;
+        if value == VALUE {
+            Ok(Self(value))
+        } else {
+            Err(serde::de::Error::custom("incorrect boolean literal"))
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[serde(transparent)]
+pub struct BoundedU64<const MIN: u64, const MAX: u64>(u64);
+
+impl<'de, const MIN: u64, const MAX: u64> serde::Deserialize<'de>
+    for BoundedU64<MIN, MAX>
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = <u64 as serde::Deserialize>::deserialize(deserializer)?;
+        if (MIN..=MAX).contains(&value) {
+            Ok(Self(value))
+        } else {
+            Err(serde::de::Error::custom("integer outside bounds"))
+        }
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
+#[serde(transparent)]
+pub struct BoundedString<const MIN: usize, const MAX: usize>(String);
+
+impl<'de, const MIN: usize, const MAX: usize> serde::Deserialize<'de>
+    for BoundedString<MIN, MAX>
+{
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let value = <String as serde::Deserialize>::deserialize(deserializer)?;
+        if (MIN..=MAX).contains(&value.chars().count()) {
+            Ok(Self(value))
+        } else {
+            Err(serde::de::Error::custom("string length outside bounds"))
+        }
+    }
+}`.trim()
+}

contracts/typespec-rust-emitter/lib/main.tsp

@@ -0,0 +1 @@
+import "../index.js";

contracts/typespec-rust-emitter/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@remora/typespec-rust",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "main": "index.js",
+  "tspMain": "lib/main.tsp",
+  "exports": {
+    ".": {
+      "typespec": "./lib/main.tsp",
+      "default": "./index.js"
+    }
+  },
+  "peerDependencies": {
+    "@typespec/compiler": "1.14.0"
+  },
+  "devDependencies": {
+    "@typespec/compiler": "1.14.0"
+  }
+}

contracts/typespec/.gitignore

@@ -0,0 +1,2 @@
+generated/
+node_modules/

contracts/typespec/generate.mjs

@@ -0,0 +1,107 @@
+import { spawnSync } from "node:child_process";
+import { readFileSync, writeFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parse, stringify } from "yaml";
+
+const root = dirname(fileURLToPath(import.meta.url));
+const repository = resolve(root, "../..");
+const check = process.argv.includes("--check");
+const compiler = resolve(root, "node_modules/.bin/tsp");
+const result = spawnSync(compiler, ["compile", root], {
+  cwd: root,
+  encoding: "utf8",
+});
+if (result.status !== 0) {
+  process.stderr.write(result.stdout);
+  process.stderr.write(result.stderr);
+  process.exit(result.status ?? 1);
+}
+const generatedRust = resolve(root, "generated/rust/remora_wire.rs");
+const rustfmt = spawnSync("rustfmt", ["--edition", "2024", generatedRust], {
+  cwd: root,
+  encoding: "utf8",
+});
+if (rustfmt.status !== 0) {
+  process.stderr.write(rustfmt.stdout);
+  process.stderr.write(rustfmt.stderr);
+  process.exit(rustfmt.status ?? 1);
+}
+
+const openapiDocument = parse(
+  readFileSync(resolve(root, "generated/openapi/openapi.yaml"), "utf8"),
+);
+openapiDocument.security = [{ oidc: [] }];
+openapiDocument.components ??= {};
+openapiDocument.components.securitySchemes = {
+  oidc: {
+    type: "openIdConnect",
+    openIdConnectUrl: "https://identity.invalid/.well-known/openid-configuration",
+  },
+  serviceAuth: { type: "mutualTLS" },
+  eventTicket: { type: "apiKey", in: "query", name: "ticket" },
+};
+for (const [path, item] of Object.entries(openapiDocument.paths)) {
+  for (const operation of Object.values(item)) {
+    if (typeof operation !== "object" || operation === null || !("responses" in operation)) {
+      continue;
+    }
+    if (path.startsWith("/internal/")) {
+      operation.security = [{ serviceAuth: [] }];
+    } else if (path === "/v1/events") {
+      operation.security = [{ oidc: [] }, { eventTicket: [] }];
+    }
+  }
+}
+const openapi = stringify(openapiDocument, { lineWidth: 0 });
+const bundle = JSON.parse(
+  readFileSync(resolve(root, "generated/json-schema/remora-wire.json"), "utf8"),
+);
+rewriteBundleReferences(bundle);
+for (const schema of Object.values(bundle.$defs)) {
+  delete schema.$schema;
+  delete schema.$id;
+}
+const events = {
+  ...bundle,
+  $id: "https://remora.invalid/api/events.schema.json",
+  title: "Remora WebSocket frame",
+  $ref: "#/$defs/EventFrame",
+};
+const outputs = [
+  [resolve(repository, "docs/api/openapi.yaml"), openapi],
+  [resolve(repository, "docs/api/events.schema.json"), `${JSON.stringify(events, null, 2)}\n`],
+  [
+    resolve(repository, "crates/remora-contracts/src/generated_wire.rs"),
+    readFileSync(generatedRust, "utf8"),
+  ],
+];
+
+let stale = false;
+for (const [path, content] of outputs) {
+  if (check) {
+    if (readFileSync(path, "utf8") !== content) {
+      console.error(`${path} is not generated from contracts/typespec/main.tsp`);
+      stale = true;
+    }
+  } else {
+    writeFileSync(path, content);
+  }
+}
+if (stale) {
+  process.exit(1);
+}
+
+function rewriteBundleReferences(value) {
+  if (Array.isArray(value)) {
+    value.forEach(rewriteBundleReferences);
+    return;
+  }
+  if (value === null || typeof value !== "object") {
+    return;
+  }
+  if (typeof value.$ref === "string" && /^[A-Za-z][A-Za-z0-9]*\.json$/.test(value.$ref)) {
+    value.$ref = `#/$defs/${value.$ref.slice(0, -5)}`;
+  }
+  Object.values(value).forEach(rewriteBundleReferences);
+}

contracts/typespec/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "remora-wire-contract",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "generate": "node generate.mjs",
+    "check": "node generate.mjs --check",
+    "compile": "tsp compile ."
+  },
+  "devDependencies": {
+    "@remora/typespec-rust": "file:../typespec-rust-emitter",
+    "@typespec/compiler": "1.14.0",
+    "@typespec/http": "1.14.0",
+    "@typespec/json-schema": "1.14.0",
+    "@typespec/openapi": "1.14.0",
+    "@typespec/openapi3": "1.14.0",
+    "yaml": "2.9.0"
+  }
+}

contracts/typespec/tspconfig.yaml

@@ -0,0 +1,22 @@
+emit:
+  - "@remora/typespec-rust"
+  - "@typespec/openapi3"
+  - "@typespec/json-schema"
+options:
+  "@remora/typespec-rust":
+    emitter-output-dir: "{output-dir}/rust"
+  "@typespec/openapi3":
+    emitter-output-dir: "{output-dir}/openapi"
+    file-type: yaml
+    openapi-versions:
+      - "3.2.0"
+    safeint-strategy: double-int
+  "@typespec/json-schema":
+    emitter-output-dir: "{output-dir}/json-schema"
+    file-type: json
+    bundleId: remora-wire.json
+    emitAllModels: true
+    emitAllRefs: true
+    int64-strategy: number
+    seal-object-schemas: true
+output-dir: "{project-root}/generated"