Add authenticated internal worker API
This commit is contained in:
parent
9d96c4d563
commit
ebf818d23c
20 changed files with 2745 additions and 43 deletions
488
Cargo.lock
generated
488
Cargo.lock
generated
|
|
@ -2,6 +2,216 @@
|
|||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "actix-codec"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a"
|
||||
dependencies = [
|
||||
"bitflags 2.13.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-http"
|
||||
version = "3.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48e2faa3e7418ed780cca54829d32782a4008a077230f67457caa063415e99c2"
|
||||
dependencies = [
|
||||
"actix-codec",
|
||||
"actix-rt",
|
||||
"actix-service",
|
||||
"actix-tls",
|
||||
"actix-utils",
|
||||
"base64",
|
||||
"bitflags 2.13.0",
|
||||
"brotli",
|
||||
"bytes",
|
||||
"bytestring",
|
||||
"derive_more",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"foldhash",
|
||||
"futures-core",
|
||||
"h2 0.3.27",
|
||||
"http 0.2.12",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"language-tags",
|
||||
"local-channel",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rand 0.10.2",
|
||||
"sha1",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-macros"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-router"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14f8c75c51892f18d9c46150c5ac7beb81c95f78c8b83a634d49f4ca32551fe7"
|
||||
dependencies = [
|
||||
"bytestring",
|
||||
"cfg-if 1.0.4",
|
||||
"http 0.2.12",
|
||||
"regex",
|
||||
"regex-lite",
|
||||
"serde",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-rt"
|
||||
version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92589714878ca59a7626ea19734f0e07a6a875197eec751bb5d3f99e64998c63"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-server"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a65064ea4a457eaf07f2fba30b4c695bf43b721790e9530d26cb6f9019ff7502"
|
||||
dependencies = [
|
||||
"actix-rt",
|
||||
"actix-service",
|
||||
"actix-utils",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"mio",
|
||||
"socket2 0.5.10",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-service"
|
||||
version = "2.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e46f36bf0e5af44bdc4bdb36fbbd421aa98c79a9bce724e1edeb3894e10dc7f"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-tls"
|
||||
version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6176099de3f58fbddac916a7f8c6db297e021d706e7a6b99947785fee14abe9f"
|
||||
dependencies = [
|
||||
"actix-rt",
|
||||
"actix-service",
|
||||
"actix-utils",
|
||||
"futures-core",
|
||||
"impl-more 0.1.9",
|
||||
"pin-project-lite",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-utils"
|
||||
version = "3.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8"
|
||||
dependencies = [
|
||||
"local-waker",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-web"
|
||||
version = "4.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df09e2d9239703dd64056359c920c7f3fba6535ec61a0059e0f44e095ffe02b4"
|
||||
dependencies = [
|
||||
"actix-codec",
|
||||
"actix-http",
|
||||
"actix-macros",
|
||||
"actix-router",
|
||||
"actix-rt",
|
||||
"actix-server",
|
||||
"actix-service",
|
||||
"actix-tls",
|
||||
"actix-utils",
|
||||
"actix-web-codegen",
|
||||
"bytes",
|
||||
"bytestring",
|
||||
"cfg-if 1.0.4",
|
||||
"cookie",
|
||||
"derive_more",
|
||||
"encoding_rs",
|
||||
"foldhash",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"impl-more 0.3.1",
|
||||
"itoa",
|
||||
"language-tags",
|
||||
"log",
|
||||
"mime",
|
||||
"once_cell",
|
||||
"pin-project-lite",
|
||||
"regex",
|
||||
"regex-lite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"smallvec",
|
||||
"socket2 0.6.5",
|
||||
"time",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "actix-web-codegen"
|
||||
version = "4.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8"
|
||||
dependencies = [
|
||||
"actix-router",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
|
|
@ -25,6 +235,21 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloc-no-stdlib"
|
||||
version = "2.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
|
||||
|
||||
[[package]]
|
||||
name = "alloc-stdlib"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
|
|
@ -421,6 +646,27 @@ version = "0.2.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "8.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
"brotli-decompressor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli-decompressor"
|
||||
version = "5.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib",
|
||||
"alloc-stdlib",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.13.0"
|
||||
|
|
@ -472,6 +718,15 @@ dependencies = [
|
|||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytestring"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86566c496f2f47d9b8147a4c8b02ffdb69c919fe0c2b2e7195d22cbba0e635c9"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.67"
|
||||
|
|
@ -626,6 +881,26 @@ version = "0.4.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
|
||||
|
||||
[[package]]
|
||||
name = "convert_case"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"time",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.4"
|
||||
|
|
@ -886,6 +1161,29 @@ dependencies = [
|
|||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
|
||||
dependencies = [
|
||||
"derive_more-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more-impl"
|
||||
version = "2.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustc_version",
|
||||
"syn",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
|
|
@ -1049,6 +1347,16 @@ version = "0.1.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fluent-uri"
|
||||
version = "0.4.1"
|
||||
|
|
@ -1300,6 +1608,25 @@ dependencies = [
|
|||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"http 0.2.12",
|
||||
"indexmap",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.15"
|
||||
|
|
@ -1536,7 +1863,7 @@ dependencies = [
|
|||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"h2",
|
||||
"h2 0.4.15",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"httparse",
|
||||
|
|
@ -1580,7 +1907,7 @@ dependencies = [
|
|||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"socket2 0.6.5",
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
|
|
@ -1721,6 +2048,18 @@ dependencies = [
|
|||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "impl-more"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2"
|
||||
|
||||
[[package]]
|
||||
name = "impl-more"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35a84fd5aa25fae5c0f4a33d9cac2ca017fc622cbd089be2229993514990f870"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.0"
|
||||
|
|
@ -1934,6 +2273,12 @@ version = "0.4.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "language-tags"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
|
|
@ -1998,6 +2343,23 @@ version = "0.8.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||
|
||||
[[package]]
|
||||
name = "local-channel"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"local-waker",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "local-waker"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
|
|
@ -2090,6 +2452,16 @@ version = "0.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.2.2"
|
||||
|
|
@ -2097,6 +2469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"wasi 0.11.1+wasi-snapshot-preview1",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
|
@ -2604,7 +2977,7 @@ dependencies = [
|
|||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2",
|
||||
"socket2 0.6.5",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
|
|
@ -2643,7 +3016,7 @@ dependencies = [
|
|||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"socket2 0.6.5",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
|
@ -2731,6 +3104,7 @@ version = "0.14.8"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
|
||||
dependencies = [
|
||||
"pem",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"time",
|
||||
|
|
@ -2766,7 +3140,7 @@ dependencies = [
|
|||
"pin-project-lite",
|
||||
"ryu",
|
||||
"sha1_smol",
|
||||
"socket2",
|
||||
"socket2 0.6.5",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"url",
|
||||
|
|
@ -2852,6 +3226,12 @@ dependencies = [
|
|||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-lite"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
|
|
@ -2880,6 +3260,31 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "remora-api"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"actix-tls",
|
||||
"actix-web",
|
||||
"getrandom 0.3.4",
|
||||
"rcgen",
|
||||
"remora-admission",
|
||||
"remora-attempt",
|
||||
"remora-contracts",
|
||||
"remora-coordination",
|
||||
"remora-credentials",
|
||||
"remora-domain",
|
||||
"remora-durable",
|
||||
"remora-identity",
|
||||
"remora-result",
|
||||
"remora-target",
|
||||
"reqwest 0.13.4",
|
||||
"rustls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
"sqlx",
|
||||
"thiserror",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "remora-attempt"
|
||||
|
|
@ -3222,7 +3627,7 @@ dependencies = [
|
|||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"h2 0.4.15",
|
||||
"http 1.4.2",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
|
|
@ -3358,6 +3763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
|
|
@ -3669,6 +4075,16 @@ version = "2.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||
dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signature"
|
||||
version = "2.2.0"
|
||||
|
|
@ -3679,6 +4095,12 @@ dependencies = [
|
|||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||
|
||||
[[package]]
|
||||
name = "simd_cesu8"
|
||||
version = "1.2.0"
|
||||
|
|
@ -3722,6 +4144,16 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.5.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.5"
|
||||
|
|
@ -4166,8 +4598,10 @@ dependencies = [
|
|||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"signal-hook-registry",
|
||||
"socket2 0.6.5",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
|
@ -4448,6 +4882,18 @@ version = "0.1.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
|
||||
[[package]]
|
||||
name = "unsafe-libyaml"
|
||||
version = "0.2.11"
|
||||
|
|
@ -5228,3 +5674,31 @@ name = "zmij"
|
|||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "2.0.16+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -257,6 +257,34 @@ COPY crates crates
|
|||
COPY --from=kind-attempt-verified /tmp/remora-attempt-tests.tar.zst /tests/remora-attempt-tests.tar.zst
|
||||
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_TARGET_ENDPOINT\" && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_RECOVERY_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-attempt-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'binary(recovery)'"]
|
||||
|
||||
FROM rust-tools AS kind-worker-api-build
|
||||
WORKDIR /workspace
|
||||
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||
COPY .config .config
|
||||
COPY contracts contracts
|
||||
COPY crates crates
|
||||
COPY docs/api docs/api
|
||||
RUN cargo nextest archive \
|
||||
--package remora-api \
|
||||
--build-jobs 4 \
|
||||
--locked \
|
||||
--archive-file /tmp/remora-api-tests.tar.zst
|
||||
|
||||
FROM kind-worker-api-build AS kind-worker-api-verified
|
||||
RUN cargo clippy \
|
||||
--package remora-api \
|
||||
--all-targets \
|
||||
--locked \
|
||||
-- -D warnings \
|
||||
&& cargo test --package remora-api --doc --locked
|
||||
|
||||
FROM rust-tools AS kind-worker-api-runner
|
||||
WORKDIR /workspace
|
||||
COPY Cargo.toml ./
|
||||
COPY crates crates
|
||||
COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
|
||||
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && export REMORA_WORKER_API_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast"]
|
||||
|
||||
FROM rust-tools AS kind-http-source-build
|
||||
WORKDIR /workspace
|
||||
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||
|
|
|
|||
5
Makefile
5
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-stream check-target check-result check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
||||
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-stream check-target check-result check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
||||
|
||||
CONTAINER_ENGINE ?= nerdctl
|
||||
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
|
||||
|
|
@ -80,6 +80,9 @@ check-attempt-mutations:
|
|||
check-recovery:
|
||||
./tests/kind/run-recovery-job.sh
|
||||
|
||||
check-worker-api:
|
||||
./tests/kind/run-worker-api-job.sh
|
||||
|
||||
check-stream:
|
||||
cargo fmt --all --check
|
||||
cargo clippy --package remora-stream --all-targets --all-features -- -D warnings
|
||||
|
|
|
|||
|
|
@ -202,12 +202,15 @@
|
|||
"kind": "success",
|
||||
"result": {
|
||||
"kind": "manifest",
|
||||
"manifest_id": "manifest_0001",
|
||||
"schema_version": 1,
|
||||
"manifest": {
|
||||
"store": "target-main",
|
||||
"bucket": "target-bucket",
|
||||
"key": "_remora/results/manifest.json",
|
||||
"version_id": "version-1",
|
||||
"checksum": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
"checksum": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"size": 512
|
||||
},
|
||||
"member_count": 1,
|
||||
"total_bytes": 4096
|
||||
|
|
|
|||
|
|
@ -4,3 +4,30 @@ edition.workspace = true
|
|||
publish = false
|
||||
rust-version.workspace = true
|
||||
version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
actix-web = { version = "=4.14.0", features = ["rustls-0_23"] }
|
||||
actix-tls = "=3.5.0"
|
||||
getrandom = "0.3.4"
|
||||
remora-attempt = { path = "../remora-attempt" }
|
||||
remora-contracts = { path = "../remora-contracts" }
|
||||
remora-domain.workspace = true
|
||||
remora-result = { path = "../remora-result" }
|
||||
remora-target = { path = "../remora-target" }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
sqlx.workspace = true
|
||||
thiserror.workspace = true
|
||||
time.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rcgen = "=0.14.8"
|
||||
remora-admission = { path = "../remora-admission" }
|
||||
remora-coordination = { path = "../remora-coordination" }
|
||||
remora-credentials = { path = "../remora-credentials" }
|
||||
remora-durable.workspace = true
|
||||
remora-identity = { path = "../remora-identity" }
|
||||
reqwest = { version = "=0.13.4", default-features = false, features = ["json", "rustls"] }
|
||||
rustls = "=0.23.42"
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
mod worker;
|
||||
|
||||
pub use worker::{
|
||||
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
|
||||
};
|
||||
|
||||
pub const COMPONENT_NAME: &str = "remora-api";
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
613
crates/remora-api/src/worker.rs
Normal file
613
crates/remora-api/src/worker.rs
Normal file
|
|
@ -0,0 +1,613 @@
|
|||
use std::{
|
||||
any::Any,
|
||||
future::{Ready, ready},
|
||||
};
|
||||
|
||||
use actix_web::{
|
||||
FromRequest, HttpRequest, HttpResponse, ResponseError, dev::Payload, http::StatusCode, web,
|
||||
};
|
||||
use remora_attempt::{
|
||||
ActivationMutationOutcome, AttemptAuthority, AttemptError, AttemptMutationOutcome,
|
||||
AttemptService, CheckpointProjection, CheckpointRecord, ClaimOutcome, DurableDescriptor,
|
||||
LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord, RenewalMutationOutcome,
|
||||
};
|
||||
use remora_contracts::{
|
||||
ActivationResponseDto, ApiErrorCodeDto, ApiErrorDto, AttemptActivateRequestDto,
|
||||
AttemptAuthorityDto, AttemptCheckpointRequestDto, AttemptOutcomeDto, AttemptOutcomeRequestDto,
|
||||
AttemptProgressRequestDto, AttemptRenewRequestDto, AttemptResolvedSourceRequestDto,
|
||||
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||
CredentialGrantVersionDto, DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto,
|
||||
ExecutionEnvelopeDto, JobPhaseProjectionDto, LeaseProjectionDto, NullableDto,
|
||||
PositiveIntegerDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||
SafeIntegerDto, SourceProjectionDto, StagingTargetDto, TargetDto,
|
||||
};
|
||||
use remora_domain::OpaqueId;
|
||||
use remora_result::PublicationInput;
|
||||
use remora_target::{ContentSha256, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, StoredObject};
|
||||
use sha2::{Digest, Sha256};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct AuthenticatedWorker {
|
||||
pub identity: OpaqueId,
|
||||
}
|
||||
|
||||
pub fn authenticate_rustls_worker(
|
||||
connection: &dyn Any,
|
||||
extensions: &mut actix_web::dev::Extensions,
|
||||
) {
|
||||
type Stream = actix_tls::accept::rustls_0_23::TlsStream<actix_web::rt::net::TcpStream>;
|
||||
let Some(stream) = connection.downcast_ref::<Stream>() else {
|
||||
return;
|
||||
};
|
||||
let Some(certificate) = stream
|
||||
.get_ref()
|
||||
.1
|
||||
.peer_certificates()
|
||||
.and_then(|chain| chain.first())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let digest = Sha256::digest(certificate.as_ref());
|
||||
let mut value = String::from("w");
|
||||
for byte in digest {
|
||||
use std::fmt::Write as _;
|
||||
write!(value, "{byte:02x}").expect("writing to String cannot fail");
|
||||
}
|
||||
extensions.insert(AuthenticatedWorker {
|
||||
identity: OpaqueId::parse(value).expect("certificate digest is a valid identity"),
|
||||
});
|
||||
}
|
||||
|
||||
impl FromRequest for AuthenticatedWorker {
|
||||
type Error = WorkerApiError;
|
||||
type Future = Ready<Result<Self, Self::Error>>;
|
||||
|
||||
fn from_request(request: &HttpRequest, _payload: &mut Payload) -> Self::Future {
|
||||
ready(
|
||||
request
|
||||
.conn_data::<Self>()
|
||||
.cloned()
|
||||
.ok_or_else(WorkerApiError::unauthenticated),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct WorkerApiState {
|
||||
attempts: AttemptService,
|
||||
}
|
||||
|
||||
impl WorkerApiState {
|
||||
#[must_use]
|
||||
pub fn new(attempts: AttemptService) -> Self {
|
||||
Self { attempts }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
||||
config.service(
|
||||
web::scope("/internal/v1")
|
||||
.app_data(
|
||||
web::JsonConfig::default()
|
||||
.limit(1024 * 1024)
|
||||
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
|
||||
)
|
||||
.route("/jobs/{job_id}/claim", web::post().to(claim))
|
||||
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
|
||||
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
|
||||
.route(
|
||||
"/attempts/{attempt_id}/resolved-source",
|
||||
web::post().to(pin_resolved_source),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/checkpoint",
|
||||
web::post().to(persist_checkpoint),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/progress",
|
||||
web::post().to(report_progress),
|
||||
)
|
||||
.route(
|
||||
"/attempts/{attempt_id}/outcome",
|
||||
web::post().to(report_outcome),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async fn claim(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<ClaimRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let job_id = parse_id(path.into_inner())?;
|
||||
let outcome = state
|
||||
.attempts
|
||||
.claim(&job_id, &request.worker_incarnation, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
let ClaimOutcome::Claimed { authority, .. } = outcome else {
|
||||
return Err(WorkerApiError::precondition_failed());
|
||||
};
|
||||
let execution = load_execution_envelope(&state.attempts, &authority).await?;
|
||||
Ok(HttpResponse::Ok().json(ClaimResponseDto {
|
||||
authority: authority_dto(&authority)?,
|
||||
execution,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn renew(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptRenewRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
match state
|
||||
.attempts
|
||||
.renew_operation(&authority, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
{
|
||||
RenewalMutationOutcome::Renewed { lease, .. } => {
|
||||
Ok(HttpResponse::Ok().json(lease_dto(lease)?))
|
||||
}
|
||||
RenewalMutationOutcome::AuthorityLost { .. } => Err(WorkerApiError::precondition_failed()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn activate(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptActivateRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let lease = match state
|
||||
.attempts
|
||||
.activate_operation(&authority, &request.operation_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
{
|
||||
ActivationMutationOutcome::Activated { lease, .. } => lease,
|
||||
ActivationMutationOutcome::PreconditionFailed => {
|
||||
return Err(WorkerApiError::precondition_failed());
|
||||
}
|
||||
};
|
||||
let execution = load_execution_envelope(&state.attempts, &authority).await?;
|
||||
Ok(HttpResponse::Ok().json(ActivationResponseDto {
|
||||
authority: authority_dto(&authority)?,
|
||||
execution,
|
||||
lease: lease_dto(lease)?,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn pin_resolved_source(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptResolvedSourceRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
accepted(
|
||||
state
|
||||
.attempts
|
||||
.pin_resolved_source(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
descriptor_from_resolved(&request.resolved_source_ref)?,
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn persist_checkpoint(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptCheckpointRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
accepted(
|
||||
state
|
||||
.attempts
|
||||
.persist_checkpoint(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
CheckpointRecord {
|
||||
descriptor: descriptor_from_checkpoint(&request.checkpoint_ref)?,
|
||||
resolved_source_id: request.resolved_source_id.clone(),
|
||||
validated_bytes: request.validated_bytes.get(),
|
||||
validated_members: request.validated_members.get(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn report_progress(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptProgressRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
accepted(
|
||||
state
|
||||
.attempts
|
||||
.report_progress(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
ProgressRecord {
|
||||
completed_bytes: request.progress.completed_bytes.get(),
|
||||
total_bytes: request.progress.total_bytes.map(SafeIntegerDto::get),
|
||||
completed_members: request.progress.completed_members.get(),
|
||||
total_members: request.progress.total_members.map(SafeIntegerDto::get),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?,
|
||||
)
|
||||
}
|
||||
|
||||
async fn report_outcome(
|
||||
_: AuthenticatedWorker,
|
||||
state: web::Data<WorkerApiState>,
|
||||
path: web::Path<String>,
|
||||
request: web::Json<AttemptOutcomeRequestDto>,
|
||||
) -> Result<HttpResponse, WorkerApiError> {
|
||||
let authority = request_authority(
|
||||
&state,
|
||||
path.into_inner(),
|
||||
request.fence.get(),
|
||||
&request.worker_incarnation,
|
||||
)
|
||||
.await?;
|
||||
let outcome = match &request.outcome {
|
||||
AttemptOutcomeDto::Success(success) => {
|
||||
let result = &success.result;
|
||||
state
|
||||
.attempts
|
||||
.publish_success_operation(
|
||||
&authority,
|
||||
&request.operation_id,
|
||||
PublicationInput {
|
||||
attempt_id: authority.attempt_id.clone(),
|
||||
manifest_id: result.manifest_id.clone(),
|
||||
schema_version: u32::try_from(result.schema_version.get())
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
manifest: StoredObject {
|
||||
identity: ObjectIdentity {
|
||||
target_store_id: result.manifest.store.clone(),
|
||||
bucket: result.manifest.bucket.clone(),
|
||||
key: result.manifest.key.clone(),
|
||||
version_id: result.manifest.version_id.clone(),
|
||||
},
|
||||
content_sha256: ContentSha256::parse(&result.manifest.checksum)
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
size: result.manifest.size.get(),
|
||||
},
|
||||
member_count: result.member_count.get(),
|
||||
total_bytes: result.total_bytes.get(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
AttemptOutcomeDto::Failure(failure) => {
|
||||
state
|
||||
.attempts
|
||||
.publish_failure_operation(&authority, &request.operation_id, failure.failure.class)
|
||||
.await
|
||||
}
|
||||
}
|
||||
.map_err(WorkerApiError::from_attempt)?;
|
||||
let phase = match outcome {
|
||||
OutcomePublication::Accepted { phase, .. } => phase,
|
||||
OutcomePublication::Rejected { reason, .. } => {
|
||||
return Err(WorkerApiError::from_rejection(reason));
|
||||
}
|
||||
};
|
||||
Ok(HttpResponse::Ok().json(JobPhaseProjectionDto { phase }))
|
||||
}
|
||||
|
||||
fn accepted(outcome: AttemptMutationOutcome) -> Result<HttpResponse, WorkerApiError> {
|
||||
match outcome {
|
||||
AttemptMutationOutcome::Accepted(_) => Ok(HttpResponse::NoContent().finish()),
|
||||
AttemptMutationOutcome::Rejected { reason, .. } => {
|
||||
Err(WorkerApiError::from_rejection(reason))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn request_authority(
|
||||
state: &WorkerApiState,
|
||||
attempt_id: String,
|
||||
fence: u64,
|
||||
worker: &OpaqueId,
|
||||
) -> Result<AttemptAuthority, WorkerApiError> {
|
||||
let attempt_id = parse_id(attempt_id)?;
|
||||
state
|
||||
.attempts
|
||||
.historical_authority(&attempt_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
.filter(|authority| authority.fence == fence && authority.worker_incarnation == *worker)
|
||||
.ok_or_else(WorkerApiError::precondition_failed)
|
||||
}
|
||||
|
||||
async fn load_execution_envelope(
|
||||
attempts: &AttemptService,
|
||||
authority: &AttemptAuthority,
|
||||
) -> Result<ExecutionEnvelopeDto, WorkerApiError> {
|
||||
let row = attempts
|
||||
.execution_envelope(&authority.attempt_id)
|
||||
.await
|
||||
.map_err(WorkerApiError::from_attempt)?
|
||||
.ok_or_else(WorkerApiError::precondition_failed)?;
|
||||
if row.job_id != authority.job_id
|
||||
|| row.attempt_id != authority.attempt_id
|
||||
|| row.fence != authority.fence
|
||||
|| row.worker_incarnation != authority.worker_incarnation
|
||||
{
|
||||
return Err(WorkerApiError::precondition_failed());
|
||||
}
|
||||
let source: SourceProjectionDto =
|
||||
serde_json::from_value(row.source).map_err(|_| WorkerApiError::unavailable())?;
|
||||
let target_prefix = row.target_prefix.join("/");
|
||||
let resolved_source_ref = row.resolved_source.map(|descriptor| {
|
||||
NullableDto(Some(descriptor_ref(
|
||||
descriptor,
|
||||
DurableDescriptorKindDto::ResolvedSource,
|
||||
)))
|
||||
});
|
||||
let checkpoint_ref = row.checkpoint.and_then(|checkpoint| match checkpoint {
|
||||
CheckpointProjection::Reusable { checkpoint } => Some(NullableDto(Some(descriptor_ref(
|
||||
checkpoint.descriptor,
|
||||
DurableDescriptorKindDto::Checkpoint,
|
||||
)))),
|
||||
CheckpointProjection::Complete { .. } => None,
|
||||
});
|
||||
Ok(ExecutionEnvelopeDto {
|
||||
id: row.id,
|
||||
job_id: authority.job_id.clone(),
|
||||
attempt_id: authority.attempt_id.clone(),
|
||||
fence: positive(authority.fence)?,
|
||||
worker_incarnation: authority.worker_incarnation.clone(),
|
||||
source,
|
||||
sensitive_source_ref: row.sensitive_source_ref,
|
||||
credential_version: NullableDto(match (row.credential_id, row.credential_version_id) {
|
||||
(Some(credential_id), Some(version_id)) => Some(CredentialGrantVersionDto {
|
||||
credential_id,
|
||||
version_id,
|
||||
}),
|
||||
(None, None) => None,
|
||||
_ => return Err(WorkerApiError::unavailable()),
|
||||
}),
|
||||
resolved_source_ref: resolved_source_ref.unwrap_or(NullableDto(None)),
|
||||
target: TargetDto {
|
||||
store: row.target_store.clone(),
|
||||
bucket: row.target_bucket.clone(),
|
||||
prefix: target_prefix,
|
||||
},
|
||||
staging_target: StagingTargetDto {
|
||||
store: row.target_store,
|
||||
bucket: row.target_bucket,
|
||||
prefix: format!(
|
||||
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/",
|
||||
authority.attempt_id.as_str()
|
||||
),
|
||||
},
|
||||
execution_grant_ref: row.execution_grant_id,
|
||||
checkpoint_ref: checkpoint_ref.unwrap_or(NullableDto(None)),
|
||||
})
|
||||
}
|
||||
|
||||
fn descriptor_ref(
|
||||
descriptor: DurableDescriptor,
|
||||
kind: DurableDescriptorKindDto,
|
||||
) -> DurableDescriptorRefDto {
|
||||
DurableDescriptorRefDto {
|
||||
id: descriptor.id,
|
||||
kind,
|
||||
schema_version: positive(descriptor.schema_version.into()).expect("validated descriptor"),
|
||||
byte_length: safe(descriptor.byte_length).expect("validated descriptor"),
|
||||
item_count: descriptor
|
||||
.item_count
|
||||
.map(|value| safe(value).expect("validated descriptor")),
|
||||
checksum: descriptor.checksum,
|
||||
}
|
||||
}
|
||||
|
||||
fn descriptor_from_resolved(
|
||||
value: &ResolvedSourceDescriptorRefDto,
|
||||
) -> Result<DurableDescriptor, WorkerApiError> {
|
||||
let ResolvedSourceDescriptorKindDto::ResolvedSource = value.kind;
|
||||
Ok(DurableDescriptor {
|
||||
id: value.id.clone(),
|
||||
schema_version: u32::try_from(value.schema_version.get())
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
byte_length: value.byte_length.get(),
|
||||
item_count: value.item_count.map(SafeIntegerDto::get),
|
||||
checksum: value.checksum.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn descriptor_from_checkpoint(
|
||||
value: &CheckpointDescriptorRefDto,
|
||||
) -> Result<DurableDescriptor, WorkerApiError> {
|
||||
let CheckpointDescriptorKindDto::Checkpoint = value.kind;
|
||||
Ok(DurableDescriptor {
|
||||
id: value.id.clone(),
|
||||
schema_version: u32::try_from(value.schema_version.get())
|
||||
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||
byte_length: value.byte_length.get(),
|
||||
item_count: value.item_count.map(SafeIntegerDto::get),
|
||||
checksum: value.checksum.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn authority_dto(authority: &AttemptAuthority) -> Result<AttemptAuthorityDto, WorkerApiError> {
|
||||
Ok(AttemptAuthorityDto {
|
||||
attempt_id: authority.attempt_id.clone(),
|
||||
fence: positive(authority.fence)?,
|
||||
worker_incarnation: authority.worker_incarnation.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
fn lease_dto(lease: LeaseProjection) -> Result<LeaseProjectionDto, WorkerApiError> {
|
||||
let observed = timestamp(lease.observed_at_ms)?;
|
||||
let expires = timestamp(lease.expires_at_ms)?;
|
||||
let renew_at = observed + (expires - observed) / 2;
|
||||
Ok(LeaseProjectionDto {
|
||||
expires_at: datetime(expires)?,
|
||||
renew_after: datetime(renew_at)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn timestamp(milliseconds: u64) -> Result<OffsetDateTime, WorkerApiError> {
|
||||
let nanoseconds = i128::from(milliseconds) * 1_000_000;
|
||||
OffsetDateTime::from_unix_timestamp_nanos(nanoseconds)
|
||||
.map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn datetime(value: OffsetDateTime) -> Result<DateTimeDto, WorkerApiError> {
|
||||
let value = value
|
||||
.format(&Rfc3339)
|
||||
.map_err(|_| WorkerApiError::unavailable())?;
|
||||
DateTimeDto::parse(value).map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn positive(value: u64) -> Result<PositiveIntegerDto, WorkerApiError> {
|
||||
PositiveIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn safe(value: u64) -> Result<SafeIntegerDto, WorkerApiError> {
|
||||
SafeIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
|
||||
}
|
||||
|
||||
fn parse_id(value: impl Into<String>) -> Result<OpaqueId, WorkerApiError> {
|
||||
OpaqueId::parse(value).map_err(|_| WorkerApiError::invalid_request())
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("worker API request failed")]
|
||||
pub struct WorkerApiError {
|
||||
status: StatusCode,
|
||||
code: ApiErrorCodeDto,
|
||||
message: &'static str,
|
||||
}
|
||||
|
||||
impl WorkerApiError {
|
||||
fn invalid_request() -> Self {
|
||||
Self::new(
|
||||
StatusCode::BAD_REQUEST,
|
||||
ApiErrorCodeDto::InvalidRequest,
|
||||
"invalid request",
|
||||
)
|
||||
}
|
||||
|
||||
fn unauthenticated() -> Self {
|
||||
Self::new(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
ApiErrorCodeDto::Unauthenticated,
|
||||
"authentication required",
|
||||
)
|
||||
}
|
||||
|
||||
fn precondition_failed() -> Self {
|
||||
Self::new(
|
||||
StatusCode::CONFLICT,
|
||||
ApiErrorCodeDto::PreconditionFailed,
|
||||
"precondition failed",
|
||||
)
|
||||
}
|
||||
|
||||
fn unavailable() -> Self {
|
||||
Self::new(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
ApiErrorCodeDto::Unavailable,
|
||||
"service unavailable",
|
||||
)
|
||||
}
|
||||
|
||||
fn new(status: StatusCode, code: ApiErrorCodeDto, message: &'static str) -> Self {
|
||||
Self {
|
||||
status,
|
||||
code,
|
||||
message,
|
||||
}
|
||||
}
|
||||
|
||||
fn from_attempt(error: AttemptError) -> Self {
|
||||
match error {
|
||||
AttemptError::Invalid(_) => Self::invalid_request(),
|
||||
AttemptError::OperationConflict => Self::precondition_failed(),
|
||||
AttemptError::Coordination(_) | AttemptError::Durable(_) => Self::unavailable(),
|
||||
}
|
||||
}
|
||||
|
||||
fn from_rejection(_reason: MutationRejection) -> Self {
|
||||
Self::precondition_failed()
|
||||
}
|
||||
}
|
||||
|
||||
impl ResponseError for WorkerApiError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
self.status
|
||||
}
|
||||
|
||||
fn error_response(&self) -> HttpResponse {
|
||||
HttpResponse::build(self.status)
|
||||
.content_type("application/problem+json")
|
||||
.json(ApiErrorDto {
|
||||
code: self.code,
|
||||
message: self.message.into(),
|
||||
request_id: request_id(),
|
||||
fields: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn request_id() -> OpaqueId {
|
||||
let mut bytes = [0_u8; 16];
|
||||
getrandom::fill(&mut bytes).expect("operating system randomness is required");
|
||||
let mut value = String::from("r");
|
||||
for byte in bytes {
|
||||
use std::fmt::Write as _;
|
||||
write!(value, "{byte:02x}").expect("writing to String cannot fail");
|
||||
}
|
||||
OpaqueId::parse(value).expect("generated request identifier is valid")
|
||||
}
|
||||
735
crates/remora-api/tests/worker_api.rs
Normal file
735
crates/remora-api/tests/worker_api.rs
Normal file
|
|
@ -0,0 +1,735 @@
|
|||
use std::{net::TcpListener, sync::Arc, time::Duration};
|
||||
|
||||
use actix_web::{App, HttpResponse, HttpServer, web};
|
||||
use rcgen::{
|
||||
BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer,
|
||||
KeyPair, KeyUsagePurpose,
|
||||
};
|
||||
use remora_admission::{
|
||||
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
|
||||
};
|
||||
use remora_api::{
|
||||
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
|
||||
};
|
||||
use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig, migrate as attempt_migrate};
|
||||
use remora_contracts::{
|
||||
ApiErrorDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto, AttemptFailureOutcomeDto,
|
||||
AttemptOutcomeDto, AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, CheckpointDescriptorKindDto,
|
||||
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, HttpSourceSubmissionDto,
|
||||
JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto, ProgressValueDto,
|
||||
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, ResultDescriptorDto,
|
||||
ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
|
||||
};
|
||||
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||
use remora_credentials::migrate as credentials_migrate;
|
||||
use remora_domain::{
|
||||
BucketName, JobPhase, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
|
||||
};
|
||||
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
||||
use remora_identity::{DefaultTarget, Principal};
|
||||
use remora_result::migrate as result_migrate;
|
||||
use remora_target::attempt_manifest_key;
|
||||
use reqwest::{Certificate as ClientCertificate, Client, Identity};
|
||||
use rustls::{
|
||||
RootCertStore, ServerConfig,
|
||||
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||
server::WebPkiClientVerifier,
|
||||
};
|
||||
use sqlx::PgPool;
|
||||
|
||||
struct Harness {
|
||||
pool: PgPool,
|
||||
coordination: CoordinationStore,
|
||||
}
|
||||
|
||||
impl Harness {
|
||||
async fn new(name: &str) -> Option<Self> {
|
||||
let database_url = required_env("DATABASE_URL")?;
|
||||
let coordination_url = required_env("REMORA_COORDINATION_URL")?;
|
||||
let pool = connect(
|
||||
&database_url,
|
||||
PoolConfig {
|
||||
max_connections: 16,
|
||||
acquire_timeout: Duration::from_secs(10),
|
||||
idle_timeout: Some(Duration::from_secs(30)),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
durable_migrate(&pool).await.unwrap();
|
||||
credentials_migrate(&pool).await.unwrap();
|
||||
admission_migrate(&pool).await.unwrap();
|
||||
attempt_migrate(&pool).await.unwrap();
|
||||
result_migrate(&pool).await.unwrap();
|
||||
sqlx::query("TRUNCATE TABLE jobs CASCADE")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let coordination = CoordinationStore::connect(
|
||||
&coordination_url,
|
||||
format!("{}wp18:{name}:{}:", coordination_namespace(), nonce()),
|
||||
StoreConfig {
|
||||
global_limit: 16,
|
||||
lease_ttl: Duration::from_secs(10),
|
||||
reserve_ttl: Duration::from_secs(10),
|
||||
progress_ttl: Duration::from_secs(10),
|
||||
cancel_ttl: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
Some(Self { pool, coordination })
|
||||
}
|
||||
|
||||
fn service(&self) -> AttemptService {
|
||||
AttemptService::new(
|
||||
self.pool.clone(),
|
||||
self.coordination.clone(),
|
||||
AttemptConfig {
|
||||
max_attempts: 3,
|
||||
local_limit: 8,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn job(&self) -> OpaqueId {
|
||||
let store = TargetStoreId::parse("primary").unwrap();
|
||||
let bucket = BucketName::parse("remora-results").unwrap();
|
||||
let prefix = TargetPrefix::parse("users/alice", ["_remora"]).unwrap();
|
||||
let principal = Principal {
|
||||
id: unique_id("owner"),
|
||||
issuer: "https://issuer.example".into(),
|
||||
subject: "subject".into(),
|
||||
administrator: false,
|
||||
scopes: [StorageScope {
|
||||
store: store.clone(),
|
||||
bucket: bucket.clone(),
|
||||
prefix: prefix.clone(),
|
||||
}]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
};
|
||||
let service = AdmissionService::new(
|
||||
self.pool.clone(),
|
||||
AdmissionConfig {
|
||||
default_target: DefaultTarget {
|
||||
store,
|
||||
bucket,
|
||||
prefix,
|
||||
},
|
||||
reserved_target_segments: vec!["_remora".into()],
|
||||
policy_version: "worker-api-v1".into(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let outcome = service
|
||||
.submit(
|
||||
&principal,
|
||||
unique_id("submit").as_str(),
|
||||
JobSubmissionDto {
|
||||
source: SourceSubmissionDto::Http(HttpSourceSubmissionDto {
|
||||
url: "https://example.invalid/artifact".into(),
|
||||
result_path: "artifact.bin".into(),
|
||||
credential_id: None,
|
||||
}),
|
||||
target: None,
|
||||
label: Some("worker-api-test".into()),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let AdmissionDecision::Accepted(job) = outcome.decision else {
|
||||
panic!("job rejected")
|
||||
};
|
||||
self.coordination
|
||||
.enqueue(job.job_id.as_str())
|
||||
.await
|
||||
.unwrap();
|
||||
job.job_id
|
||||
}
|
||||
|
||||
async fn seal_result(&self, attempt_id: &OpaqueId) -> ResultDescriptorDto {
|
||||
let manifest_id = unique_id("manifest");
|
||||
let checksum = format!("sha256:{}", "11".repeat(32));
|
||||
let key = attempt_manifest_key(attempt_id, &manifest_id);
|
||||
sqlx::query(
|
||||
"INSERT INTO result_inventories (
|
||||
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||
manifest_size, member_count, total_bytes
|
||||
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
|
||||
'version-1', $4, 128, 1, 4096)",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.bind(manifest_id.as_str())
|
||||
.bind(&key)
|
||||
.bind(&checksum)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
ResultDescriptorDto {
|
||||
kind: ResultKindDto::Manifest,
|
||||
manifest_id,
|
||||
schema_version: positive(1),
|
||||
manifest: remora_contracts::ObjectIdentityDto {
|
||||
store: "primary".into(),
|
||||
bucket: "remora-results".into(),
|
||||
key,
|
||||
version_id: Some("version-1".into()),
|
||||
checksum,
|
||||
size: safe(128),
|
||||
},
|
||||
member_count: positive(1),
|
||||
total_bytes: safe(4096),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
||||
let Some(harness) = Harness::new("flow").await else {
|
||||
return;
|
||||
};
|
||||
let state = WorkerApiState::new(harness.service());
|
||||
let (base, server, client) = start_authenticated_server(state).await;
|
||||
let job_id = harness.job().await;
|
||||
let worker = unique_id("worker");
|
||||
let claim_operation = unique_id("operation");
|
||||
let claim = ClaimRequestDto {
|
||||
operation_id: claim_operation.clone(),
|
||||
worker_incarnation: worker.clone(),
|
||||
};
|
||||
|
||||
let unauthenticated = actix_web::test::init_service(App::new().route(
|
||||
"/",
|
||||
web::post().to(|_: AuthenticatedWorker| async { HttpResponse::NoContent().finish() }),
|
||||
))
|
||||
.await;
|
||||
let response = actix_web::test::call_service(
|
||||
&unauthenticated,
|
||||
actix_web::test::TestRequest::post().uri("/").to_request(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), 401);
|
||||
assert_eq!(
|
||||
response.headers().get("content-type").unwrap(),
|
||||
"application/problem+json"
|
||||
);
|
||||
|
||||
let claim_url = format!("{base}/internal/v1/jobs/{job_id}/claim");
|
||||
let response = client.post(&claim_url).json(&claim).send().await.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let claimed: ClaimResponseDto = response.json().await.unwrap();
|
||||
assert_eq!(claimed.authority.worker_incarnation, worker);
|
||||
assert_eq!(claimed.execution.attempt_id, claimed.authority.attempt_id);
|
||||
assert_eq!(claimed.execution.job_id, job_id);
|
||||
assert!(
|
||||
claimed
|
||||
.execution
|
||||
.staging_target
|
||||
.prefix
|
||||
.contains(claimed.authority.attempt_id.as_str())
|
||||
);
|
||||
|
||||
let changed_claim = client
|
||||
.post(&claim_url)
|
||||
.json(&ClaimRequestDto {
|
||||
operation_id: claim_operation,
|
||||
worker_incarnation: unique_id("worker"),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(changed_claim.status(), 409);
|
||||
assert_eq!(
|
||||
changed_claim.headers().get("content-type").unwrap(),
|
||||
"application/problem+json"
|
||||
);
|
||||
|
||||
let attempt_url = format!(
|
||||
"{base}/internal/v1/attempts/{}",
|
||||
claimed.authority.attempt_id
|
||||
);
|
||||
let activation_operation = unique_id("operation");
|
||||
let activation = AttemptActivateRequestDto {
|
||||
operation_id: activation_operation,
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
};
|
||||
let response = client
|
||||
.post(format!("{attempt_url}/activate"))
|
||||
.json(&activation)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let activation_body: serde_json::Value = response.json().await.unwrap();
|
||||
let activated: remora_contracts::ActivationResponseDto =
|
||||
serde_json::from_value(activation_body.clone()).unwrap();
|
||||
assert!(activated.lease.renew_after.as_str() < activated.lease.expires_at.as_str());
|
||||
|
||||
let renewal_operation = unique_id("operation");
|
||||
let renewal = AttemptRenewRequestDto {
|
||||
operation_id: renewal_operation.clone(),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
};
|
||||
let first_renewal = client
|
||||
.post(format!("{attempt_url}/renew"))
|
||||
.json(&renewal)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first_renewal.status(), 200);
|
||||
let first_lease: serde_json::Value = first_renewal.json().await.unwrap();
|
||||
let replayed_lease: serde_json::Value = client
|
||||
.post(format!("{attempt_url}/renew"))
|
||||
.json(&renewal)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replayed_lease, first_lease);
|
||||
let replayed_activation: serde_json::Value = client
|
||||
.post(format!("{attempt_url}/activate"))
|
||||
.json(&activation)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replayed_activation, activation_body);
|
||||
|
||||
let authority_mismatches = [
|
||||
(
|
||||
format!("{base}/internal/v1/attempts/{}/renew", unique_id("attempt")),
|
||||
claimed.authority.fence,
|
||||
worker.clone(),
|
||||
),
|
||||
(
|
||||
format!("{attempt_url}/renew"),
|
||||
positive(claimed.authority.fence.get() + 1),
|
||||
worker.clone(),
|
||||
),
|
||||
(
|
||||
format!("{attempt_url}/renew"),
|
||||
claimed.authority.fence,
|
||||
unique_id("worker"),
|
||||
),
|
||||
];
|
||||
for (url, fence, worker_incarnation) in authority_mismatches {
|
||||
let response = client
|
||||
.post(url)
|
||||
.json(&AttemptRenewRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence,
|
||||
worker_incarnation,
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 409);
|
||||
}
|
||||
|
||||
let resolved_id = unique_id("resolved");
|
||||
let resolved = AttemptResolvedSourceRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
resolved_source_ref: ResolvedSourceDescriptorRefDto {
|
||||
id: resolved_id.clone(),
|
||||
kind: ResolvedSourceDescriptorKindDto::ResolvedSource,
|
||||
schema_version: positive(1),
|
||||
byte_length: safe(64),
|
||||
item_count: Some(safe(1)),
|
||||
checksum: format!("sha256:{}", "22".repeat(32)),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/resolved-source"))
|
||||
.json(&resolved)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
let checkpoint = AttemptCheckpointRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
checkpoint_ref: CheckpointDescriptorRefDto {
|
||||
id: unique_id("checkpoint"),
|
||||
kind: CheckpointDescriptorKindDto::Checkpoint,
|
||||
schema_version: positive(1),
|
||||
byte_length: safe(32),
|
||||
item_count: Some(safe(1)),
|
||||
checksum: format!("sha256:{}", "33".repeat(32)),
|
||||
},
|
||||
resolved_source_id: resolved_id,
|
||||
validated_bytes: safe(1024),
|
||||
validated_members: safe(1),
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/checkpoint"))
|
||||
.json(&checkpoint)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
let progress = AttemptProgressRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
progress: ProgressValueDto {
|
||||
completed_bytes: safe(2048),
|
||||
total_bytes: Some(safe(4096)),
|
||||
completed_members: safe(1),
|
||||
total_members: Some(safe(1)),
|
||||
},
|
||||
};
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/progress"))
|
||||
.json(&progress)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
204
|
||||
);
|
||||
|
||||
let replayed_claim: ClaimResponseDto = client
|
||||
.post(&claim_url)
|
||||
.json(&claim)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(replayed_claim).unwrap(),
|
||||
serde_json::to_value(&claimed).unwrap()
|
||||
);
|
||||
|
||||
let failure = AttemptOutcomeRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
outcome: AttemptOutcomeDto::Failure(AttemptFailureOutcomeDto {
|
||||
failure: AttemptRuntimeFailureDto {
|
||||
class: RuntimeFailureClass::TransientSource,
|
||||
message: "secret-value-must-not-escape".into(),
|
||||
safe_details: None,
|
||||
},
|
||||
}),
|
||||
};
|
||||
let outcome_url = format!("{attempt_url}/outcome");
|
||||
let response = client
|
||||
.post(&outcome_url)
|
||||
.json(&failure)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let bytes = response.bytes().await.unwrap();
|
||||
assert!(!String::from_utf8_lossy(&bytes).contains("secret-value-must-not-escape"));
|
||||
let projection: JobPhaseProjectionDto = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(projection.phase, JobPhase::RetryWait);
|
||||
let report = harness
|
||||
.service()
|
||||
.reconcile_once(&RecoveryConfig {
|
||||
retry_delay: Duration::ZERO,
|
||||
cleanup_claim_ttl: Duration::from_secs(1),
|
||||
batch_size: 128,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(report.retries_scheduled, 1);
|
||||
let replay = client
|
||||
.post(&outcome_url)
|
||||
.json(&failure)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(replay.status(), 200);
|
||||
assert_eq!(replay.bytes().await.unwrap(), bytes);
|
||||
|
||||
let cross_kind = client
|
||||
.post(format!("{attempt_url}/progress"))
|
||||
.json(&AttemptProgressRequestDto {
|
||||
operation_id: renewal_operation,
|
||||
fence: claimed.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
progress: ProgressValueDto {
|
||||
completed_bytes: safe(2048),
|
||||
total_bytes: Some(safe(4096)),
|
||||
completed_members: safe(1),
|
||||
total_members: Some(safe(1)),
|
||||
},
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cross_kind.status(), 409);
|
||||
assert_eq!(
|
||||
cross_kind.headers().get("content-type").unwrap(),
|
||||
"application/problem+json"
|
||||
);
|
||||
let error: ApiErrorDto = cross_kind.json().await.unwrap();
|
||||
assert_eq!(error.message, "precondition failed");
|
||||
server.stop(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn success_outcome_publishes_the_exact_manifest_and_replays() {
|
||||
let Some(harness) = Harness::new("success").await else {
|
||||
return;
|
||||
};
|
||||
let state = WorkerApiState::new(harness.service());
|
||||
let (base, server, client) = start_authenticated_server(state).await;
|
||||
let job_id = harness.job().await;
|
||||
let worker = unique_id("worker");
|
||||
let claim: ClaimResponseDto = client
|
||||
.post(format!("{base}/internal/v1/jobs/{job_id}/claim"))
|
||||
.json(&ClaimRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
worker_incarnation: worker.clone(),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let attempt_url = format!("{base}/internal/v1/attempts/{}", claim.authority.attempt_id);
|
||||
assert_eq!(
|
||||
client
|
||||
.post(format!("{attempt_url}/activate"))
|
||||
.json(&AttemptActivateRequestDto {
|
||||
operation_id: unique_id("operation"),
|
||||
fence: claim.authority.fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.status(),
|
||||
200
|
||||
);
|
||||
let result = harness.seal_result(&claim.authority.attempt_id).await;
|
||||
let operation = unique_id("operation");
|
||||
let outcome = AttemptOutcomeRequestDto {
|
||||
operation_id: operation,
|
||||
fence: claim.authority.fence,
|
||||
worker_incarnation: worker,
|
||||
outcome: AttemptOutcomeDto::Success(remora_contracts::AttemptSuccessOutcomeDto { result }),
|
||||
};
|
||||
let outcome_url = format!("{attempt_url}/outcome");
|
||||
for _ in 0..2 {
|
||||
let response = client
|
||||
.post(&outcome_url)
|
||||
.json(&outcome)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
assert_eq!(
|
||||
response
|
||||
.json::<JobPhaseProjectionDto>()
|
||||
.await
|
||||
.unwrap()
|
||||
.phase,
|
||||
JobPhase::Succeeded
|
||||
);
|
||||
}
|
||||
server.stop(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mutual_tls_requires_a_trusted_client_certificate() {
|
||||
let certificates = certificates();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = HttpServer::new(|| {
|
||||
App::new().route(
|
||||
"/identity",
|
||||
web::get().to(|identity: AuthenticatedWorker| async move {
|
||||
HttpResponse::Ok().body(identity.identity.to_string())
|
||||
}),
|
||||
)
|
||||
})
|
||||
.on_connect(authenticate_rustls_worker)
|
||||
.listen_rustls_0_23(listener, certificates.server_config)
|
||||
.unwrap()
|
||||
.run();
|
||||
let handle = server.handle();
|
||||
tokio::spawn(server);
|
||||
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
|
||||
let untrusted = Client::builder()
|
||||
.add_root_certificate(root.clone())
|
||||
.build()
|
||||
.unwrap();
|
||||
assert!(
|
||||
untrusted
|
||||
.get(format!("https://localhost:{}/identity", address.port()))
|
||||
.send()
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut client_pem = certificates.client.pem();
|
||||
client_pem.push_str(&certificates.client_key.serialize_pem());
|
||||
let trusted = Client::builder()
|
||||
.add_root_certificate(root)
|
||||
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
|
||||
.build()
|
||||
.unwrap();
|
||||
let response = trusted
|
||||
.get(format!("https://localhost:{}/identity", address.port()))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), 200);
|
||||
let expected = certificate_identity(certificates.client.der().as_ref());
|
||||
assert_eq!(response.text().await.unwrap(), expected);
|
||||
handle.stop(true).await;
|
||||
}
|
||||
|
||||
async fn start_authenticated_server(
|
||||
state: WorkerApiState,
|
||||
) -> (String, actix_web::dev::ServerHandle, Client) {
|
||||
let certificates = certificates();
|
||||
let client = trusted_client(&certificates);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = HttpServer::new(move || {
|
||||
App::new()
|
||||
.app_data(web::Data::new(state.clone()))
|
||||
.configure(configure_worker_api)
|
||||
})
|
||||
.on_connect(authenticate_rustls_worker)
|
||||
.listen_rustls_0_23(listener, certificates.server_config)
|
||||
.unwrap()
|
||||
.run();
|
||||
let handle = server.handle();
|
||||
tokio::spawn(server);
|
||||
(
|
||||
format!("https://localhost:{}", address.port()),
|
||||
handle,
|
||||
client,
|
||||
)
|
||||
}
|
||||
|
||||
fn trusted_client(certificates: &Certificates) -> Client {
|
||||
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
|
||||
let mut client_pem = certificates.client.pem();
|
||||
client_pem.push_str(&certificates.client_key.serialize_pem());
|
||||
Client::builder()
|
||||
.add_root_certificate(root)
|
||||
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn certificate_identity(certificate: &[u8]) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
let mut value = String::from("w");
|
||||
for byte in Sha256::digest(certificate) {
|
||||
use std::fmt::Write as _;
|
||||
write!(value, "{byte:02x}").unwrap();
|
||||
}
|
||||
value
|
||||
}
|
||||
|
||||
struct Certificates {
|
||||
ca: Certificate,
|
||||
client: Certificate,
|
||||
client_key: KeyPair,
|
||||
server_config: ServerConfig,
|
||||
}
|
||||
|
||||
fn certificates() -> Certificates {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let mut ca_params = CertificateParams::new(Vec::new()).unwrap();
|
||||
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||
ca_params.key_usages = vec![
|
||||
KeyUsagePurpose::DigitalSignature,
|
||||
KeyUsagePurpose::KeyCertSign,
|
||||
KeyUsagePurpose::CrlSign,
|
||||
];
|
||||
let ca_key = KeyPair::generate().unwrap();
|
||||
let ca = ca_params.self_signed(&ca_key).unwrap();
|
||||
let issuer = Issuer::new(ca_params, ca_key);
|
||||
|
||||
let server_key = KeyPair::generate().unwrap();
|
||||
let mut server_params = CertificateParams::new(vec!["localhost".into()]).unwrap();
|
||||
server_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
|
||||
server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
|
||||
let server = server_params.signed_by(&server_key, &issuer).unwrap();
|
||||
|
||||
let client_key = KeyPair::generate().unwrap();
|
||||
let mut client_params = CertificateParams::new(Vec::new()).unwrap();
|
||||
client_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
|
||||
client_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
|
||||
let client = client_params.signed_by(&client_key, &issuer).unwrap();
|
||||
|
||||
let mut roots = RootCertStore::empty();
|
||||
roots.add(ca.der().clone()).unwrap();
|
||||
let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
|
||||
.build()
|
||||
.unwrap();
|
||||
let server_key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(server_key.serialize_der()));
|
||||
let server_config = ServerConfig::builder()
|
||||
.with_client_cert_verifier(verifier)
|
||||
.with_single_cert(vec![server.der().clone()], server_key_der)
|
||||
.unwrap();
|
||||
Certificates {
|
||||
ca,
|
||||
client,
|
||||
client_key,
|
||||
server_config,
|
||||
}
|
||||
}
|
||||
|
||||
fn required_env(name: &str) -> Option<String> {
|
||||
match std::env::var(name) {
|
||||
Ok(value) if !value.is_empty() => Some(value),
|
||||
_ if std::env::var_os("REMORA_WORKER_API_REQUIRE").is_some() => {
|
||||
panic!("{name} is required")
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn coordination_namespace() -> String {
|
||||
std::env::var("REMORA_COORDINATION_NAMESPACE").unwrap_or_else(|_| "remora:test:".into())
|
||||
}
|
||||
|
||||
fn nonce() -> u128 {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
(u128::from(std::process::id()) << 64) | u128::from(COUNTER.fetch_add(1, Ordering::Relaxed))
|
||||
}
|
||||
|
||||
fn unique_id(prefix: &str) -> OpaqueId {
|
||||
OpaqueId::parse(format!("{prefix}{:032x}", nonce())).unwrap()
|
||||
}
|
||||
|
||||
fn positive(value: u64) -> PositiveIntegerDto {
|
||||
PositiveIntegerDto::new(value).unwrap()
|
||||
}
|
||||
|
||||
fn safe(value: u64) -> SafeIntegerDto {
|
||||
SafeIntegerDto::new(value).unwrap()
|
||||
}
|
||||
2
crates/remora-attempt/migrations/0004_worker_api.sql
Normal file
2
crates/remora-attempt/migrations/0004_worker_api.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE attempts
|
||||
ADD COLUMN execution_envelope JSONB;
|
||||
|
|
@ -20,6 +20,7 @@ pub use recovery_types::{
|
|||
};
|
||||
pub use service::{AttemptConfig, AttemptService};
|
||||
pub use types::{
|
||||
ActivationOutcome, AttemptAuthority, AttemptFaultInjector, AttemptFaultPoint, AttemptState,
|
||||
ClaimOutcome, ClaimRejection, RecoveryOutcome, RenewalOutcome,
|
||||
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptFaultInjector,
|
||||
AttemptFaultPoint, AttemptState, ClaimOutcome, ClaimRejection, ExecutionEnvelopeRecord,
|
||||
LeaseProjection, OutcomePublication, RecoveryOutcome, RenewalMutationOutcome, RenewalOutcome,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ use time::OffsetDateTime;
|
|||
use crate::{
|
||||
AttemptAuthority, AttemptError, AttemptMutationOutcome, AttemptService, CheckpointProjection,
|
||||
CheckpointRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
|
||||
JobMutationProjection, MutationRejection, ProgressRecord,
|
||||
JobMutationProjection, MutationRejection, OutcomePublication, ProgressRecord,
|
||||
};
|
||||
|
||||
impl AttemptService {
|
||||
|
|
@ -64,6 +64,18 @@ impl AttemptService {
|
|||
.await
|
||||
}
|
||||
|
||||
pub async fn publish_success_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
result: PublicationInput,
|
||||
) -> Result<OutcomePublication, AttemptError> {
|
||||
let outcome = self
|
||||
.publish_success(authority, operation_id, result)
|
||||
.await?;
|
||||
self.publication_outcome(operation_id, outcome).await
|
||||
}
|
||||
|
||||
pub async fn persist_checkpoint(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
|
|
@ -124,6 +136,16 @@ impl AttemptService {
|
|||
.await
|
||||
}
|
||||
|
||||
pub async fn publish_failure_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
class: RuntimeFailureClass,
|
||||
) -> Result<OutcomePublication, AttemptError> {
|
||||
let outcome = self.publish_failure(authority, operation_id, class).await?;
|
||||
self.publication_outcome(operation_id, outcome).await
|
||||
}
|
||||
|
||||
pub async fn report_progress(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
|
|
@ -227,7 +249,7 @@ impl AttemptService {
|
|||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
enqueue(tx, "job.updated", json!({ "job_id": authority.job_id })).await?;
|
||||
accepted()
|
||||
accepted_phase(JobPhase::Succeeded)
|
||||
})
|
||||
},
|
||||
)
|
||||
|
|
@ -307,7 +329,11 @@ impl AttemptService {
|
|||
json!({ "job_id": authority.job_id }),
|
||||
)
|
||||
.await?;
|
||||
accepted()
|
||||
accepted_phase(if retry {
|
||||
JobPhase::RetryWait
|
||||
} else {
|
||||
JobPhase::Failed
|
||||
})
|
||||
})
|
||||
},
|
||||
)
|
||||
|
|
@ -329,6 +355,43 @@ impl AttemptService {
|
|||
row.map(ProjectionRow::try_into_projection).transpose()
|
||||
}
|
||||
|
||||
async fn publication_outcome(
|
||||
&self,
|
||||
operation_id: &OpaqueId,
|
||||
outcome: AttemptMutationOutcome,
|
||||
) -> Result<OutcomePublication, AttemptError> {
|
||||
match outcome {
|
||||
AttemptMutationOutcome::Rejected {
|
||||
reason,
|
||||
disposition,
|
||||
} => Ok(OutcomePublication::Rejected {
|
||||
reason,
|
||||
disposition,
|
||||
}),
|
||||
AttemptMutationOutcome::Accepted(disposition) => {
|
||||
let response = sqlx::query_scalar::<_, Json<Value>>(
|
||||
"SELECT response_body FROM mutation_records
|
||||
WHERE scope_subject = $1 AND scope_operation = 'attempt.mutation'
|
||||
AND operation_key = $2",
|
||||
)
|
||||
.bind(self.operation_subject.as_str())
|
||||
.bind(operation_id.as_str())
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(DurableError::from)?;
|
||||
let phase = serde_json::from_value(
|
||||
response
|
||||
.0
|
||||
.get("phase")
|
||||
.cloned()
|
||||
.ok_or_else(|| AttemptError::Invalid("missing outcome phase".into()))?,
|
||||
)
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
Ok(OutcomePublication::Accepted { disposition, phase })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_durable_mutation<T, D>(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
|
|
@ -423,6 +486,13 @@ fn accepted() -> Result<(MutationStatus, Value), DurableError> {
|
|||
Ok((MutationStatus::Accepted, json!({ "accepted": true })))
|
||||
}
|
||||
|
||||
fn accepted_phase(phase: JobPhase) -> Result<(MutationStatus, Value), DurableError> {
|
||||
Ok((
|
||||
MutationStatus::Accepted,
|
||||
json!({ "accepted": true, "phase": phase }),
|
||||
))
|
||||
}
|
||||
|
||||
fn rejected(reason: MutationRejection) -> Result<(MutationStatus, Value), DurableError> {
|
||||
Ok((MutationStatus::Rejected, json!({ "reason": reason })))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@ use sqlx::{PgPool, types::Json};
|
|||
use time::OffsetDateTime;
|
||||
|
||||
use crate::{
|
||||
ActivationOutcome, AttemptAuthority, AttemptError, AttemptFaultInjector, AttemptFaultPoint,
|
||||
AttemptState, CheckpointProjection, ClaimOutcome, ClaimRejection, RecoveryOutcome,
|
||||
RenewalOutcome,
|
||||
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptError,
|
||||
AttemptFaultInjector, AttemptFaultPoint, AttemptState, CheckpointProjection, ClaimOutcome,
|
||||
ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, RecoveryOutcome,
|
||||
RenewalMutationOutcome, RenewalOutcome,
|
||||
};
|
||||
|
||||
struct NoFaults;
|
||||
|
|
@ -179,6 +180,23 @@ impl AttemptService {
|
|||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
) -> Result<ActivationOutcome, AttemptError> {
|
||||
Ok(
|
||||
match self.activate_operation(authority, operation_id).await? {
|
||||
ActivationMutationOutcome::Activated { disposition, .. } => {
|
||||
ActivationOutcome::Activated(disposition)
|
||||
}
|
||||
ActivationMutationOutcome::PreconditionFailed => {
|
||||
ActivationOutcome::PreconditionFailed
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn activate_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
) -> Result<ActivationMutationOutcome, AttemptError> {
|
||||
validate_authority(authority)?;
|
||||
let request_fingerprint = fingerprint(&ActivationRequest {
|
||||
job_id: &authority.job_id,
|
||||
|
|
@ -203,16 +221,17 @@ impl AttemptService {
|
|||
request_fingerprint,
|
||||
move |tx| {
|
||||
Box::pin(async move {
|
||||
let accepted =
|
||||
let lease =
|
||||
activate_attempt(tx, &coordination, &authority_for_effect, faults.as_ref())
|
||||
.await?;
|
||||
Ok((
|
||||
if accepted {
|
||||
if lease.is_some() {
|
||||
MutationStatus::Accepted
|
||||
} else {
|
||||
MutationStatus::Rejected
|
||||
},
|
||||
json!({ "activated": accepted }),
|
||||
serde_json::to_value(ActivationStored { lease })
|
||||
.map_err(DurableError::effect)?,
|
||||
))
|
||||
})
|
||||
},
|
||||
|
|
@ -225,7 +244,14 @@ impl AttemptService {
|
|||
.exact_authority_is_executing_and_live(authority)
|
||||
.await?
|
||||
{
|
||||
return Ok(ActivationOutcome::Activated(MutationDisposition::First));
|
||||
let lease = self
|
||||
.live_lease(authority)
|
||||
.await?
|
||||
.ok_or_else(|| AttemptError::Invalid("live lease disappeared".into()))?;
|
||||
return Ok(ActivationMutationOutcome::Activated {
|
||||
disposition: MutationDisposition::First,
|
||||
lease,
|
||||
});
|
||||
}
|
||||
self.release_exact(authority).await;
|
||||
return Err(error.into());
|
||||
|
|
@ -237,16 +263,24 @@ impl AttemptService {
|
|||
}
|
||||
if outcome.status == MutationStatus::Rejected {
|
||||
self.release_exact(authority).await;
|
||||
return Ok(ActivationOutcome::PreconditionFailed);
|
||||
return Ok(ActivationMutationOutcome::PreconditionFailed);
|
||||
}
|
||||
if !self
|
||||
.exact_authority_is_executing_and_live(authority)
|
||||
.await?
|
||||
{
|
||||
self.release_exact(authority).await;
|
||||
return Ok(ActivationOutcome::PreconditionFailed);
|
||||
return Ok(ActivationMutationOutcome::PreconditionFailed);
|
||||
}
|
||||
Ok(ActivationOutcome::Activated(outcome.disposition))
|
||||
let stored: ActivationStored = serde_json::from_value(outcome.response)
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
let lease = stored
|
||||
.lease
|
||||
.ok_or_else(|| AttemptError::Invalid("accepted activation omitted its lease".into()))?;
|
||||
Ok(ActivationMutationOutcome::Activated {
|
||||
disposition: outcome.disposition,
|
||||
lease,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn renew(
|
||||
|
|
@ -269,6 +303,103 @@ impl AttemptService {
|
|||
Ok(RenewalOutcome::Renewed)
|
||||
}
|
||||
|
||||
pub async fn renew_operation(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
operation_id: &OpaqueId,
|
||||
) -> Result<RenewalMutationOutcome, AttemptError> {
|
||||
validate_authority(authority)?;
|
||||
let request_fingerprint = fingerprint(&ActivationRequest {
|
||||
job_id: &authority.job_id,
|
||||
attempt_id: &authority.attempt_id,
|
||||
fence: authority.fence,
|
||||
worker_incarnation: &authority.worker_incarnation,
|
||||
})
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
let eligibility = authority_binding(authority);
|
||||
let authority = authority.clone();
|
||||
let coordination = self.coordination.clone();
|
||||
let faults = self.faults.clone();
|
||||
let outcome = execute_mutation(
|
||||
&self.pool,
|
||||
MutationScope {
|
||||
subject: &self.operation_subject,
|
||||
operation: "attempt.mutation",
|
||||
key: operation_id.as_str(),
|
||||
},
|
||||
"renew_attempt",
|
||||
&eligibility,
|
||||
request_fingerprint,
|
||||
move |tx| {
|
||||
Box::pin(async move {
|
||||
let Some(row) = lock_authority(tx, &authority.attempt_id).await? else {
|
||||
return Ok((
|
||||
MutationStatus::Rejected,
|
||||
serde_json::to_value(RenewalMutationOutcome::AuthorityLost {
|
||||
disposition: None,
|
||||
})
|
||||
.map_err(DurableError::effect)?,
|
||||
));
|
||||
};
|
||||
if row.state != AttemptState::Executing
|
||||
|| !row.matches(
|
||||
&authority.job_id,
|
||||
authority.fence,
|
||||
&authority.worker_incarnation,
|
||||
)
|
||||
{
|
||||
return Ok((
|
||||
MutationStatus::Rejected,
|
||||
serde_json::to_value(RenewalMutationOutcome::AuthorityLost {
|
||||
disposition: None,
|
||||
})
|
||||
.map_err(DurableError::effect)?,
|
||||
));
|
||||
}
|
||||
faults.hit(AttemptFaultPoint::RenewalBeforeLease);
|
||||
let expected = lease_owner(&authority).map_err(DurableError::effect)?;
|
||||
let renewed = coordination
|
||||
.renew(authority.job_id.as_str(), &expected)
|
||||
.await
|
||||
.map_err(DurableError::effect)?;
|
||||
faults.hit(AttemptFaultPoint::RenewalAfterLease);
|
||||
let live = coordination
|
||||
.get_live_lease(authority.job_id.as_str())
|
||||
.await
|
||||
.map_err(DurableError::effect)?;
|
||||
let response = match live
|
||||
.filter(|lease| renewed.ok() && lease.record.owner() == expected)
|
||||
{
|
||||
Some(lease) => RenewalMutationOutcome::Renewed {
|
||||
lease: LeaseProjection {
|
||||
expires_at_ms: lease.record.expires_at,
|
||||
observed_at_ms: lease.now_ms,
|
||||
},
|
||||
disposition: None,
|
||||
},
|
||||
None => RenewalMutationOutcome::AuthorityLost { disposition: None },
|
||||
};
|
||||
let status = if matches!(response, RenewalMutationOutcome::Renewed { .. }) {
|
||||
MutationStatus::Accepted
|
||||
} else {
|
||||
MutationStatus::Rejected
|
||||
};
|
||||
Ok((
|
||||
status,
|
||||
serde_json::to_value(response).map_err(DurableError::effect)?,
|
||||
))
|
||||
})
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if outcome.disposition == MutationDisposition::Conflict {
|
||||
return Err(AttemptError::OperationConflict);
|
||||
}
|
||||
let response: RenewalMutationOutcome = serde_json::from_value(outcome.response)
|
||||
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||
Ok(response.with_disposition(outcome.disposition))
|
||||
}
|
||||
|
||||
pub async fn recover_inactive(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
|
|
@ -372,6 +503,45 @@ impl AttemptService {
|
|||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
pub async fn historical_authority(
|
||||
&self,
|
||||
attempt_id: &OpaqueId,
|
||||
) -> Result<Option<AttemptAuthority>, AttemptError> {
|
||||
load_historical_authority(&self.pool, attempt_id)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
pub async fn execution_envelope(
|
||||
&self,
|
||||
attempt_id: &OpaqueId,
|
||||
) -> Result<Option<ExecutionEnvelopeRecord>, AttemptError> {
|
||||
let value = sqlx::query_scalar::<_, Option<Json<ExecutionEnvelopeRecord>>>(
|
||||
"SELECT execution_envelope FROM attempts WHERE attempt_id = $1",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.fetch_optional(&self.pool)
|
||||
.await?
|
||||
.flatten();
|
||||
Ok(value.map(|value| value.0))
|
||||
}
|
||||
|
||||
pub async fn live_lease(
|
||||
&self,
|
||||
authority: &AttemptAuthority,
|
||||
) -> Result<Option<LeaseProjection>, AttemptError> {
|
||||
let expected = lease_owner(authority)?;
|
||||
Ok(self
|
||||
.coordination
|
||||
.get_live_lease(authority.job_id.as_str())
|
||||
.await?
|
||||
.filter(|lease| lease.record.owner() == expected)
|
||||
.map(|lease| LeaseProjection {
|
||||
expires_at_ms: lease.record.expires_at,
|
||||
observed_at_ms: lease.now_ms,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn release_local(&self, job_id: &OpaqueId) -> bool {
|
||||
self.local_permits.release_all(job_id.as_str())
|
||||
}
|
||||
|
|
@ -451,6 +621,11 @@ struct ActivationRequest<'a> {
|
|||
worker_incarnation: &'a OpaqueId,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize, Serialize)]
|
||||
struct ActivationStored {
|
||||
lease: Option<LeaseProjection>,
|
||||
}
|
||||
|
||||
struct AuthorityRow {
|
||||
job_id: OpaqueId,
|
||||
fence: u64,
|
||||
|
|
@ -534,11 +709,14 @@ async fn authorize_attempt(
|
|||
_ => 0,
|
||||
};
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let envelope =
|
||||
load_execution_envelope_record(tx, input.job_id, input.attempt_id, fence, input.worker)
|
||||
.await?;
|
||||
sqlx::query(
|
||||
"INSERT INTO attempts (
|
||||
attempt_id, job_id, fence, worker_incarnation, state,
|
||||
initial_progress, authorized_at, executing_at, reaped_at
|
||||
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL)",
|
||||
initial_progress, authorized_at, executing_at, reaped_at, execution_envelope
|
||||
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL, $7)",
|
||||
)
|
||||
.bind(input.attempt_id.as_str())
|
||||
.bind(input.job_id.as_str())
|
||||
|
|
@ -546,6 +724,7 @@ async fn authorize_attempt(
|
|||
.bind(input.worker.as_str())
|
||||
.bind(i64::try_from(initial_progress).map_err(DurableError::effect)?)
|
||||
.bind(now)
|
||||
.bind(Json(&envelope))
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
sqlx::query(
|
||||
|
|
@ -572,21 +751,79 @@ async fn authorize_attempt(
|
|||
})
|
||||
}
|
||||
|
||||
async fn load_execution_envelope_record(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
job_id: &OpaqueId,
|
||||
attempt_id: &OpaqueId,
|
||||
fence: u64,
|
||||
worker: &OpaqueId,
|
||||
) -> Result<ExecutionEnvelopeRecord, DurableError> {
|
||||
type Row = (
|
||||
Json<serde_json::Value>,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
Json<Vec<String>>,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Option<Json<crate::DurableDescriptor>>,
|
||||
Option<Json<CheckpointProjection>>,
|
||||
);
|
||||
let row = sqlx::query_as::<_, Row>(
|
||||
"SELECT j.source_projection, j.sensitive_source_ref, j.target_store,
|
||||
j.target_bucket, j.target_prefix, j.execution_grant_id,
|
||||
g.credential_id, g.credential_version_id, j.resolved_source, j.checkpoint
|
||||
FROM jobs j
|
||||
JOIN execution_grants g ON g.grant_id = j.execution_grant_id
|
||||
WHERE j.job_id = $1",
|
||||
)
|
||||
.bind(job_id.as_str())
|
||||
.fetch_one(tx.as_mut())
|
||||
.await?;
|
||||
let grant_id = OpaqueId::parse(row.5).map_err(DurableError::effect)?;
|
||||
Ok(ExecutionEnvelopeRecord {
|
||||
id: grant_id.clone(),
|
||||
job_id: job_id.clone(),
|
||||
attempt_id: attempt_id.clone(),
|
||||
fence,
|
||||
worker_incarnation: worker.clone(),
|
||||
source: row.0.0,
|
||||
sensitive_source_ref: OpaqueId::parse(row.1).map_err(DurableError::effect)?,
|
||||
credential_id: row
|
||||
.6
|
||||
.map(OpaqueId::parse)
|
||||
.transpose()
|
||||
.map_err(DurableError::effect)?,
|
||||
credential_version_id: row
|
||||
.7
|
||||
.map(OpaqueId::parse)
|
||||
.transpose()
|
||||
.map_err(DurableError::effect)?,
|
||||
resolved_source: row.8.map(|value| value.0),
|
||||
target_store: row.2,
|
||||
target_bucket: row.3,
|
||||
target_prefix: row.4.0,
|
||||
execution_grant_id: grant_id,
|
||||
checkpoint: row.9.map(|value| value.0),
|
||||
})
|
||||
}
|
||||
|
||||
async fn activate_attempt(
|
||||
tx: &mut remora_durable::PgTx<'_>,
|
||||
coordination: &CoordinationStore,
|
||||
authority: &AttemptAuthority,
|
||||
faults: &dyn AttemptFaultInjector,
|
||||
) -> Result<bool, DurableError> {
|
||||
) -> Result<Option<LeaseProjection>, DurableError> {
|
||||
let Some(row) = lock_authority(tx, &authority.attempt_id).await? else {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
};
|
||||
if !row.matches(
|
||||
&authority.job_id,
|
||||
authority.fence,
|
||||
&authority.worker_incarnation,
|
||||
) {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
}
|
||||
if row.state == AttemptState::Executing {
|
||||
let live = coordination
|
||||
|
|
@ -595,16 +832,28 @@ async fn activate_attempt(
|
|||
.map_err(DurableError::effect)?;
|
||||
let expected = lease_owner(authority).map_err(DurableError::effect)?;
|
||||
if !live.is_some_and(|lease| lease.record.owner() == expected) {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(coordination
|
||||
if !coordination
|
||||
.renew(authority.job_id.as_str(), &expected)
|
||||
.await
|
||||
.map_err(DurableError::effect)?
|
||||
.ok());
|
||||
.ok()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(coordination
|
||||
.get_live_lease(authority.job_id.as_str())
|
||||
.await
|
||||
.map_err(DurableError::effect)?
|
||||
.filter(|lease| lease.record.owner() == expected)
|
||||
.map(|lease| LeaseProjection {
|
||||
expires_at_ms: lease.record.expires_at,
|
||||
observed_at_ms: lease.now_ms,
|
||||
}));
|
||||
}
|
||||
if row.state != AttemptState::Authorized {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
}
|
||||
faults.hit(AttemptFaultPoint::ActivationBeforeLease);
|
||||
let owner = lease_owner(authority).map_err(DurableError::effect)?;
|
||||
|
|
@ -620,7 +869,7 @@ async fn activate_attempt(
|
|||
.await
|
||||
.map_err(DurableError::effect)?;
|
||||
if !coordinated.ok() {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
}
|
||||
faults.hit(AttemptFaultPoint::ActivationAfterLease);
|
||||
let live = coordination
|
||||
|
|
@ -628,9 +877,9 @@ async fn activate_attempt(
|
|||
.await
|
||||
.map_err(DurableError::effect)?;
|
||||
let expected = lease_owner(authority).map_err(DurableError::effect)?;
|
||||
if !live.is_some_and(|lease| lease.record.owner() == expected) {
|
||||
return Ok(false);
|
||||
}
|
||||
let Some(live) = live.filter(|lease| lease.record.owner() == expected) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let now = OffsetDateTime::now_utc();
|
||||
sqlx::query(
|
||||
"UPDATE attempts SET state = 'executing', executing_at = $2
|
||||
|
|
@ -640,7 +889,10 @@ async fn activate_attempt(
|
|||
.bind(now)
|
||||
.execute(tx.as_mut())
|
||||
.await?;
|
||||
Ok(true)
|
||||
Ok(Some(LeaseProjection {
|
||||
expires_at_ms: live.record.expires_at,
|
||||
observed_at_ms: live.now_ms,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn lock_authority(
|
||||
|
|
@ -692,6 +944,30 @@ async fn load_authority(
|
|||
.transpose()
|
||||
}
|
||||
|
||||
async fn load_historical_authority(
|
||||
pool: &PgPool,
|
||||
attempt_id: &OpaqueId,
|
||||
) -> Result<Option<AttemptAuthority>, DurableError> {
|
||||
let row = sqlx::query_as::<_, (String, String, i64, String, String)>(
|
||||
"SELECT job_id, attempt_id, fence, worker_incarnation, state
|
||||
FROM attempts WHERE attempt_id = $1",
|
||||
)
|
||||
.bind(attempt_id.as_str())
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
row.map(|(job, attempt, fence, worker, state)| {
|
||||
Ok(AttemptAuthority {
|
||||
job_id: OpaqueId::parse(job).map_err(DurableError::effect)?,
|
||||
attempt_id: OpaqueId::parse(attempt).map_err(DurableError::effect)?,
|
||||
fence: u64::try_from(fence).map_err(DurableError::effect)?,
|
||||
worker_incarnation: OpaqueId::parse(worker).map_err(DurableError::effect)?,
|
||||
state: AttemptState::parse(&state)
|
||||
.ok_or_else(|| DurableError::Config(format!("unknown attempt state '{state}'")))?,
|
||||
})
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn authority_row(row: (String, i64, String, String)) -> Result<AuthorityRow, DurableError> {
|
||||
Ok(AuthorityRow {
|
||||
job_id: OpaqueId::parse(row.0).map_err(DurableError::effect)?,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use remora_domain::OpaqueId;
|
||||
use remora_domain::{JobPhase, OpaqueId};
|
||||
use remora_durable::MutationDisposition;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
|
|
@ -95,12 +95,83 @@ pub enum ActivationOutcome {
|
|||
PreconditionFailed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ActivationMutationOutcome {
|
||||
Activated {
|
||||
disposition: MutationDisposition,
|
||||
lease: LeaseProjection,
|
||||
},
|
||||
PreconditionFailed,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum RenewalOutcome {
|
||||
Renewed,
|
||||
AuthorityLost,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct LeaseProjection {
|
||||
pub expires_at_ms: u64,
|
||||
pub observed_at_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(tag = "status", rename_all = "snake_case")]
|
||||
pub enum RenewalMutationOutcome {
|
||||
Renewed {
|
||||
lease: LeaseProjection,
|
||||
#[serde(skip)]
|
||||
disposition: Option<MutationDisposition>,
|
||||
},
|
||||
AuthorityLost {
|
||||
#[serde(skip)]
|
||||
disposition: Option<MutationDisposition>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum OutcomePublication {
|
||||
Accepted {
|
||||
disposition: MutationDisposition,
|
||||
phase: JobPhase,
|
||||
},
|
||||
Rejected {
|
||||
reason: crate::MutationRejection,
|
||||
disposition: MutationDisposition,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
pub struct ExecutionEnvelopeRecord {
|
||||
pub id: OpaqueId,
|
||||
pub job_id: OpaqueId,
|
||||
pub attempt_id: OpaqueId,
|
||||
pub fence: u64,
|
||||
pub worker_incarnation: OpaqueId,
|
||||
pub source: serde_json::Value,
|
||||
pub sensitive_source_ref: OpaqueId,
|
||||
pub credential_id: Option<OpaqueId>,
|
||||
pub credential_version_id: Option<OpaqueId>,
|
||||
pub resolved_source: Option<crate::DurableDescriptor>,
|
||||
pub target_store: String,
|
||||
pub target_bucket: String,
|
||||
pub target_prefix: Vec<String>,
|
||||
pub execution_grant_id: OpaqueId,
|
||||
pub checkpoint: Option<crate::CheckpointProjection>,
|
||||
}
|
||||
|
||||
impl RenewalMutationOutcome {
|
||||
pub(crate) fn with_disposition(mut self, value: MutationDisposition) -> Self {
|
||||
match &mut self {
|
||||
Self::Renewed { disposition, .. } | Self::AuthorityLost { disposition } => {
|
||||
*disposition = Some(value);
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum RecoveryOutcome {
|
||||
StillOwned,
|
||||
|
|
|
|||
|
|
@ -315,6 +315,8 @@ pub enum SafeDetailValueDto {
|
|||
#[serde(deny_unknown_fields)]
|
||||
pub struct ResultDescriptorDto {
|
||||
pub kind: ResultKindDto,
|
||||
pub manifest_id: OpaqueId,
|
||||
pub schema_version: PositiveIntegerDto,
|
||||
pub manifest: ObjectIdentityDto,
|
||||
pub member_count: PositiveIntegerDto,
|
||||
pub total_bytes: SafeIntegerDto,
|
||||
|
|
@ -339,6 +341,7 @@ pub struct ObjectIdentityDto {
|
|||
)]
|
||||
pub version_id: Option<String>,
|
||||
pub checksum: String,
|
||||
pub size: SafeIntegerDto,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
|
|
|
|||
|
|
@ -157,6 +157,9 @@ pub struct AttemptCheckpointRequestDto {
|
|||
pub fence: PositiveIntegerDto,
|
||||
pub worker_incarnation: OpaqueId,
|
||||
pub checkpoint_ref: CheckpointDescriptorRefDto,
|
||||
pub resolved_source_id: OpaqueId,
|
||||
pub validated_bytes: SafeIntegerDto,
|
||||
pub validated_members: SafeIntegerDto,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
use std::{collections::BTreeSet, fs};
|
||||
|
||||
use remora_contracts::{
|
||||
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
|
||||
AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||
AttemptResolvedSourceRequestDto, ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto,
|
||||
LeaseProjectionDto,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use serde_json::Value;
|
||||
|
||||
#[test]
|
||||
|
|
@ -48,6 +55,273 @@ fn api_contract_documents_parse_and_local_references_resolve() {
|
|||
validate_openapi_corpus(&component_schema, &corpus, "JobFailure", "job_failures");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
|
||||
let openapi: Value =
|
||||
serde_yaml_ng::from_str(&fs::read_to_string("../../docs/api/openapi.yaml").unwrap())
|
||||
.unwrap();
|
||||
let component_schema = openapi_component_schema(&openapi);
|
||||
let mutation = serde_json::json!({
|
||||
"operation_id": "operation_0001",
|
||||
"fence": 1,
|
||||
"worker_incarnation": "worker_0001"
|
||||
});
|
||||
let descriptor = serde_json::json!({
|
||||
"id": "descriptor_0001",
|
||||
"kind": "resolved_source",
|
||||
"schema_version": 1,
|
||||
"byte_length": 64,
|
||||
"item_count": 1,
|
||||
"checksum": format!("sha256:{}", "22".repeat(32))
|
||||
});
|
||||
let execution = serde_json::json!({
|
||||
"id": "grant_0001",
|
||||
"job_id": "job_00000001",
|
||||
"attempt_id": "attempt_0001",
|
||||
"fence": 1,
|
||||
"worker_incarnation": "worker_0001",
|
||||
"source": {
|
||||
"kind": "http",
|
||||
"display_url": "https://example.invalid/artifact",
|
||||
"result_path": "artifact.bin"
|
||||
},
|
||||
"sensitive_source_ref": "source_00001",
|
||||
"credential_version": null,
|
||||
"resolved_source_ref": null,
|
||||
"target": {
|
||||
"store": "primary",
|
||||
"bucket": "remora-results",
|
||||
"prefix": "users/alice"
|
||||
},
|
||||
"staging_target": {
|
||||
"store": "primary",
|
||||
"bucket": "remora-results",
|
||||
"prefix": "_remora/a/attempt_0001/"
|
||||
},
|
||||
"execution_grant_ref": "grant_0001",
|
||||
"checkpoint_ref": null
|
||||
});
|
||||
let authority = serde_json::json!({
|
||||
"attempt_id": "attempt_0001",
|
||||
"fence": 1,
|
||||
"worker_incarnation": "worker_0001"
|
||||
});
|
||||
let lease = serde_json::json!({
|
||||
"expires_at": "2026-07-17T00:01:00Z",
|
||||
"renew_after": "2026-07-17T00:00:30Z"
|
||||
});
|
||||
|
||||
let claim_request = serde_json::json!({
|
||||
"operation_id": "operation_0001",
|
||||
"worker_incarnation": "worker_0001"
|
||||
});
|
||||
assert_dto_and_schema::<ClaimRequestDto>(&component_schema, "ClaimRequest", claim_request);
|
||||
let claim_response = serde_json::json!({
|
||||
"authority": authority,
|
||||
"execution": execution
|
||||
});
|
||||
assert_dto_and_schema::<ClaimResponseDto>(
|
||||
&component_schema,
|
||||
"ClaimResponse",
|
||||
claim_response.clone(),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptRenewRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptRenewRequest",
|
||||
mutation.clone(),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptActivateRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptActivateRequest",
|
||||
mutation.clone(),
|
||||
);
|
||||
assert_dto_and_schema::<ActivationResponseDto>(
|
||||
&component_schema,
|
||||
"ActivationResponse",
|
||||
serde_json::json!({
|
||||
"authority": claim_response["authority"],
|
||||
"execution": claim_response["execution"],
|
||||
"lease": lease.clone()
|
||||
}),
|
||||
);
|
||||
assert_dto_and_schema::<LeaseProjectionDto>(
|
||||
&component_schema,
|
||||
"LeaseProjection",
|
||||
lease.clone(),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptResolvedSourceRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptResolvedSourceRequest",
|
||||
merged(
|
||||
&mutation,
|
||||
serde_json::json!({ "resolved_source_ref": descriptor.clone() }),
|
||||
),
|
||||
);
|
||||
let mut checkpoint_descriptor = descriptor.clone();
|
||||
checkpoint_descriptor["kind"] = Value::String("checkpoint".into());
|
||||
assert_dto_and_schema::<AttemptCheckpointRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptCheckpointRequest",
|
||||
merged(
|
||||
&mutation,
|
||||
serde_json::json!({
|
||||
"checkpoint_ref": checkpoint_descriptor,
|
||||
"resolved_source_id": "descriptor_0001",
|
||||
"validated_bytes": 1024,
|
||||
"validated_members": 1
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptProgressRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptProgressRequest",
|
||||
merged(
|
||||
&mutation,
|
||||
serde_json::json!({
|
||||
"progress": {
|
||||
"completed_bytes": 1024,
|
||||
"total_bytes": 4096,
|
||||
"completed_members": 1,
|
||||
"total_members": 1
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert_dto_and_schema::<AttemptOutcomeRequestDto>(
|
||||
&component_schema,
|
||||
"AttemptOutcomeRequest",
|
||||
merged(
|
||||
&mutation,
|
||||
serde_json::json!({
|
||||
"outcome": {
|
||||
"kind": "failure",
|
||||
"failure": {
|
||||
"class": "transient_source",
|
||||
"message": "temporarily unavailable"
|
||||
}
|
||||
}
|
||||
}),
|
||||
),
|
||||
);
|
||||
assert_dto_and_schema::<JobPhaseProjectionDto>(
|
||||
&component_schema,
|
||||
"JobPhaseProjection",
|
||||
serde_json::json!({ "phase": "RetryWait" }),
|
||||
);
|
||||
|
||||
let routes = [
|
||||
(
|
||||
"/internal/v1/jobs/{job_id}/claim",
|
||||
"ClaimRequest",
|
||||
"200",
|
||||
Some("ClaimResponse"),
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/renew",
|
||||
"AttemptRenewRequest",
|
||||
"200",
|
||||
Some("LeaseProjection"),
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/activate",
|
||||
"AttemptActivateRequest",
|
||||
"200",
|
||||
Some("ActivationResponse"),
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/resolved-source",
|
||||
"AttemptResolvedSourceRequest",
|
||||
"204",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/checkpoint",
|
||||
"AttemptCheckpointRequest",
|
||||
"204",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/progress",
|
||||
"AttemptProgressRequest",
|
||||
"204",
|
||||
None,
|
||||
),
|
||||
(
|
||||
"/internal/v1/attempts/{attempt_id}/outcome",
|
||||
"AttemptOutcomeRequest",
|
||||
"200",
|
||||
Some("JobPhaseProjection"),
|
||||
),
|
||||
];
|
||||
for (path, request, status, response) in routes {
|
||||
assert_worker_route(&openapi, path, request, status, response);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_dto_and_schema<T>(component_schema: &Value, definition: &str, value: Value)
|
||||
where
|
||||
T: DeserializeOwned + Serialize,
|
||||
{
|
||||
let dto: T = serde_json::from_value(value.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(dto).unwrap(), value);
|
||||
validate_openapi_value(component_schema, definition, &value);
|
||||
}
|
||||
|
||||
fn validate_openapi_value(component_schema: &Value, definition: &str, value: &Value) {
|
||||
let mut schema = component_schema.clone();
|
||||
schema.as_object_mut().unwrap().insert(
|
||||
"$ref".to_owned(),
|
||||
Value::String(format!("#/$defs/{definition}")),
|
||||
);
|
||||
jsonschema::draft202012::options()
|
||||
.should_validate_formats(true)
|
||||
.build(&schema)
|
||||
.unwrap()
|
||||
.validate(value)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn merged(base: &Value, extension: Value) -> Value {
|
||||
let mut value = base.clone();
|
||||
value
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.extend(extension.as_object().unwrap().clone());
|
||||
value
|
||||
}
|
||||
|
||||
fn assert_worker_route(
|
||||
openapi: &Value,
|
||||
path: &str,
|
||||
request_schema: &str,
|
||||
success_status: &str,
|
||||
response_schema: Option<&str>,
|
||||
) {
|
||||
let operation = &openapi["paths"][path]["post"];
|
||||
assert_eq!(
|
||||
operation.pointer("/security/0/serviceAuth"),
|
||||
Some(&serde_json::json!([]))
|
||||
);
|
||||
assert_eq!(
|
||||
operation.pointer("/requestBody/content/application~1json/schema/$ref"),
|
||||
Some(&Value::String(format!(
|
||||
"#/components/schemas/{request_schema}"
|
||||
)))
|
||||
);
|
||||
assert!(operation["responses"].get("409").is_some());
|
||||
assert!(operation["responses"].get("default").is_some());
|
||||
let success = &operation["responses"][success_status];
|
||||
match response_schema {
|
||||
Some(response_schema) => assert_eq!(
|
||||
success.pointer("/content/application~1json/schema/$ref"),
|
||||
Some(&Value::String(format!(
|
||||
"#/components/schemas/{response_schema}"
|
||||
)))
|
||||
),
|
||||
None => assert!(success.get("content").is_none()),
|
||||
}
|
||||
}
|
||||
|
||||
fn openapi_component_schema(openapi: &Value) -> Value {
|
||||
let mut definitions = openapi
|
||||
.pointer("/components/schemas")
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ const PATH_SET: &AsciiSet = &CONTROLS
|
|||
.add(b'}');
|
||||
|
||||
/// One version or delete marker listed under a key/prefix.
|
||||
#[cfg(feature = "test-utils")]
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ObjectVersion {
|
||||
pub key: String,
|
||||
|
|
@ -81,6 +82,7 @@ impl MultipartClient {
|
|||
/// Enable bucket versioning when the peer supports it.
|
||||
///
|
||||
/// Peers that return NotImplemented are reported as `false` without error.
|
||||
#[cfg(feature = "test-utils")]
|
||||
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
|
||||
let body = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
|
|
@ -341,6 +343,7 @@ impl MultipartClient {
|
|||
}
|
||||
|
||||
/// List object versions and delete markers under an optional key prefix.
|
||||
#[cfg(feature = "test-utils")]
|
||||
pub async fn list_object_versions(
|
||||
&self,
|
||||
prefix: &str,
|
||||
|
|
@ -410,6 +413,7 @@ impl MultipartClient {
|
|||
}
|
||||
|
||||
/// Delete one exact object version or delete marker.
|
||||
#[cfg(feature = "test-utils")]
|
||||
pub async fn delete_object_version(
|
||||
&self,
|
||||
key: &str,
|
||||
|
|
@ -556,6 +560,7 @@ struct CompletePart {
|
|||
etag: String,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-utils")]
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ListVersionsResult {
|
||||
#[serde(rename = "Version", default)]
|
||||
|
|
@ -570,6 +575,7 @@ struct ListVersionsResult {
|
|||
next_version_id_marker: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-utils")]
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ListedVersion {
|
||||
#[serde(rename = "Key", default)]
|
||||
|
|
|
|||
|
|
@ -551,7 +551,7 @@ paths:
|
|||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Job'
|
||||
$ref: '#/components/schemas/JobPhaseProjection'
|
||||
'409':
|
||||
$ref: '#/components/responses/PreconditionFailed'
|
||||
default:
|
||||
|
|
@ -1109,6 +1109,13 @@ components:
|
|||
JobPhase:
|
||||
type: string
|
||||
enum: [Queued, Running, RetryWait, Cancelling, Succeeded, Failed, Cancelled]
|
||||
JobPhaseProjection:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [phase]
|
||||
properties:
|
||||
phase:
|
||||
$ref: '#/components/schemas/JobPhase'
|
||||
TerminalJobPhase:
|
||||
type: string
|
||||
enum: [Succeeded, Failed, Cancelled]
|
||||
|
|
@ -1247,10 +1254,15 @@ components:
|
|||
ResultDescriptor:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [kind, manifest, member_count, total_bytes]
|
||||
required: [kind, manifest_id, schema_version, manifest, member_count, total_bytes]
|
||||
properties:
|
||||
kind:
|
||||
const: manifest
|
||||
manifest_id:
|
||||
$ref: '#/components/schemas/OpaqueId'
|
||||
schema_version:
|
||||
type: integer
|
||||
minimum: 1
|
||||
manifest:
|
||||
$ref: '#/components/schemas/ObjectIdentity'
|
||||
member_count:
|
||||
|
|
@ -1262,7 +1274,7 @@ components:
|
|||
ObjectIdentity:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [store, bucket, key, checksum]
|
||||
required: [store, bucket, key, checksum, size]
|
||||
properties:
|
||||
store:
|
||||
type: string
|
||||
|
|
@ -1274,6 +1286,9 @@ components:
|
|||
type: string
|
||||
checksum:
|
||||
$ref: '#/components/schemas/Sha256Digest'
|
||||
size:
|
||||
type: integer
|
||||
minimum: 0
|
||||
Sha256Digest:
|
||||
type: string
|
||||
pattern: '^sha256:[a-f0-9]{64}$'
|
||||
|
|
@ -1657,7 +1672,7 @@ components:
|
|||
allOf:
|
||||
- $ref: '#/components/schemas/AttemptMutation'
|
||||
- type: object
|
||||
required: [checkpoint_ref]
|
||||
required: [checkpoint_ref, resolved_source_id, validated_bytes, validated_members]
|
||||
properties:
|
||||
checkpoint_ref:
|
||||
allOf:
|
||||
|
|
@ -1666,6 +1681,14 @@ components:
|
|||
properties:
|
||||
kind:
|
||||
const: checkpoint
|
||||
resolved_source_id:
|
||||
$ref: '#/components/schemas/OpaqueId'
|
||||
validated_bytes:
|
||||
type: integer
|
||||
minimum: 0
|
||||
validated_members:
|
||||
type: integer
|
||||
minimum: 0
|
||||
AttemptProgressRequest:
|
||||
unevaluatedProperties: false
|
||||
allOf:
|
||||
|
|
|
|||
50
tests/kind/run-worker-api-job.sh
Executable file
50
tests/kind/run-worker-api-job.sh
Executable file
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
source "$ROOT/tests/kind/lib.sh"
|
||||
|
||||
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||
SUCCEEDED=false
|
||||
OWNS_FIXTURE=false
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
trap - EXIT INT TERM
|
||||
if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||||
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||
printf 'retained context: %s\n' "$CONTEXT"
|
||||
exit "$status"
|
||||
fi
|
||||
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||||
destroy_fixture || status=1
|
||||
fi
|
||||
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||
status=1
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
acquire_fixture_lock
|
||||
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
||||
export REMORA_REAL_NERDCTL
|
||||
"$REMORA_REAL_NERDCTL" --namespace default build \
|
||||
--file "$ROOT/Dockerfile.test" \
|
||||
--target kind-worker-api-runner \
|
||||
--tag remora-kind-worker-api:local \
|
||||
"$ROOT"
|
||||
OWNS_FIXTURE=true
|
||||
setup_fixture durable coordination
|
||||
load_local_image remora-kind-worker-api:local
|
||||
run_job_matrix \
|
||||
worker-api \
|
||||
"$TESTBED/worker-api-job.yaml.in" \
|
||||
remora-kind-worker-api:local \
|
||||
durable coordination
|
||||
SUCCEEDED=true
|
||||
printf 'worker API package test Jobs passed\n'
|
||||
34
tests/kind/worker-api-job.yaml.in
Normal file
34
tests/kind/worker-api-job.yaml.in
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: @@JOB_NAME@@
|
||||
namespace: remora-testbed
|
||||
spec:
|
||||
activeDeadlineSeconds: 600
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 300
|
||||
template:
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: tests
|
||||
image: @@TEST_IMAGE@@
|
||||
imagePullPolicy: Never
|
||||
envFrom:
|
||||
- configMapRef: {name: @@PROFILE_NAME@@}
|
||||
env:
|
||||
- name: REMORA_WORKER_API_REQUIRE
|
||||
value: "1"
|
||||
volumeMounts:
|
||||
- {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 256Mi}
|
||||
limits: {cpu: "2", memory: 1Gi}
|
||||
volumes:
|
||||
- name: fixture-profile
|
||||
configMap:
|
||||
name: @@PROFILE_NAME@@-file
|
||||
items:
|
||||
- {key: profile.env, path: profile.env}
|
||||
Loading…
Reference in a new issue