Add authenticated internal worker API
This commit is contained in:
parent
9d96c4d563
commit
ebf818d23c
20 changed files with 2745 additions and 43 deletions
488
Cargo.lock
generated
488
Cargo.lock
generated
|
|
@ -2,6 +2,216 @@
|
||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 4
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-codec"
|
||||||
|
version = "0.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.0",
|
||||||
|
"bytes",
|
||||||
|
"futures-core",
|
||||||
|
"futures-sink",
|
||||||
|
"memchr",
|
||||||
|
"pin-project-lite",
|
||||||
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-http"
|
||||||
|
version = "3.13.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "48e2faa3e7418ed780cca54829d32782a4008a077230f67457caa063415e99c2"
|
||||||
|
dependencies = [
|
||||||
|
"actix-codec",
|
||||||
|
"actix-rt",
|
||||||
|
"actix-service",
|
||||||
|
"actix-tls",
|
||||||
|
"actix-utils",
|
||||||
|
"base64",
|
||||||
|
"bitflags 2.13.0",
|
||||||
|
"brotli",
|
||||||
|
"bytes",
|
||||||
|
"bytestring",
|
||||||
|
"derive_more",
|
||||||
|
"encoding_rs",
|
||||||
|
"flate2",
|
||||||
|
"foldhash",
|
||||||
|
"futures-core",
|
||||||
|
"h2 0.3.27",
|
||||||
|
"http 0.2.12",
|
||||||
|
"httparse",
|
||||||
|
"httpdate",
|
||||||
|
"itoa",
|
||||||
|
"language-tags",
|
||||||
|
"local-channel",
|
||||||
|
"mime",
|
||||||
|
"percent-encoding",
|
||||||
|
"pin-project-lite",
|
||||||
|
"rand 0.10.2",
|
||||||
|
"sha1",
|
||||||
|
"smallvec",
|
||||||
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
|
"tracing",
|
||||||
|
"zstd",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-macros"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
|
||||||
|
dependencies = [
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-router"
|
||||||
|
version = "0.5.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "14f8c75c51892f18d9c46150c5ac7beb81c95f78c8b83a634d49f4ca32551fe7"
|
||||||
|
dependencies = [
|
||||||
|
"bytestring",
|
||||||
|
"cfg-if 1.0.4",
|
||||||
|
"http 0.2.12",
|
||||||
|
"regex",
|
||||||
|
"regex-lite",
|
||||||
|
"serde",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-rt"
|
||||||
|
version = "2.11.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "92589714878ca59a7626ea19734f0e07a6a875197eec751bb5d3f99e64998c63"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-server"
|
||||||
|
version = "2.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a65064ea4a457eaf07f2fba30b4c695bf43b721790e9530d26cb6f9019ff7502"
|
||||||
|
dependencies = [
|
||||||
|
"actix-rt",
|
||||||
|
"actix-service",
|
||||||
|
"actix-utils",
|
||||||
|
"futures-core",
|
||||||
|
"futures-util",
|
||||||
|
"mio",
|
||||||
|
"socket2 0.5.10",
|
||||||
|
"tokio",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-service"
|
||||||
|
version = "2.0.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9e46f36bf0e5af44bdc4bdb36fbbd421aa98c79a9bce724e1edeb3894e10dc7f"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"pin-project-lite",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-tls"
|
||||||
|
version = "3.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6176099de3f58fbddac916a7f8c6db297e021d706e7a6b99947785fee14abe9f"
|
||||||
|
dependencies = [
|
||||||
|
"actix-rt",
|
||||||
|
"actix-service",
|
||||||
|
"actix-utils",
|
||||||
|
"futures-core",
|
||||||
|
"impl-more 0.1.9",
|
||||||
|
"pin-project-lite",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"tokio",
|
||||||
|
"tokio-rustls",
|
||||||
|
"tokio-util",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-utils"
|
||||||
|
version = "3.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8"
|
||||||
|
dependencies = [
|
||||||
|
"local-waker",
|
||||||
|
"pin-project-lite",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-web"
|
||||||
|
version = "4.14.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "df09e2d9239703dd64056359c920c7f3fba6535ec61a0059e0f44e095ffe02b4"
|
||||||
|
dependencies = [
|
||||||
|
"actix-codec",
|
||||||
|
"actix-http",
|
||||||
|
"actix-macros",
|
||||||
|
"actix-router",
|
||||||
|
"actix-rt",
|
||||||
|
"actix-server",
|
||||||
|
"actix-service",
|
||||||
|
"actix-tls",
|
||||||
|
"actix-utils",
|
||||||
|
"actix-web-codegen",
|
||||||
|
"bytes",
|
||||||
|
"bytestring",
|
||||||
|
"cfg-if 1.0.4",
|
||||||
|
"cookie",
|
||||||
|
"derive_more",
|
||||||
|
"encoding_rs",
|
||||||
|
"foldhash",
|
||||||
|
"futures-core",
|
||||||
|
"futures-util",
|
||||||
|
"impl-more 0.3.1",
|
||||||
|
"itoa",
|
||||||
|
"language-tags",
|
||||||
|
"log",
|
||||||
|
"mime",
|
||||||
|
"once_cell",
|
||||||
|
"pin-project-lite",
|
||||||
|
"regex",
|
||||||
|
"regex-lite",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"serde_urlencoded",
|
||||||
|
"smallvec",
|
||||||
|
"socket2 0.6.5",
|
||||||
|
"time",
|
||||||
|
"tracing",
|
||||||
|
"url",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "actix-web-codegen"
|
||||||
|
version = "4.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8"
|
||||||
|
dependencies = [
|
||||||
|
"actix-router",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "adler2"
|
||||||
|
version = "2.0.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ahash"
|
name = "ahash"
|
||||||
version = "0.8.12"
|
version = "0.8.12"
|
||||||
|
|
@ -25,6 +235,21 @@ dependencies = [
|
||||||
"memchr",
|
"memchr",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "alloc-no-stdlib"
|
||||||
|
version = "2.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "alloc-stdlib"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195"
|
||||||
|
dependencies = [
|
||||||
|
"alloc-no-stdlib",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "allocator-api2"
|
name = "allocator-api2"
|
||||||
version = "0.2.21"
|
version = "0.2.21"
|
||||||
|
|
@ -421,6 +646,27 @@ version = "0.2.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
|
checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "brotli"
|
||||||
|
version = "8.0.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3"
|
||||||
|
dependencies = [
|
||||||
|
"alloc-no-stdlib",
|
||||||
|
"alloc-stdlib",
|
||||||
|
"brotli-decompressor",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "brotli-decompressor"
|
||||||
|
version = "5.0.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583"
|
||||||
|
dependencies = [
|
||||||
|
"alloc-no-stdlib",
|
||||||
|
"alloc-stdlib",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bstr"
|
name = "bstr"
|
||||||
version = "1.13.0"
|
version = "1.13.0"
|
||||||
|
|
@ -472,6 +718,15 @@ dependencies = [
|
||||||
"either",
|
"either",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bytestring"
|
||||||
|
version = "1.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "86566c496f2f47d9b8147a4c8b02ffdb69c919fe0c2b2e7195d22cbba0e635c9"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.2.67"
|
version = "1.2.67"
|
||||||
|
|
@ -626,6 +881,26 @@ version = "0.4.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
|
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "convert_case"
|
||||||
|
version = "0.10.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9"
|
||||||
|
dependencies = [
|
||||||
|
"unicode-segmentation",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cookie"
|
||||||
|
version = "0.16.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb"
|
||||||
|
dependencies = [
|
||||||
|
"percent-encoding",
|
||||||
|
"time",
|
||||||
|
"version_check",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation"
|
name = "core-foundation"
|
||||||
version = "0.9.4"
|
version = "0.9.4"
|
||||||
|
|
@ -886,6 +1161,29 @@ dependencies = [
|
||||||
"serde_core",
|
"serde_core",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "derive_more"
|
||||||
|
version = "2.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
|
||||||
|
dependencies = [
|
||||||
|
"derive_more-impl",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "derive_more-impl"
|
||||||
|
version = "2.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb"
|
||||||
|
dependencies = [
|
||||||
|
"convert_case",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"rustc_version",
|
||||||
|
"syn",
|
||||||
|
"unicode-xid",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "digest"
|
name = "digest"
|
||||||
version = "0.10.7"
|
version = "0.10.7"
|
||||||
|
|
@ -1049,6 +1347,16 @@ version = "0.1.9"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "flate2"
|
||||||
|
version = "1.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||||
|
dependencies = [
|
||||||
|
"crc32fast",
|
||||||
|
"miniz_oxide",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fluent-uri"
|
name = "fluent-uri"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
|
|
@ -1300,6 +1608,25 @@ dependencies = [
|
||||||
"regex-syntax",
|
"regex-syntax",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "h2"
|
||||||
|
version = "0.3.27"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"fnv",
|
||||||
|
"futures-core",
|
||||||
|
"futures-sink",
|
||||||
|
"futures-util",
|
||||||
|
"http 0.2.12",
|
||||||
|
"indexmap",
|
||||||
|
"slab",
|
||||||
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h2"
|
name = "h2"
|
||||||
version = "0.4.15"
|
version = "0.4.15"
|
||||||
|
|
@ -1536,7 +1863,7 @@ dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"h2",
|
"h2 0.4.15",
|
||||||
"http 1.4.2",
|
"http 1.4.2",
|
||||||
"http-body 1.1.0",
|
"http-body 1.1.0",
|
||||||
"httparse",
|
"httparse",
|
||||||
|
|
@ -1580,7 +1907,7 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"socket2",
|
"socket2 0.6.5",
|
||||||
"system-configuration",
|
"system-configuration",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
|
|
@ -1721,6 +2048,18 @@ dependencies = [
|
||||||
"icu_properties",
|
"icu_properties",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "impl-more"
|
||||||
|
version = "0.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "impl-more"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "35a84fd5aa25fae5c0f4a33d9cac2ca017fc622cbd089be2229993514990f870"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "indexmap"
|
name = "indexmap"
|
||||||
version = "2.14.0"
|
version = "2.14.0"
|
||||||
|
|
@ -1934,6 +2273,12 @@ version = "0.4.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "language-tags"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lazy_static"
|
name = "lazy_static"
|
||||||
version = "1.5.0"
|
version = "1.5.0"
|
||||||
|
|
@ -1998,6 +2343,23 @@ version = "0.8.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "local-channel"
|
||||||
|
version = "0.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"futures-sink",
|
||||||
|
"local-waker",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "local-waker"
|
||||||
|
version = "0.1.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lock_api"
|
name = "lock_api"
|
||||||
version = "0.4.14"
|
version = "0.4.14"
|
||||||
|
|
@ -2090,6 +2452,16 @@ version = "0.2.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "miniz_oxide"
|
||||||
|
version = "0.8.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||||
|
dependencies = [
|
||||||
|
"adler2",
|
||||||
|
"simd-adler32",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mio"
|
name = "mio"
|
||||||
version = "1.2.2"
|
version = "1.2.2"
|
||||||
|
|
@ -2097,6 +2469,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
|
checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
|
"log",
|
||||||
"wasi 0.11.1+wasi-snapshot-preview1",
|
"wasi 0.11.1+wasi-snapshot-preview1",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
@ -2604,7 +2977,7 @@ dependencies = [
|
||||||
"quinn-udp",
|
"quinn-udp",
|
||||||
"rustc-hash",
|
"rustc-hash",
|
||||||
"rustls",
|
"rustls",
|
||||||
"socket2",
|
"socket2 0.6.5",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|
@ -2643,7 +3016,7 @@ dependencies = [
|
||||||
"cfg_aliases",
|
"cfg_aliases",
|
||||||
"libc",
|
"libc",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"socket2",
|
"socket2 0.6.5",
|
||||||
"tracing",
|
"tracing",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
@ -2731,6 +3104,7 @@ version = "0.14.8"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
|
checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"pem",
|
||||||
"ring",
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"time",
|
"time",
|
||||||
|
|
@ -2766,7 +3140,7 @@ dependencies = [
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"ryu",
|
"ryu",
|
||||||
"sha1_smol",
|
"sha1_smol",
|
||||||
"socket2",
|
"socket2 0.6.5",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"url",
|
"url",
|
||||||
|
|
@ -2852,6 +3226,12 @@ dependencies = [
|
||||||
"regex-syntax",
|
"regex-syntax",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "regex-lite"
|
||||||
|
version = "0.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "regex-syntax"
|
name = "regex-syntax"
|
||||||
version = "0.8.11"
|
version = "0.8.11"
|
||||||
|
|
@ -2880,6 +3260,31 @@ dependencies = [
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "remora-api"
|
name = "remora-api"
|
||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
|
dependencies = [
|
||||||
|
"actix-tls",
|
||||||
|
"actix-web",
|
||||||
|
"getrandom 0.3.4",
|
||||||
|
"rcgen",
|
||||||
|
"remora-admission",
|
||||||
|
"remora-attempt",
|
||||||
|
"remora-contracts",
|
||||||
|
"remora-coordination",
|
||||||
|
"remora-credentials",
|
||||||
|
"remora-domain",
|
||||||
|
"remora-durable",
|
||||||
|
"remora-identity",
|
||||||
|
"remora-result",
|
||||||
|
"remora-target",
|
||||||
|
"reqwest 0.13.4",
|
||||||
|
"rustls",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"sha2 0.11.0",
|
||||||
|
"sqlx",
|
||||||
|
"thiserror",
|
||||||
|
"time",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "remora-attempt"
|
name = "remora-attempt"
|
||||||
|
|
@ -3222,7 +3627,7 @@ dependencies = [
|
||||||
"encoding_rs",
|
"encoding_rs",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"h2",
|
"h2 0.4.15",
|
||||||
"http 1.4.2",
|
"http 1.4.2",
|
||||||
"http-body 1.1.0",
|
"http-body 1.1.0",
|
||||||
"http-body-util",
|
"http-body-util",
|
||||||
|
|
@ -3358,6 +3763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
|
checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
|
"log",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"ring",
|
"ring",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
|
|
@ -3669,6 +4075,16 @@ version = "2.0.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "signal-hook-registry"
|
||||||
|
version = "1.4.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
|
||||||
|
dependencies = [
|
||||||
|
"errno",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "signature"
|
name = "signature"
|
||||||
version = "2.2.0"
|
version = "2.2.0"
|
||||||
|
|
@ -3679,6 +4095,12 @@ dependencies = [
|
||||||
"rand_core 0.6.4",
|
"rand_core 0.6.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "simd-adler32"
|
||||||
|
version = "0.3.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "simd_cesu8"
|
name = "simd_cesu8"
|
||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
|
|
@ -3722,6 +4144,16 @@ dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "socket2"
|
||||||
|
version = "0.5.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"windows-sys 0.52.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "socket2"
|
name = "socket2"
|
||||||
version = "0.6.5"
|
version = "0.6.5"
|
||||||
|
|
@ -4166,8 +4598,10 @@ dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
"libc",
|
"libc",
|
||||||
"mio",
|
"mio",
|
||||||
|
"parking_lot",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"socket2",
|
"signal-hook-registry",
|
||||||
|
"socket2 0.6.5",
|
||||||
"tokio-macros",
|
"tokio-macros",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
@ -4448,6 +4882,18 @@ version = "0.1.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
|
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-segmentation"
|
||||||
|
version = "1.13.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-xid"
|
||||||
|
version = "0.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unsafe-libyaml"
|
name = "unsafe-libyaml"
|
||||||
version = "0.2.11"
|
version = "0.2.11"
|
||||||
|
|
@ -5228,3 +5674,31 @@ name = "zmij"
|
||||||
version = "1.0.23"
|
version = "1.0.23"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd"
|
||||||
|
version = "0.13.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||||
|
dependencies = [
|
||||||
|
"zstd-safe",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd-safe"
|
||||||
|
version = "7.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
|
||||||
|
dependencies = [
|
||||||
|
"zstd-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zstd-sys"
|
||||||
|
version = "2.0.16+zstd.1.5.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"pkg-config",
|
||||||
|
]
|
||||||
|
|
|
||||||
|
|
@ -257,6 +257,34 @@ COPY crates crates
|
||||||
COPY --from=kind-attempt-verified /tmp/remora-attempt-tests.tar.zst /tests/remora-attempt-tests.tar.zst
|
COPY --from=kind-attempt-verified /tmp/remora-attempt-tests.tar.zst /tests/remora-attempt-tests.tar.zst
|
||||||
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_TARGET_ENDPOINT\" && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_RECOVERY_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-attempt-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'binary(recovery)'"]
|
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && test -n \"$REMORA_TARGET_ENDPOINT\" && case $REMORA_TARGET_PROVIDER in garage|versitygw) ;; *) exit 1 ;; esac && export REMORA_RECOVERY_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-attempt-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast -E 'binary(recovery)'"]
|
||||||
|
|
||||||
|
FROM rust-tools AS kind-worker-api-build
|
||||||
|
WORKDIR /workspace
|
||||||
|
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||||
|
COPY .config .config
|
||||||
|
COPY contracts contracts
|
||||||
|
COPY crates crates
|
||||||
|
COPY docs/api docs/api
|
||||||
|
RUN cargo nextest archive \
|
||||||
|
--package remora-api \
|
||||||
|
--build-jobs 4 \
|
||||||
|
--locked \
|
||||||
|
--archive-file /tmp/remora-api-tests.tar.zst
|
||||||
|
|
||||||
|
FROM kind-worker-api-build AS kind-worker-api-verified
|
||||||
|
RUN cargo clippy \
|
||||||
|
--package remora-api \
|
||||||
|
--all-targets \
|
||||||
|
--locked \
|
||||||
|
-- -D warnings \
|
||||||
|
&& cargo test --package remora-api --doc --locked
|
||||||
|
|
||||||
|
FROM rust-tools AS kind-worker-api-runner
|
||||||
|
WORKDIR /workspace
|
||||||
|
COPY Cargo.toml ./
|
||||||
|
COPY crates crates
|
||||||
|
COPY --from=kind-worker-api-verified /tmp/remora-api-tests.tar.zst /tests/remora-api-tests.tar.zst
|
||||||
|
CMD ["bash", "-ec", "test -s /run/remora-fixture/profile.env && test -n \"$DATABASE_URL\" && test -n \"$REMORA_COORDINATION_URL\" && export REMORA_WORKER_API_REQUIRE=1 && cargo-nextest nextest run --archive-file /tests/remora-api-tests.tar.zst --workspace-remap /workspace --test-threads=1 --no-fail-fast"]
|
||||||
|
|
||||||
FROM rust-tools AS kind-http-source-build
|
FROM rust-tools AS kind-http-source-build
|
||||||
WORKDIR /workspace
|
WORKDIR /workspace
|
||||||
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||||
|
|
|
||||||
5
Makefile
5
Makefile
|
|
@ -1,4 +1,4 @@
|
||||||
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-stream check-target check-result check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
.PHONY: check-dependencies check-fixture-lib check-kind-sample check-durable check-coordination check-identity check-credentials check-admission check-attempt check-attempt-mutations check-recovery check-worker-api check-stream check-target check-result check-outbound check-http-source check-object-source check-hf-source check-fast check-full check-fast-host check-full-host rust-check web-install web-check web-test-fast web-test-full
|
||||||
|
|
||||||
CONTAINER_ENGINE ?= nerdctl
|
CONTAINER_ENGINE ?= nerdctl
|
||||||
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
|
CONTAINER_RUN_FLAGS ?= --memory=16g --memory-swap=24g --cpus=4
|
||||||
|
|
@ -80,6 +80,9 @@ check-attempt-mutations:
|
||||||
check-recovery:
|
check-recovery:
|
||||||
./tests/kind/run-recovery-job.sh
|
./tests/kind/run-recovery-job.sh
|
||||||
|
|
||||||
|
check-worker-api:
|
||||||
|
./tests/kind/run-worker-api-job.sh
|
||||||
|
|
||||||
check-stream:
|
check-stream:
|
||||||
cargo fmt --all --check
|
cargo fmt --all --check
|
||||||
cargo clippy --package remora-stream --all-targets --all-features -- -D warnings
|
cargo clippy --package remora-stream --all-targets --all-features -- -D warnings
|
||||||
|
|
|
||||||
|
|
@ -202,12 +202,15 @@
|
||||||
"kind": "success",
|
"kind": "success",
|
||||||
"result": {
|
"result": {
|
||||||
"kind": "manifest",
|
"kind": "manifest",
|
||||||
|
"manifest_id": "manifest_0001",
|
||||||
|
"schema_version": 1,
|
||||||
"manifest": {
|
"manifest": {
|
||||||
"store": "target-main",
|
"store": "target-main",
|
||||||
"bucket": "target-bucket",
|
"bucket": "target-bucket",
|
||||||
"key": "_remora/results/manifest.json",
|
"key": "_remora/results/manifest.json",
|
||||||
"version_id": "version-1",
|
"version_id": "version-1",
|
||||||
"checksum": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
"checksum": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||||
|
"size": 512
|
||||||
},
|
},
|
||||||
"member_count": 1,
|
"member_count": 1,
|
||||||
"total_bytes": 4096
|
"total_bytes": 4096
|
||||||
|
|
|
||||||
|
|
@ -4,3 +4,30 @@ edition.workspace = true
|
||||||
publish = false
|
publish = false
|
||||||
rust-version.workspace = true
|
rust-version.workspace = true
|
||||||
version.workspace = true
|
version.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
actix-web = { version = "=4.14.0", features = ["rustls-0_23"] }
|
||||||
|
actix-tls = "=3.5.0"
|
||||||
|
getrandom = "0.3.4"
|
||||||
|
remora-attempt = { path = "../remora-attempt" }
|
||||||
|
remora-contracts = { path = "../remora-contracts" }
|
||||||
|
remora-domain.workspace = true
|
||||||
|
remora-result = { path = "../remora-result" }
|
||||||
|
remora-target = { path = "../remora-target" }
|
||||||
|
serde.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
|
sha2.workspace = true
|
||||||
|
sqlx.workspace = true
|
||||||
|
thiserror.workspace = true
|
||||||
|
time.workspace = true
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
rcgen = "=0.14.8"
|
||||||
|
remora-admission = { path = "../remora-admission" }
|
||||||
|
remora-coordination = { path = "../remora-coordination" }
|
||||||
|
remora-credentials = { path = "../remora-credentials" }
|
||||||
|
remora-durable.workspace = true
|
||||||
|
remora-identity = { path = "../remora-identity" }
|
||||||
|
reqwest = { version = "=0.13.4", default-features = false, features = ["json", "rustls"] }
|
||||||
|
rustls = "=0.23.42"
|
||||||
|
tokio.workspace = true
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,9 @@
|
||||||
|
mod worker;
|
||||||
|
|
||||||
|
pub use worker::{
|
||||||
|
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
|
||||||
|
};
|
||||||
|
|
||||||
pub const COMPONENT_NAME: &str = "remora-api";
|
pub const COMPONENT_NAME: &str = "remora-api";
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
|
||||||
613
crates/remora-api/src/worker.rs
Normal file
613
crates/remora-api/src/worker.rs
Normal file
|
|
@ -0,0 +1,613 @@
|
||||||
|
use std::{
|
||||||
|
any::Any,
|
||||||
|
future::{Ready, ready},
|
||||||
|
};
|
||||||
|
|
||||||
|
use actix_web::{
|
||||||
|
FromRequest, HttpRequest, HttpResponse, ResponseError, dev::Payload, http::StatusCode, web,
|
||||||
|
};
|
||||||
|
use remora_attempt::{
|
||||||
|
ActivationMutationOutcome, AttemptAuthority, AttemptError, AttemptMutationOutcome,
|
||||||
|
AttemptService, CheckpointProjection, CheckpointRecord, ClaimOutcome, DurableDescriptor,
|
||||||
|
LeaseProjection, MutationRejection, OutcomePublication, ProgressRecord, RenewalMutationOutcome,
|
||||||
|
};
|
||||||
|
use remora_contracts::{
|
||||||
|
ActivationResponseDto, ApiErrorCodeDto, ApiErrorDto, AttemptActivateRequestDto,
|
||||||
|
AttemptAuthorityDto, AttemptCheckpointRequestDto, AttemptOutcomeDto, AttemptOutcomeRequestDto,
|
||||||
|
AttemptProgressRequestDto, AttemptRenewRequestDto, AttemptResolvedSourceRequestDto,
|
||||||
|
CheckpointDescriptorKindDto, CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto,
|
||||||
|
CredentialGrantVersionDto, DateTimeDto, DurableDescriptorKindDto, DurableDescriptorRefDto,
|
||||||
|
ExecutionEnvelopeDto, JobPhaseProjectionDto, LeaseProjectionDto, NullableDto,
|
||||||
|
PositiveIntegerDto, ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto,
|
||||||
|
SafeIntegerDto, SourceProjectionDto, StagingTargetDto, TargetDto,
|
||||||
|
};
|
||||||
|
use remora_domain::OpaqueId;
|
||||||
|
use remora_result::PublicationInput;
|
||||||
|
use remora_target::{ContentSha256, ObjectIdentity, RESERVED_NAMESPACE_SEGMENT, StoredObject};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
|
pub struct AuthenticatedWorker {
|
||||||
|
pub identity: OpaqueId,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn authenticate_rustls_worker(
|
||||||
|
connection: &dyn Any,
|
||||||
|
extensions: &mut actix_web::dev::Extensions,
|
||||||
|
) {
|
||||||
|
type Stream = actix_tls::accept::rustls_0_23::TlsStream<actix_web::rt::net::TcpStream>;
|
||||||
|
let Some(stream) = connection.downcast_ref::<Stream>() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let Some(certificate) = stream
|
||||||
|
.get_ref()
|
||||||
|
.1
|
||||||
|
.peer_certificates()
|
||||||
|
.and_then(|chain| chain.first())
|
||||||
|
else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let digest = Sha256::digest(certificate.as_ref());
|
||||||
|
let mut value = String::from("w");
|
||||||
|
for byte in digest {
|
||||||
|
use std::fmt::Write as _;
|
||||||
|
write!(value, "{byte:02x}").expect("writing to String cannot fail");
|
||||||
|
}
|
||||||
|
extensions.insert(AuthenticatedWorker {
|
||||||
|
identity: OpaqueId::parse(value).expect("certificate digest is a valid identity"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FromRequest for AuthenticatedWorker {
|
||||||
|
type Error = WorkerApiError;
|
||||||
|
type Future = Ready<Result<Self, Self::Error>>;
|
||||||
|
|
||||||
|
fn from_request(request: &HttpRequest, _payload: &mut Payload) -> Self::Future {
|
||||||
|
ready(
|
||||||
|
request
|
||||||
|
.conn_data::<Self>()
|
||||||
|
.cloned()
|
||||||
|
.ok_or_else(WorkerApiError::unauthenticated),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct WorkerApiState {
|
||||||
|
attempts: AttemptService,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerApiState {
|
||||||
|
#[must_use]
|
||||||
|
pub fn new(attempts: AttemptService) -> Self {
|
||||||
|
Self { attempts }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn configure_worker_api(config: &mut web::ServiceConfig) {
|
||||||
|
config.service(
|
||||||
|
web::scope("/internal/v1")
|
||||||
|
.app_data(
|
||||||
|
web::JsonConfig::default()
|
||||||
|
.limit(1024 * 1024)
|
||||||
|
.error_handler(|_, _| WorkerApiError::invalid_request().into()),
|
||||||
|
)
|
||||||
|
.route("/jobs/{job_id}/claim", web::post().to(claim))
|
||||||
|
.route("/attempts/{attempt_id}/renew", web::post().to(renew))
|
||||||
|
.route("/attempts/{attempt_id}/activate", web::post().to(activate))
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/resolved-source",
|
||||||
|
web::post().to(pin_resolved_source),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/checkpoint",
|
||||||
|
web::post().to(persist_checkpoint),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/progress",
|
||||||
|
web::post().to(report_progress),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/attempts/{attempt_id}/outcome",
|
||||||
|
web::post().to(report_outcome),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn claim(
|
||||||
|
_: AuthenticatedWorker,
|
||||||
|
state: web::Data<WorkerApiState>,
|
||||||
|
path: web::Path<String>,
|
||||||
|
request: web::Json<ClaimRequestDto>,
|
||||||
|
) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
let job_id = parse_id(path.into_inner())?;
|
||||||
|
let outcome = state
|
||||||
|
.attempts
|
||||||
|
.claim(&job_id, &request.worker_incarnation, &request.operation_id)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?;
|
||||||
|
let ClaimOutcome::Claimed { authority, .. } = outcome else {
|
||||||
|
return Err(WorkerApiError::precondition_failed());
|
||||||
|
};
|
||||||
|
let execution = load_execution_envelope(&state.attempts, &authority).await?;
|
||||||
|
Ok(HttpResponse::Ok().json(ClaimResponseDto {
|
||||||
|
authority: authority_dto(&authority)?,
|
||||||
|
execution,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn renew(
|
||||||
|
_: AuthenticatedWorker,
|
||||||
|
state: web::Data<WorkerApiState>,
|
||||||
|
path: web::Path<String>,
|
||||||
|
request: web::Json<AttemptRenewRequestDto>,
|
||||||
|
) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
let authority = request_authority(
|
||||||
|
&state,
|
||||||
|
path.into_inner(),
|
||||||
|
request.fence.get(),
|
||||||
|
&request.worker_incarnation,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
match state
|
||||||
|
.attempts
|
||||||
|
.renew_operation(&authority, &request.operation_id)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?
|
||||||
|
{
|
||||||
|
RenewalMutationOutcome::Renewed { lease, .. } => {
|
||||||
|
Ok(HttpResponse::Ok().json(lease_dto(lease)?))
|
||||||
|
}
|
||||||
|
RenewalMutationOutcome::AuthorityLost { .. } => Err(WorkerApiError::precondition_failed()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
_: AuthenticatedWorker,
|
||||||
|
state: web::Data<WorkerApiState>,
|
||||||
|
path: web::Path<String>,
|
||||||
|
request: web::Json<AttemptActivateRequestDto>,
|
||||||
|
) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
let authority = request_authority(
|
||||||
|
&state,
|
||||||
|
path.into_inner(),
|
||||||
|
request.fence.get(),
|
||||||
|
&request.worker_incarnation,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let lease = match state
|
||||||
|
.attempts
|
||||||
|
.activate_operation(&authority, &request.operation_id)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?
|
||||||
|
{
|
||||||
|
ActivationMutationOutcome::Activated { lease, .. } => lease,
|
||||||
|
ActivationMutationOutcome::PreconditionFailed => {
|
||||||
|
return Err(WorkerApiError::precondition_failed());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let execution = load_execution_envelope(&state.attempts, &authority).await?;
|
||||||
|
Ok(HttpResponse::Ok().json(ActivationResponseDto {
|
||||||
|
authority: authority_dto(&authority)?,
|
||||||
|
execution,
|
||||||
|
lease: lease_dto(lease)?,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn pin_resolved_source(
|
||||||
|
_: AuthenticatedWorker,
|
||||||
|
state: web::Data<WorkerApiState>,
|
||||||
|
path: web::Path<String>,
|
||||||
|
request: web::Json<AttemptResolvedSourceRequestDto>,
|
||||||
|
) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
let authority = request_authority(
|
||||||
|
&state,
|
||||||
|
path.into_inner(),
|
||||||
|
request.fence.get(),
|
||||||
|
&request.worker_incarnation,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
accepted(
|
||||||
|
state
|
||||||
|
.attempts
|
||||||
|
.pin_resolved_source(
|
||||||
|
&authority,
|
||||||
|
&request.operation_id,
|
||||||
|
descriptor_from_resolved(&request.resolved_source_ref)?,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn persist_checkpoint(
|
||||||
|
_: AuthenticatedWorker,
|
||||||
|
state: web::Data<WorkerApiState>,
|
||||||
|
path: web::Path<String>,
|
||||||
|
request: web::Json<AttemptCheckpointRequestDto>,
|
||||||
|
) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
let authority = request_authority(
|
||||||
|
&state,
|
||||||
|
path.into_inner(),
|
||||||
|
request.fence.get(),
|
||||||
|
&request.worker_incarnation,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
accepted(
|
||||||
|
state
|
||||||
|
.attempts
|
||||||
|
.persist_checkpoint(
|
||||||
|
&authority,
|
||||||
|
&request.operation_id,
|
||||||
|
CheckpointRecord {
|
||||||
|
descriptor: descriptor_from_checkpoint(&request.checkpoint_ref)?,
|
||||||
|
resolved_source_id: request.resolved_source_id.clone(),
|
||||||
|
validated_bytes: request.validated_bytes.get(),
|
||||||
|
validated_members: request.validated_members.get(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn report_progress(
|
||||||
|
_: AuthenticatedWorker,
|
||||||
|
state: web::Data<WorkerApiState>,
|
||||||
|
path: web::Path<String>,
|
||||||
|
request: web::Json<AttemptProgressRequestDto>,
|
||||||
|
) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
let authority = request_authority(
|
||||||
|
&state,
|
||||||
|
path.into_inner(),
|
||||||
|
request.fence.get(),
|
||||||
|
&request.worker_incarnation,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
accepted(
|
||||||
|
state
|
||||||
|
.attempts
|
||||||
|
.report_progress(
|
||||||
|
&authority,
|
||||||
|
&request.operation_id,
|
||||||
|
ProgressRecord {
|
||||||
|
completed_bytes: request.progress.completed_bytes.get(),
|
||||||
|
total_bytes: request.progress.total_bytes.map(SafeIntegerDto::get),
|
||||||
|
completed_members: request.progress.completed_members.get(),
|
||||||
|
total_members: request.progress.total_members.map(SafeIntegerDto::get),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn report_outcome(
|
||||||
|
_: AuthenticatedWorker,
|
||||||
|
state: web::Data<WorkerApiState>,
|
||||||
|
path: web::Path<String>,
|
||||||
|
request: web::Json<AttemptOutcomeRequestDto>,
|
||||||
|
) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
let authority = request_authority(
|
||||||
|
&state,
|
||||||
|
path.into_inner(),
|
||||||
|
request.fence.get(),
|
||||||
|
&request.worker_incarnation,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let outcome = match &request.outcome {
|
||||||
|
AttemptOutcomeDto::Success(success) => {
|
||||||
|
let result = &success.result;
|
||||||
|
state
|
||||||
|
.attempts
|
||||||
|
.publish_success_operation(
|
||||||
|
&authority,
|
||||||
|
&request.operation_id,
|
||||||
|
PublicationInput {
|
||||||
|
attempt_id: authority.attempt_id.clone(),
|
||||||
|
manifest_id: result.manifest_id.clone(),
|
||||||
|
schema_version: u32::try_from(result.schema_version.get())
|
||||||
|
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||||
|
manifest: StoredObject {
|
||||||
|
identity: ObjectIdentity {
|
||||||
|
target_store_id: result.manifest.store.clone(),
|
||||||
|
bucket: result.manifest.bucket.clone(),
|
||||||
|
key: result.manifest.key.clone(),
|
||||||
|
version_id: result.manifest.version_id.clone(),
|
||||||
|
},
|
||||||
|
content_sha256: ContentSha256::parse(&result.manifest.checksum)
|
||||||
|
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||||
|
size: result.manifest.size.get(),
|
||||||
|
},
|
||||||
|
member_count: result.member_count.get(),
|
||||||
|
total_bytes: result.total_bytes.get(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
AttemptOutcomeDto::Failure(failure) => {
|
||||||
|
state
|
||||||
|
.attempts
|
||||||
|
.publish_failure_operation(&authority, &request.operation_id, failure.failure.class)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.map_err(WorkerApiError::from_attempt)?;
|
||||||
|
let phase = match outcome {
|
||||||
|
OutcomePublication::Accepted { phase, .. } => phase,
|
||||||
|
OutcomePublication::Rejected { reason, .. } => {
|
||||||
|
return Err(WorkerApiError::from_rejection(reason));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(HttpResponse::Ok().json(JobPhaseProjectionDto { phase }))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn accepted(outcome: AttemptMutationOutcome) -> Result<HttpResponse, WorkerApiError> {
|
||||||
|
match outcome {
|
||||||
|
AttemptMutationOutcome::Accepted(_) => Ok(HttpResponse::NoContent().finish()),
|
||||||
|
AttemptMutationOutcome::Rejected { reason, .. } => {
|
||||||
|
Err(WorkerApiError::from_rejection(reason))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn request_authority(
|
||||||
|
state: &WorkerApiState,
|
||||||
|
attempt_id: String,
|
||||||
|
fence: u64,
|
||||||
|
worker: &OpaqueId,
|
||||||
|
) -> Result<AttemptAuthority, WorkerApiError> {
|
||||||
|
let attempt_id = parse_id(attempt_id)?;
|
||||||
|
state
|
||||||
|
.attempts
|
||||||
|
.historical_authority(&attempt_id)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?
|
||||||
|
.filter(|authority| authority.fence == fence && authority.worker_incarnation == *worker)
|
||||||
|
.ok_or_else(WorkerApiError::precondition_failed)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_execution_envelope(
|
||||||
|
attempts: &AttemptService,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
) -> Result<ExecutionEnvelopeDto, WorkerApiError> {
|
||||||
|
let row = attempts
|
||||||
|
.execution_envelope(&authority.attempt_id)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerApiError::from_attempt)?
|
||||||
|
.ok_or_else(WorkerApiError::precondition_failed)?;
|
||||||
|
if row.job_id != authority.job_id
|
||||||
|
|| row.attempt_id != authority.attempt_id
|
||||||
|
|| row.fence != authority.fence
|
||||||
|
|| row.worker_incarnation != authority.worker_incarnation
|
||||||
|
{
|
||||||
|
return Err(WorkerApiError::precondition_failed());
|
||||||
|
}
|
||||||
|
let source: SourceProjectionDto =
|
||||||
|
serde_json::from_value(row.source).map_err(|_| WorkerApiError::unavailable())?;
|
||||||
|
let target_prefix = row.target_prefix.join("/");
|
||||||
|
let resolved_source_ref = row.resolved_source.map(|descriptor| {
|
||||||
|
NullableDto(Some(descriptor_ref(
|
||||||
|
descriptor,
|
||||||
|
DurableDescriptorKindDto::ResolvedSource,
|
||||||
|
)))
|
||||||
|
});
|
||||||
|
let checkpoint_ref = row.checkpoint.and_then(|checkpoint| match checkpoint {
|
||||||
|
CheckpointProjection::Reusable { checkpoint } => Some(NullableDto(Some(descriptor_ref(
|
||||||
|
checkpoint.descriptor,
|
||||||
|
DurableDescriptorKindDto::Checkpoint,
|
||||||
|
)))),
|
||||||
|
CheckpointProjection::Complete { .. } => None,
|
||||||
|
});
|
||||||
|
Ok(ExecutionEnvelopeDto {
|
||||||
|
id: row.id,
|
||||||
|
job_id: authority.job_id.clone(),
|
||||||
|
attempt_id: authority.attempt_id.clone(),
|
||||||
|
fence: positive(authority.fence)?,
|
||||||
|
worker_incarnation: authority.worker_incarnation.clone(),
|
||||||
|
source,
|
||||||
|
sensitive_source_ref: row.sensitive_source_ref,
|
||||||
|
credential_version: NullableDto(match (row.credential_id, row.credential_version_id) {
|
||||||
|
(Some(credential_id), Some(version_id)) => Some(CredentialGrantVersionDto {
|
||||||
|
credential_id,
|
||||||
|
version_id,
|
||||||
|
}),
|
||||||
|
(None, None) => None,
|
||||||
|
_ => return Err(WorkerApiError::unavailable()),
|
||||||
|
}),
|
||||||
|
resolved_source_ref: resolved_source_ref.unwrap_or(NullableDto(None)),
|
||||||
|
target: TargetDto {
|
||||||
|
store: row.target_store.clone(),
|
||||||
|
bucket: row.target_bucket.clone(),
|
||||||
|
prefix: target_prefix,
|
||||||
|
},
|
||||||
|
staging_target: StagingTargetDto {
|
||||||
|
store: row.target_store,
|
||||||
|
bucket: row.target_bucket,
|
||||||
|
prefix: format!(
|
||||||
|
"{RESERVED_NAMESPACE_SEGMENT}/a/{}/",
|
||||||
|
authority.attempt_id.as_str()
|
||||||
|
),
|
||||||
|
},
|
||||||
|
execution_grant_ref: row.execution_grant_id,
|
||||||
|
checkpoint_ref: checkpoint_ref.unwrap_or(NullableDto(None)),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn descriptor_ref(
|
||||||
|
descriptor: DurableDescriptor,
|
||||||
|
kind: DurableDescriptorKindDto,
|
||||||
|
) -> DurableDescriptorRefDto {
|
||||||
|
DurableDescriptorRefDto {
|
||||||
|
id: descriptor.id,
|
||||||
|
kind,
|
||||||
|
schema_version: positive(descriptor.schema_version.into()).expect("validated descriptor"),
|
||||||
|
byte_length: safe(descriptor.byte_length).expect("validated descriptor"),
|
||||||
|
item_count: descriptor
|
||||||
|
.item_count
|
||||||
|
.map(|value| safe(value).expect("validated descriptor")),
|
||||||
|
checksum: descriptor.checksum,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn descriptor_from_resolved(
|
||||||
|
value: &ResolvedSourceDescriptorRefDto,
|
||||||
|
) -> Result<DurableDescriptor, WorkerApiError> {
|
||||||
|
let ResolvedSourceDescriptorKindDto::ResolvedSource = value.kind;
|
||||||
|
Ok(DurableDescriptor {
|
||||||
|
id: value.id.clone(),
|
||||||
|
schema_version: u32::try_from(value.schema_version.get())
|
||||||
|
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||||
|
byte_length: value.byte_length.get(),
|
||||||
|
item_count: value.item_count.map(SafeIntegerDto::get),
|
||||||
|
checksum: value.checksum.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn descriptor_from_checkpoint(
|
||||||
|
value: &CheckpointDescriptorRefDto,
|
||||||
|
) -> Result<DurableDescriptor, WorkerApiError> {
|
||||||
|
let CheckpointDescriptorKindDto::Checkpoint = value.kind;
|
||||||
|
Ok(DurableDescriptor {
|
||||||
|
id: value.id.clone(),
|
||||||
|
schema_version: u32::try_from(value.schema_version.get())
|
||||||
|
.map_err(|_| WorkerApiError::invalid_request())?,
|
||||||
|
byte_length: value.byte_length.get(),
|
||||||
|
item_count: value.item_count.map(SafeIntegerDto::get),
|
||||||
|
checksum: value.checksum.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn authority_dto(authority: &AttemptAuthority) -> Result<AttemptAuthorityDto, WorkerApiError> {
|
||||||
|
Ok(AttemptAuthorityDto {
|
||||||
|
attempt_id: authority.attempt_id.clone(),
|
||||||
|
fence: positive(authority.fence)?,
|
||||||
|
worker_incarnation: authority.worker_incarnation.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lease_dto(lease: LeaseProjection) -> Result<LeaseProjectionDto, WorkerApiError> {
|
||||||
|
let observed = timestamp(lease.observed_at_ms)?;
|
||||||
|
let expires = timestamp(lease.expires_at_ms)?;
|
||||||
|
let renew_at = observed + (expires - observed) / 2;
|
||||||
|
Ok(LeaseProjectionDto {
|
||||||
|
expires_at: datetime(expires)?,
|
||||||
|
renew_after: datetime(renew_at)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn timestamp(milliseconds: u64) -> Result<OffsetDateTime, WorkerApiError> {
|
||||||
|
let nanoseconds = i128::from(milliseconds) * 1_000_000;
|
||||||
|
OffsetDateTime::from_unix_timestamp_nanos(nanoseconds)
|
||||||
|
.map_err(|_| WorkerApiError::unavailable())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn datetime(value: OffsetDateTime) -> Result<DateTimeDto, WorkerApiError> {
|
||||||
|
let value = value
|
||||||
|
.format(&Rfc3339)
|
||||||
|
.map_err(|_| WorkerApiError::unavailable())?;
|
||||||
|
DateTimeDto::parse(value).map_err(|_| WorkerApiError::unavailable())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn positive(value: u64) -> Result<PositiveIntegerDto, WorkerApiError> {
|
||||||
|
PositiveIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn safe(value: u64) -> Result<SafeIntegerDto, WorkerApiError> {
|
||||||
|
SafeIntegerDto::new(value).map_err(|_| WorkerApiError::unavailable())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_id(value: impl Into<String>) -> Result<OpaqueId, WorkerApiError> {
|
||||||
|
OpaqueId::parse(value).map_err(|_| WorkerApiError::invalid_request())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
#[error("worker API request failed")]
|
||||||
|
pub struct WorkerApiError {
|
||||||
|
status: StatusCode,
|
||||||
|
code: ApiErrorCodeDto,
|
||||||
|
message: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerApiError {
|
||||||
|
fn invalid_request() -> Self {
|
||||||
|
Self::new(
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
ApiErrorCodeDto::InvalidRequest,
|
||||||
|
"invalid request",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unauthenticated() -> Self {
|
||||||
|
Self::new(
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
ApiErrorCodeDto::Unauthenticated,
|
||||||
|
"authentication required",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn precondition_failed() -> Self {
|
||||||
|
Self::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
ApiErrorCodeDto::PreconditionFailed,
|
||||||
|
"precondition failed",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unavailable() -> Self {
|
||||||
|
Self::new(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
ApiErrorCodeDto::Unavailable,
|
||||||
|
"service unavailable",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn new(status: StatusCode, code: ApiErrorCodeDto, message: &'static str) -> Self {
|
||||||
|
Self {
|
||||||
|
status,
|
||||||
|
code,
|
||||||
|
message,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_attempt(error: AttemptError) -> Self {
|
||||||
|
match error {
|
||||||
|
AttemptError::Invalid(_) => Self::invalid_request(),
|
||||||
|
AttemptError::OperationConflict => Self::precondition_failed(),
|
||||||
|
AttemptError::Coordination(_) | AttemptError::Durable(_) => Self::unavailable(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_rejection(_reason: MutationRejection) -> Self {
|
||||||
|
Self::precondition_failed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ResponseError for WorkerApiError {
|
||||||
|
fn status_code(&self) -> StatusCode {
|
||||||
|
self.status
|
||||||
|
}
|
||||||
|
|
||||||
|
fn error_response(&self) -> HttpResponse {
|
||||||
|
HttpResponse::build(self.status)
|
||||||
|
.content_type("application/problem+json")
|
||||||
|
.json(ApiErrorDto {
|
||||||
|
code: self.code,
|
||||||
|
message: self.message.into(),
|
||||||
|
request_id: request_id(),
|
||||||
|
fields: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request_id() -> OpaqueId {
|
||||||
|
let mut bytes = [0_u8; 16];
|
||||||
|
getrandom::fill(&mut bytes).expect("operating system randomness is required");
|
||||||
|
let mut value = String::from("r");
|
||||||
|
for byte in bytes {
|
||||||
|
use std::fmt::Write as _;
|
||||||
|
write!(value, "{byte:02x}").expect("writing to String cannot fail");
|
||||||
|
}
|
||||||
|
OpaqueId::parse(value).expect("generated request identifier is valid")
|
||||||
|
}
|
||||||
735
crates/remora-api/tests/worker_api.rs
Normal file
735
crates/remora-api/tests/worker_api.rs
Normal file
|
|
@ -0,0 +1,735 @@
|
||||||
|
use std::{net::TcpListener, sync::Arc, time::Duration};
|
||||||
|
|
||||||
|
use actix_web::{App, HttpResponse, HttpServer, web};
|
||||||
|
use rcgen::{
|
||||||
|
BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, Issuer,
|
||||||
|
KeyPair, KeyUsagePurpose,
|
||||||
|
};
|
||||||
|
use remora_admission::{
|
||||||
|
AdmissionConfig, AdmissionDecision, AdmissionService, migrate as admission_migrate,
|
||||||
|
};
|
||||||
|
use remora_api::{
|
||||||
|
AuthenticatedWorker, WorkerApiState, authenticate_rustls_worker, configure_worker_api,
|
||||||
|
};
|
||||||
|
use remora_attempt::{AttemptConfig, AttemptService, RecoveryConfig, migrate as attempt_migrate};
|
||||||
|
use remora_contracts::{
|
||||||
|
ApiErrorDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto, AttemptFailureOutcomeDto,
|
||||||
|
AttemptOutcomeDto, AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||||
|
AttemptResolvedSourceRequestDto, AttemptRuntimeFailureDto, CheckpointDescriptorKindDto,
|
||||||
|
CheckpointDescriptorRefDto, ClaimRequestDto, ClaimResponseDto, HttpSourceSubmissionDto,
|
||||||
|
JobPhaseProjectionDto, JobSubmissionDto, PositiveIntegerDto, ProgressValueDto,
|
||||||
|
ResolvedSourceDescriptorKindDto, ResolvedSourceDescriptorRefDto, ResultDescriptorDto,
|
||||||
|
ResultKindDto, SafeIntegerDto, SourceSubmissionDto,
|
||||||
|
};
|
||||||
|
use remora_coordination::{CoordinationStore, StoreConfig};
|
||||||
|
use remora_credentials::migrate as credentials_migrate;
|
||||||
|
use remora_domain::{
|
||||||
|
BucketName, JobPhase, OpaqueId, RuntimeFailureClass, StorageScope, TargetPrefix, TargetStoreId,
|
||||||
|
};
|
||||||
|
use remora_durable::{PoolConfig, connect, migrate as durable_migrate};
|
||||||
|
use remora_identity::{DefaultTarget, Principal};
|
||||||
|
use remora_result::migrate as result_migrate;
|
||||||
|
use remora_target::attempt_manifest_key;
|
||||||
|
use reqwest::{Certificate as ClientCertificate, Client, Identity};
|
||||||
|
use rustls::{
|
||||||
|
RootCertStore, ServerConfig,
|
||||||
|
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
|
||||||
|
server::WebPkiClientVerifier,
|
||||||
|
};
|
||||||
|
use sqlx::PgPool;
|
||||||
|
|
||||||
|
struct Harness {
|
||||||
|
pool: PgPool,
|
||||||
|
coordination: CoordinationStore,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Harness {
|
||||||
|
async fn new(name: &str) -> Option<Self> {
|
||||||
|
let database_url = required_env("DATABASE_URL")?;
|
||||||
|
let coordination_url = required_env("REMORA_COORDINATION_URL")?;
|
||||||
|
let pool = connect(
|
||||||
|
&database_url,
|
||||||
|
PoolConfig {
|
||||||
|
max_connections: 16,
|
||||||
|
acquire_timeout: Duration::from_secs(10),
|
||||||
|
idle_timeout: Some(Duration::from_secs(30)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
durable_migrate(&pool).await.unwrap();
|
||||||
|
credentials_migrate(&pool).await.unwrap();
|
||||||
|
admission_migrate(&pool).await.unwrap();
|
||||||
|
attempt_migrate(&pool).await.unwrap();
|
||||||
|
result_migrate(&pool).await.unwrap();
|
||||||
|
sqlx::query("TRUNCATE TABLE jobs CASCADE")
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let coordination = CoordinationStore::connect(
|
||||||
|
&coordination_url,
|
||||||
|
format!("{}wp18:{name}:{}:", coordination_namespace(), nonce()),
|
||||||
|
StoreConfig {
|
||||||
|
global_limit: 16,
|
||||||
|
lease_ttl: Duration::from_secs(10),
|
||||||
|
reserve_ttl: Duration::from_secs(10),
|
||||||
|
progress_ttl: Duration::from_secs(10),
|
||||||
|
cancel_ttl: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
Some(Self { pool, coordination })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn service(&self) -> AttemptService {
|
||||||
|
AttemptService::new(
|
||||||
|
self.pool.clone(),
|
||||||
|
self.coordination.clone(),
|
||||||
|
AttemptConfig {
|
||||||
|
max_attempts: 3,
|
||||||
|
local_limit: 8,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn job(&self) -> OpaqueId {
|
||||||
|
let store = TargetStoreId::parse("primary").unwrap();
|
||||||
|
let bucket = BucketName::parse("remora-results").unwrap();
|
||||||
|
let prefix = TargetPrefix::parse("users/alice", ["_remora"]).unwrap();
|
||||||
|
let principal = Principal {
|
||||||
|
id: unique_id("owner"),
|
||||||
|
issuer: "https://issuer.example".into(),
|
||||||
|
subject: "subject".into(),
|
||||||
|
administrator: false,
|
||||||
|
scopes: [StorageScope {
|
||||||
|
store: store.clone(),
|
||||||
|
bucket: bucket.clone(),
|
||||||
|
prefix: prefix.clone(),
|
||||||
|
}]
|
||||||
|
.into_iter()
|
||||||
|
.collect(),
|
||||||
|
};
|
||||||
|
let service = AdmissionService::new(
|
||||||
|
self.pool.clone(),
|
||||||
|
AdmissionConfig {
|
||||||
|
default_target: DefaultTarget {
|
||||||
|
store,
|
||||||
|
bucket,
|
||||||
|
prefix,
|
||||||
|
},
|
||||||
|
reserved_target_segments: vec!["_remora".into()],
|
||||||
|
policy_version: "worker-api-v1".into(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let outcome = service
|
||||||
|
.submit(
|
||||||
|
&principal,
|
||||||
|
unique_id("submit").as_str(),
|
||||||
|
JobSubmissionDto {
|
||||||
|
source: SourceSubmissionDto::Http(HttpSourceSubmissionDto {
|
||||||
|
url: "https://example.invalid/artifact".into(),
|
||||||
|
result_path: "artifact.bin".into(),
|
||||||
|
credential_id: None,
|
||||||
|
}),
|
||||||
|
target: None,
|
||||||
|
label: Some("worker-api-test".into()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let AdmissionDecision::Accepted(job) = outcome.decision else {
|
||||||
|
panic!("job rejected")
|
||||||
|
};
|
||||||
|
self.coordination
|
||||||
|
.enqueue(job.job_id.as_str())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
job.job_id
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn seal_result(&self, attempt_id: &OpaqueId) -> ResultDescriptorDto {
|
||||||
|
let manifest_id = unique_id("manifest");
|
||||||
|
let checksum = format!("sha256:{}", "11".repeat(32));
|
||||||
|
let key = attempt_manifest_key(attempt_id, &manifest_id);
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO result_inventories (
|
||||||
|
attempt_id, state, manifest_schema_version, manifest_id, manifest_store,
|
||||||
|
manifest_bucket, manifest_key, manifest_version_id, manifest_sha256,
|
||||||
|
manifest_size, member_count, total_bytes
|
||||||
|
) VALUES ($1, 'sealed', 1, $2, 'primary', 'remora-results', $3,
|
||||||
|
'version-1', $4, 128, 1, 4096)",
|
||||||
|
)
|
||||||
|
.bind(attempt_id.as_str())
|
||||||
|
.bind(manifest_id.as_str())
|
||||||
|
.bind(&key)
|
||||||
|
.bind(&checksum)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
ResultDescriptorDto {
|
||||||
|
kind: ResultKindDto::Manifest,
|
||||||
|
manifest_id,
|
||||||
|
schema_version: positive(1),
|
||||||
|
manifest: remora_contracts::ObjectIdentityDto {
|
||||||
|
store: "primary".into(),
|
||||||
|
bucket: "remora-results".into(),
|
||||||
|
key,
|
||||||
|
version_id: Some("version-1".into()),
|
||||||
|
checksum,
|
||||||
|
size: safe(128),
|
||||||
|
},
|
||||||
|
member_count: positive(1),
|
||||||
|
total_bytes: safe(4096),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_operations_are_authenticated_fenced_replayable_and_redacted() {
|
||||||
|
let Some(harness) = Harness::new("flow").await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let state = WorkerApiState::new(harness.service());
|
||||||
|
let (base, server, client) = start_authenticated_server(state).await;
|
||||||
|
let job_id = harness.job().await;
|
||||||
|
let worker = unique_id("worker");
|
||||||
|
let claim_operation = unique_id("operation");
|
||||||
|
let claim = ClaimRequestDto {
|
||||||
|
operation_id: claim_operation.clone(),
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let unauthenticated = actix_web::test::init_service(App::new().route(
|
||||||
|
"/",
|
||||||
|
web::post().to(|_: AuthenticatedWorker| async { HttpResponse::NoContent().finish() }),
|
||||||
|
))
|
||||||
|
.await;
|
||||||
|
let response = actix_web::test::call_service(
|
||||||
|
&unauthenticated,
|
||||||
|
actix_web::test::TestRequest::post().uri("/").to_request(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), 401);
|
||||||
|
assert_eq!(
|
||||||
|
response.headers().get("content-type").unwrap(),
|
||||||
|
"application/problem+json"
|
||||||
|
);
|
||||||
|
|
||||||
|
let claim_url = format!("{base}/internal/v1/jobs/{job_id}/claim");
|
||||||
|
let response = client.post(&claim_url).json(&claim).send().await.unwrap();
|
||||||
|
assert_eq!(response.status(), 200);
|
||||||
|
let claimed: ClaimResponseDto = response.json().await.unwrap();
|
||||||
|
assert_eq!(claimed.authority.worker_incarnation, worker);
|
||||||
|
assert_eq!(claimed.execution.attempt_id, claimed.authority.attempt_id);
|
||||||
|
assert_eq!(claimed.execution.job_id, job_id);
|
||||||
|
assert!(
|
||||||
|
claimed
|
||||||
|
.execution
|
||||||
|
.staging_target
|
||||||
|
.prefix
|
||||||
|
.contains(claimed.authority.attempt_id.as_str())
|
||||||
|
);
|
||||||
|
|
||||||
|
let changed_claim = client
|
||||||
|
.post(&claim_url)
|
||||||
|
.json(&ClaimRequestDto {
|
||||||
|
operation_id: claim_operation,
|
||||||
|
worker_incarnation: unique_id("worker"),
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(changed_claim.status(), 409);
|
||||||
|
assert_eq!(
|
||||||
|
changed_claim.headers().get("content-type").unwrap(),
|
||||||
|
"application/problem+json"
|
||||||
|
);
|
||||||
|
|
||||||
|
let attempt_url = format!(
|
||||||
|
"{base}/internal/v1/attempts/{}",
|
||||||
|
claimed.authority.attempt_id
|
||||||
|
);
|
||||||
|
let activation_operation = unique_id("operation");
|
||||||
|
let activation = AttemptActivateRequestDto {
|
||||||
|
operation_id: activation_operation,
|
||||||
|
fence: claimed.authority.fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
};
|
||||||
|
let response = client
|
||||||
|
.post(format!("{attempt_url}/activate"))
|
||||||
|
.json(&activation)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), 200);
|
||||||
|
let activation_body: serde_json::Value = response.json().await.unwrap();
|
||||||
|
let activated: remora_contracts::ActivationResponseDto =
|
||||||
|
serde_json::from_value(activation_body.clone()).unwrap();
|
||||||
|
assert!(activated.lease.renew_after.as_str() < activated.lease.expires_at.as_str());
|
||||||
|
|
||||||
|
let renewal_operation = unique_id("operation");
|
||||||
|
let renewal = AttemptRenewRequestDto {
|
||||||
|
operation_id: renewal_operation.clone(),
|
||||||
|
fence: claimed.authority.fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
};
|
||||||
|
let first_renewal = client
|
||||||
|
.post(format!("{attempt_url}/renew"))
|
||||||
|
.json(&renewal)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(first_renewal.status(), 200);
|
||||||
|
let first_lease: serde_json::Value = first_renewal.json().await.unwrap();
|
||||||
|
let replayed_lease: serde_json::Value = client
|
||||||
|
.post(format!("{attempt_url}/renew"))
|
||||||
|
.json(&renewal)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(replayed_lease, first_lease);
|
||||||
|
let replayed_activation: serde_json::Value = client
|
||||||
|
.post(format!("{attempt_url}/activate"))
|
||||||
|
.json(&activation)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(replayed_activation, activation_body);
|
||||||
|
|
||||||
|
let authority_mismatches = [
|
||||||
|
(
|
||||||
|
format!("{base}/internal/v1/attempts/{}/renew", unique_id("attempt")),
|
||||||
|
claimed.authority.fence,
|
||||||
|
worker.clone(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
format!("{attempt_url}/renew"),
|
||||||
|
positive(claimed.authority.fence.get() + 1),
|
||||||
|
worker.clone(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
format!("{attempt_url}/renew"),
|
||||||
|
claimed.authority.fence,
|
||||||
|
unique_id("worker"),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
for (url, fence, worker_incarnation) in authority_mismatches {
|
||||||
|
let response = client
|
||||||
|
.post(url)
|
||||||
|
.json(&AttemptRenewRequestDto {
|
||||||
|
operation_id: unique_id("operation"),
|
||||||
|
fence,
|
||||||
|
worker_incarnation,
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), 409);
|
||||||
|
}
|
||||||
|
|
||||||
|
let resolved_id = unique_id("resolved");
|
||||||
|
let resolved = AttemptResolvedSourceRequestDto {
|
||||||
|
operation_id: unique_id("operation"),
|
||||||
|
fence: claimed.authority.fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
resolved_source_ref: ResolvedSourceDescriptorRefDto {
|
||||||
|
id: resolved_id.clone(),
|
||||||
|
kind: ResolvedSourceDescriptorKindDto::ResolvedSource,
|
||||||
|
schema_version: positive(1),
|
||||||
|
byte_length: safe(64),
|
||||||
|
item_count: Some(safe(1)),
|
||||||
|
checksum: format!("sha256:{}", "22".repeat(32)),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
client
|
||||||
|
.post(format!("{attempt_url}/resolved-source"))
|
||||||
|
.json(&resolved)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.status(),
|
||||||
|
204
|
||||||
|
);
|
||||||
|
let checkpoint = AttemptCheckpointRequestDto {
|
||||||
|
operation_id: unique_id("operation"),
|
||||||
|
fence: claimed.authority.fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
checkpoint_ref: CheckpointDescriptorRefDto {
|
||||||
|
id: unique_id("checkpoint"),
|
||||||
|
kind: CheckpointDescriptorKindDto::Checkpoint,
|
||||||
|
schema_version: positive(1),
|
||||||
|
byte_length: safe(32),
|
||||||
|
item_count: Some(safe(1)),
|
||||||
|
checksum: format!("sha256:{}", "33".repeat(32)),
|
||||||
|
},
|
||||||
|
resolved_source_id: resolved_id,
|
||||||
|
validated_bytes: safe(1024),
|
||||||
|
validated_members: safe(1),
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
client
|
||||||
|
.post(format!("{attempt_url}/checkpoint"))
|
||||||
|
.json(&checkpoint)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.status(),
|
||||||
|
204
|
||||||
|
);
|
||||||
|
let progress = AttemptProgressRequestDto {
|
||||||
|
operation_id: unique_id("operation"),
|
||||||
|
fence: claimed.authority.fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
progress: ProgressValueDto {
|
||||||
|
completed_bytes: safe(2048),
|
||||||
|
total_bytes: Some(safe(4096)),
|
||||||
|
completed_members: safe(1),
|
||||||
|
total_members: Some(safe(1)),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
client
|
||||||
|
.post(format!("{attempt_url}/progress"))
|
||||||
|
.json(&progress)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.status(),
|
||||||
|
204
|
||||||
|
);
|
||||||
|
|
||||||
|
let replayed_claim: ClaimResponseDto = client
|
||||||
|
.post(&claim_url)
|
||||||
|
.json(&claim)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(replayed_claim).unwrap(),
|
||||||
|
serde_json::to_value(&claimed).unwrap()
|
||||||
|
);
|
||||||
|
|
||||||
|
let failure = AttemptOutcomeRequestDto {
|
||||||
|
operation_id: unique_id("operation"),
|
||||||
|
fence: claimed.authority.fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
outcome: AttemptOutcomeDto::Failure(AttemptFailureOutcomeDto {
|
||||||
|
failure: AttemptRuntimeFailureDto {
|
||||||
|
class: RuntimeFailureClass::TransientSource,
|
||||||
|
message: "secret-value-must-not-escape".into(),
|
||||||
|
safe_details: None,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let outcome_url = format!("{attempt_url}/outcome");
|
||||||
|
let response = client
|
||||||
|
.post(&outcome_url)
|
||||||
|
.json(&failure)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), 200);
|
||||||
|
let bytes = response.bytes().await.unwrap();
|
||||||
|
assert!(!String::from_utf8_lossy(&bytes).contains("secret-value-must-not-escape"));
|
||||||
|
let projection: JobPhaseProjectionDto = serde_json::from_slice(&bytes).unwrap();
|
||||||
|
assert_eq!(projection.phase, JobPhase::RetryWait);
|
||||||
|
let report = harness
|
||||||
|
.service()
|
||||||
|
.reconcile_once(&RecoveryConfig {
|
||||||
|
retry_delay: Duration::ZERO,
|
||||||
|
cleanup_claim_ttl: Duration::from_secs(1),
|
||||||
|
batch_size: 128,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(report.retries_scheduled, 1);
|
||||||
|
let replay = client
|
||||||
|
.post(&outcome_url)
|
||||||
|
.json(&failure)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(replay.status(), 200);
|
||||||
|
assert_eq!(replay.bytes().await.unwrap(), bytes);
|
||||||
|
|
||||||
|
let cross_kind = client
|
||||||
|
.post(format!("{attempt_url}/progress"))
|
||||||
|
.json(&AttemptProgressRequestDto {
|
||||||
|
operation_id: renewal_operation,
|
||||||
|
fence: claimed.authority.fence,
|
||||||
|
worker_incarnation: worker,
|
||||||
|
progress: ProgressValueDto {
|
||||||
|
completed_bytes: safe(2048),
|
||||||
|
total_bytes: Some(safe(4096)),
|
||||||
|
completed_members: safe(1),
|
||||||
|
total_members: Some(safe(1)),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(cross_kind.status(), 409);
|
||||||
|
assert_eq!(
|
||||||
|
cross_kind.headers().get("content-type").unwrap(),
|
||||||
|
"application/problem+json"
|
||||||
|
);
|
||||||
|
let error: ApiErrorDto = cross_kind.json().await.unwrap();
|
||||||
|
assert_eq!(error.message, "precondition failed");
|
||||||
|
server.stop(true).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn success_outcome_publishes_the_exact_manifest_and_replays() {
|
||||||
|
let Some(harness) = Harness::new("success").await else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let state = WorkerApiState::new(harness.service());
|
||||||
|
let (base, server, client) = start_authenticated_server(state).await;
|
||||||
|
let job_id = harness.job().await;
|
||||||
|
let worker = unique_id("worker");
|
||||||
|
let claim: ClaimResponseDto = client
|
||||||
|
.post(format!("{base}/internal/v1/jobs/{job_id}/claim"))
|
||||||
|
.json(&ClaimRequestDto {
|
||||||
|
operation_id: unique_id("operation"),
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let attempt_url = format!("{base}/internal/v1/attempts/{}", claim.authority.attempt_id);
|
||||||
|
assert_eq!(
|
||||||
|
client
|
||||||
|
.post(format!("{attempt_url}/activate"))
|
||||||
|
.json(&AttemptActivateRequestDto {
|
||||||
|
operation_id: unique_id("operation"),
|
||||||
|
fence: claim.authority.fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.status(),
|
||||||
|
200
|
||||||
|
);
|
||||||
|
let result = harness.seal_result(&claim.authority.attempt_id).await;
|
||||||
|
let operation = unique_id("operation");
|
||||||
|
let outcome = AttemptOutcomeRequestDto {
|
||||||
|
operation_id: operation,
|
||||||
|
fence: claim.authority.fence,
|
||||||
|
worker_incarnation: worker,
|
||||||
|
outcome: AttemptOutcomeDto::Success(remora_contracts::AttemptSuccessOutcomeDto { result }),
|
||||||
|
};
|
||||||
|
let outcome_url = format!("{attempt_url}/outcome");
|
||||||
|
for _ in 0..2 {
|
||||||
|
let response = client
|
||||||
|
.post(&outcome_url)
|
||||||
|
.json(&outcome)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), 200);
|
||||||
|
assert_eq!(
|
||||||
|
response
|
||||||
|
.json::<JobPhaseProjectionDto>()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.phase,
|
||||||
|
JobPhase::Succeeded
|
||||||
|
);
|
||||||
|
}
|
||||||
|
server.stop(true).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn mutual_tls_requires_a_trusted_client_certificate() {
|
||||||
|
let certificates = certificates();
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
let server = HttpServer::new(|| {
|
||||||
|
App::new().route(
|
||||||
|
"/identity",
|
||||||
|
web::get().to(|identity: AuthenticatedWorker| async move {
|
||||||
|
HttpResponse::Ok().body(identity.identity.to_string())
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.on_connect(authenticate_rustls_worker)
|
||||||
|
.listen_rustls_0_23(listener, certificates.server_config)
|
||||||
|
.unwrap()
|
||||||
|
.run();
|
||||||
|
let handle = server.handle();
|
||||||
|
tokio::spawn(server);
|
||||||
|
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
|
||||||
|
let untrusted = Client::builder()
|
||||||
|
.add_root_certificate(root.clone())
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
untrusted
|
||||||
|
.get(format!("https://localhost:{}/identity", address.port()))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
let mut client_pem = certificates.client.pem();
|
||||||
|
client_pem.push_str(&certificates.client_key.serialize_pem());
|
||||||
|
let trusted = Client::builder()
|
||||||
|
.add_root_certificate(root)
|
||||||
|
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
let response = trusted
|
||||||
|
.get(format!("https://localhost:{}/identity", address.port()))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), 200);
|
||||||
|
let expected = certificate_identity(certificates.client.der().as_ref());
|
||||||
|
assert_eq!(response.text().await.unwrap(), expected);
|
||||||
|
handle.stop(true).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_authenticated_server(
|
||||||
|
state: WorkerApiState,
|
||||||
|
) -> (String, actix_web::dev::ServerHandle, Client) {
|
||||||
|
let certificates = certificates();
|
||||||
|
let client = trusted_client(&certificates);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let address = listener.local_addr().unwrap();
|
||||||
|
let server = HttpServer::new(move || {
|
||||||
|
App::new()
|
||||||
|
.app_data(web::Data::new(state.clone()))
|
||||||
|
.configure(configure_worker_api)
|
||||||
|
})
|
||||||
|
.on_connect(authenticate_rustls_worker)
|
||||||
|
.listen_rustls_0_23(listener, certificates.server_config)
|
||||||
|
.unwrap()
|
||||||
|
.run();
|
||||||
|
let handle = server.handle();
|
||||||
|
tokio::spawn(server);
|
||||||
|
(
|
||||||
|
format!("https://localhost:{}", address.port()),
|
||||||
|
handle,
|
||||||
|
client,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trusted_client(certificates: &Certificates) -> Client {
|
||||||
|
let root = ClientCertificate::from_der(certificates.ca.der()).unwrap();
|
||||||
|
let mut client_pem = certificates.client.pem();
|
||||||
|
client_pem.push_str(&certificates.client_key.serialize_pem());
|
||||||
|
Client::builder()
|
||||||
|
.add_root_certificate(root)
|
||||||
|
.identity(Identity::from_pem(client_pem.as_bytes()).unwrap())
|
||||||
|
.build()
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn certificate_identity(certificate: &[u8]) -> String {
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
let mut value = String::from("w");
|
||||||
|
for byte in Sha256::digest(certificate) {
|
||||||
|
use std::fmt::Write as _;
|
||||||
|
write!(value, "{byte:02x}").unwrap();
|
||||||
|
}
|
||||||
|
value
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Certificates {
|
||||||
|
ca: Certificate,
|
||||||
|
client: Certificate,
|
||||||
|
client_key: KeyPair,
|
||||||
|
server_config: ServerConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn certificates() -> Certificates {
|
||||||
|
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||||
|
let mut ca_params = CertificateParams::new(Vec::new()).unwrap();
|
||||||
|
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
|
||||||
|
ca_params.key_usages = vec![
|
||||||
|
KeyUsagePurpose::DigitalSignature,
|
||||||
|
KeyUsagePurpose::KeyCertSign,
|
||||||
|
KeyUsagePurpose::CrlSign,
|
||||||
|
];
|
||||||
|
let ca_key = KeyPair::generate().unwrap();
|
||||||
|
let ca = ca_params.self_signed(&ca_key).unwrap();
|
||||||
|
let issuer = Issuer::new(ca_params, ca_key);
|
||||||
|
|
||||||
|
let server_key = KeyPair::generate().unwrap();
|
||||||
|
let mut server_params = CertificateParams::new(vec!["localhost".into()]).unwrap();
|
||||||
|
server_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
|
||||||
|
server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth];
|
||||||
|
let server = server_params.signed_by(&server_key, &issuer).unwrap();
|
||||||
|
|
||||||
|
let client_key = KeyPair::generate().unwrap();
|
||||||
|
let mut client_params = CertificateParams::new(Vec::new()).unwrap();
|
||||||
|
client_params.key_usages = vec![KeyUsagePurpose::DigitalSignature];
|
||||||
|
client_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth];
|
||||||
|
let client = client_params.signed_by(&client_key, &issuer).unwrap();
|
||||||
|
|
||||||
|
let mut roots = RootCertStore::empty();
|
||||||
|
roots.add(ca.der().clone()).unwrap();
|
||||||
|
let verifier = WebPkiClientVerifier::builder(Arc::new(roots))
|
||||||
|
.build()
|
||||||
|
.unwrap();
|
||||||
|
let server_key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(server_key.serialize_der()));
|
||||||
|
let server_config = ServerConfig::builder()
|
||||||
|
.with_client_cert_verifier(verifier)
|
||||||
|
.with_single_cert(vec![server.der().clone()], server_key_der)
|
||||||
|
.unwrap();
|
||||||
|
Certificates {
|
||||||
|
ca,
|
||||||
|
client,
|
||||||
|
client_key,
|
||||||
|
server_config,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn required_env(name: &str) -> Option<String> {
|
||||||
|
match std::env::var(name) {
|
||||||
|
Ok(value) if !value.is_empty() => Some(value),
|
||||||
|
_ if std::env::var_os("REMORA_WORKER_API_REQUIRE").is_some() => {
|
||||||
|
panic!("{name} is required")
|
||||||
|
}
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn coordination_namespace() -> String {
|
||||||
|
std::env::var("REMORA_COORDINATION_NAMESPACE").unwrap_or_else(|_| "remora:test:".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn nonce() -> u128 {
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||||
|
(u128::from(std::process::id()) << 64) | u128::from(COUNTER.fetch_add(1, Ordering::Relaxed))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unique_id(prefix: &str) -> OpaqueId {
|
||||||
|
OpaqueId::parse(format!("{prefix}{:032x}", nonce())).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn positive(value: u64) -> PositiveIntegerDto {
|
||||||
|
PositiveIntegerDto::new(value).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn safe(value: u64) -> SafeIntegerDto {
|
||||||
|
SafeIntegerDto::new(value).unwrap()
|
||||||
|
}
|
||||||
2
crates/remora-attempt/migrations/0004_worker_api.sql
Normal file
2
crates/remora-attempt/migrations/0004_worker_api.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
ALTER TABLE attempts
|
||||||
|
ADD COLUMN execution_envelope JSONB;
|
||||||
|
|
@ -20,6 +20,7 @@ pub use recovery_types::{
|
||||||
};
|
};
|
||||||
pub use service::{AttemptConfig, AttemptService};
|
pub use service::{AttemptConfig, AttemptService};
|
||||||
pub use types::{
|
pub use types::{
|
||||||
ActivationOutcome, AttemptAuthority, AttemptFaultInjector, AttemptFaultPoint, AttemptState,
|
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptFaultInjector,
|
||||||
ClaimOutcome, ClaimRejection, RecoveryOutcome, RenewalOutcome,
|
AttemptFaultPoint, AttemptState, ClaimOutcome, ClaimRejection, ExecutionEnvelopeRecord,
|
||||||
|
LeaseProjection, OutcomePublication, RecoveryOutcome, RenewalMutationOutcome, RenewalOutcome,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ use time::OffsetDateTime;
|
||||||
use crate::{
|
use crate::{
|
||||||
AttemptAuthority, AttemptError, AttemptMutationOutcome, AttemptService, CheckpointProjection,
|
AttemptAuthority, AttemptError, AttemptMutationOutcome, AttemptService, CheckpointProjection,
|
||||||
CheckpointRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
|
CheckpointRecord, CommittedResultRecord, DurableDescriptor, FailureRecord,
|
||||||
JobMutationProjection, MutationRejection, ProgressRecord,
|
JobMutationProjection, MutationRejection, OutcomePublication, ProgressRecord,
|
||||||
};
|
};
|
||||||
|
|
||||||
impl AttemptService {
|
impl AttemptService {
|
||||||
|
|
@ -64,6 +64,18 @@ impl AttemptService {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn publish_success_operation(
|
||||||
|
&self,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
operation_id: &OpaqueId,
|
||||||
|
result: PublicationInput,
|
||||||
|
) -> Result<OutcomePublication, AttemptError> {
|
||||||
|
let outcome = self
|
||||||
|
.publish_success(authority, operation_id, result)
|
||||||
|
.await?;
|
||||||
|
self.publication_outcome(operation_id, outcome).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn persist_checkpoint(
|
pub async fn persist_checkpoint(
|
||||||
&self,
|
&self,
|
||||||
authority: &AttemptAuthority,
|
authority: &AttemptAuthority,
|
||||||
|
|
@ -124,6 +136,16 @@ impl AttemptService {
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn publish_failure_operation(
|
||||||
|
&self,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
operation_id: &OpaqueId,
|
||||||
|
class: RuntimeFailureClass,
|
||||||
|
) -> Result<OutcomePublication, AttemptError> {
|
||||||
|
let outcome = self.publish_failure(authority, operation_id, class).await?;
|
||||||
|
self.publication_outcome(operation_id, outcome).await
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn report_progress(
|
pub async fn report_progress(
|
||||||
&self,
|
&self,
|
||||||
authority: &AttemptAuthority,
|
authority: &AttemptAuthority,
|
||||||
|
|
@ -227,7 +249,7 @@ impl AttemptService {
|
||||||
.execute(tx.as_mut())
|
.execute(tx.as_mut())
|
||||||
.await?;
|
.await?;
|
||||||
enqueue(tx, "job.updated", json!({ "job_id": authority.job_id })).await?;
|
enqueue(tx, "job.updated", json!({ "job_id": authority.job_id })).await?;
|
||||||
accepted()
|
accepted_phase(JobPhase::Succeeded)
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -307,7 +329,11 @@ impl AttemptService {
|
||||||
json!({ "job_id": authority.job_id }),
|
json!({ "job_id": authority.job_id }),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
accepted()
|
accepted_phase(if retry {
|
||||||
|
JobPhase::RetryWait
|
||||||
|
} else {
|
||||||
|
JobPhase::Failed
|
||||||
|
})
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -329,6 +355,43 @@ impl AttemptService {
|
||||||
row.map(ProjectionRow::try_into_projection).transpose()
|
row.map(ProjectionRow::try_into_projection).transpose()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn publication_outcome(
|
||||||
|
&self,
|
||||||
|
operation_id: &OpaqueId,
|
||||||
|
outcome: AttemptMutationOutcome,
|
||||||
|
) -> Result<OutcomePublication, AttemptError> {
|
||||||
|
match outcome {
|
||||||
|
AttemptMutationOutcome::Rejected {
|
||||||
|
reason,
|
||||||
|
disposition,
|
||||||
|
} => Ok(OutcomePublication::Rejected {
|
||||||
|
reason,
|
||||||
|
disposition,
|
||||||
|
}),
|
||||||
|
AttemptMutationOutcome::Accepted(disposition) => {
|
||||||
|
let response = sqlx::query_scalar::<_, Json<Value>>(
|
||||||
|
"SELECT response_body FROM mutation_records
|
||||||
|
WHERE scope_subject = $1 AND scope_operation = 'attempt.mutation'
|
||||||
|
AND operation_key = $2",
|
||||||
|
)
|
||||||
|
.bind(self.operation_subject.as_str())
|
||||||
|
.bind(operation_id.as_str())
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::from)?;
|
||||||
|
let phase = serde_json::from_value(
|
||||||
|
response
|
||||||
|
.0
|
||||||
|
.get("phase")
|
||||||
|
.cloned()
|
||||||
|
.ok_or_else(|| AttemptError::Invalid("missing outcome phase".into()))?,
|
||||||
|
)
|
||||||
|
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||||
|
Ok(OutcomePublication::Accepted { disposition, phase })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn execute_durable_mutation<T, D>(
|
async fn execute_durable_mutation<T, D>(
|
||||||
&self,
|
&self,
|
||||||
authority: &AttemptAuthority,
|
authority: &AttemptAuthority,
|
||||||
|
|
@ -423,6 +486,13 @@ fn accepted() -> Result<(MutationStatus, Value), DurableError> {
|
||||||
Ok((MutationStatus::Accepted, json!({ "accepted": true })))
|
Ok((MutationStatus::Accepted, json!({ "accepted": true })))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn accepted_phase(phase: JobPhase) -> Result<(MutationStatus, Value), DurableError> {
|
||||||
|
Ok((
|
||||||
|
MutationStatus::Accepted,
|
||||||
|
json!({ "accepted": true, "phase": phase }),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
fn rejected(reason: MutationRejection) -> Result<(MutationStatus, Value), DurableError> {
|
fn rejected(reason: MutationRejection) -> Result<(MutationStatus, Value), DurableError> {
|
||||||
Ok((MutationStatus::Rejected, json!({ "reason": reason })))
|
Ok((MutationStatus::Rejected, json!({ "reason": reason })))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -14,9 +14,10 @@ use sqlx::{PgPool, types::Json};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
ActivationOutcome, AttemptAuthority, AttemptError, AttemptFaultInjector, AttemptFaultPoint,
|
ActivationMutationOutcome, ActivationOutcome, AttemptAuthority, AttemptError,
|
||||||
AttemptState, CheckpointProjection, ClaimOutcome, ClaimRejection, RecoveryOutcome,
|
AttemptFaultInjector, AttemptFaultPoint, AttemptState, CheckpointProjection, ClaimOutcome,
|
||||||
RenewalOutcome,
|
ClaimRejection, ExecutionEnvelopeRecord, LeaseProjection, RecoveryOutcome,
|
||||||
|
RenewalMutationOutcome, RenewalOutcome,
|
||||||
};
|
};
|
||||||
|
|
||||||
struct NoFaults;
|
struct NoFaults;
|
||||||
|
|
@ -179,6 +180,23 @@ impl AttemptService {
|
||||||
authority: &AttemptAuthority,
|
authority: &AttemptAuthority,
|
||||||
operation_id: &OpaqueId,
|
operation_id: &OpaqueId,
|
||||||
) -> Result<ActivationOutcome, AttemptError> {
|
) -> Result<ActivationOutcome, AttemptError> {
|
||||||
|
Ok(
|
||||||
|
match self.activate_operation(authority, operation_id).await? {
|
||||||
|
ActivationMutationOutcome::Activated { disposition, .. } => {
|
||||||
|
ActivationOutcome::Activated(disposition)
|
||||||
|
}
|
||||||
|
ActivationMutationOutcome::PreconditionFailed => {
|
||||||
|
ActivationOutcome::PreconditionFailed
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn activate_operation(
|
||||||
|
&self,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
operation_id: &OpaqueId,
|
||||||
|
) -> Result<ActivationMutationOutcome, AttemptError> {
|
||||||
validate_authority(authority)?;
|
validate_authority(authority)?;
|
||||||
let request_fingerprint = fingerprint(&ActivationRequest {
|
let request_fingerprint = fingerprint(&ActivationRequest {
|
||||||
job_id: &authority.job_id,
|
job_id: &authority.job_id,
|
||||||
|
|
@ -203,16 +221,17 @@ impl AttemptService {
|
||||||
request_fingerprint,
|
request_fingerprint,
|
||||||
move |tx| {
|
move |tx| {
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
let accepted =
|
let lease =
|
||||||
activate_attempt(tx, &coordination, &authority_for_effect, faults.as_ref())
|
activate_attempt(tx, &coordination, &authority_for_effect, faults.as_ref())
|
||||||
.await?;
|
.await?;
|
||||||
Ok((
|
Ok((
|
||||||
if accepted {
|
if lease.is_some() {
|
||||||
MutationStatus::Accepted
|
MutationStatus::Accepted
|
||||||
} else {
|
} else {
|
||||||
MutationStatus::Rejected
|
MutationStatus::Rejected
|
||||||
},
|
},
|
||||||
json!({ "activated": accepted }),
|
serde_json::to_value(ActivationStored { lease })
|
||||||
|
.map_err(DurableError::effect)?,
|
||||||
))
|
))
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
|
|
@ -225,7 +244,14 @@ impl AttemptService {
|
||||||
.exact_authority_is_executing_and_live(authority)
|
.exact_authority_is_executing_and_live(authority)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
return Ok(ActivationOutcome::Activated(MutationDisposition::First));
|
let lease = self
|
||||||
|
.live_lease(authority)
|
||||||
|
.await?
|
||||||
|
.ok_or_else(|| AttemptError::Invalid("live lease disappeared".into()))?;
|
||||||
|
return Ok(ActivationMutationOutcome::Activated {
|
||||||
|
disposition: MutationDisposition::First,
|
||||||
|
lease,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
self.release_exact(authority).await;
|
self.release_exact(authority).await;
|
||||||
return Err(error.into());
|
return Err(error.into());
|
||||||
|
|
@ -237,16 +263,24 @@ impl AttemptService {
|
||||||
}
|
}
|
||||||
if outcome.status == MutationStatus::Rejected {
|
if outcome.status == MutationStatus::Rejected {
|
||||||
self.release_exact(authority).await;
|
self.release_exact(authority).await;
|
||||||
return Ok(ActivationOutcome::PreconditionFailed);
|
return Ok(ActivationMutationOutcome::PreconditionFailed);
|
||||||
}
|
}
|
||||||
if !self
|
if !self
|
||||||
.exact_authority_is_executing_and_live(authority)
|
.exact_authority_is_executing_and_live(authority)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
self.release_exact(authority).await;
|
self.release_exact(authority).await;
|
||||||
return Ok(ActivationOutcome::PreconditionFailed);
|
return Ok(ActivationMutationOutcome::PreconditionFailed);
|
||||||
}
|
}
|
||||||
Ok(ActivationOutcome::Activated(outcome.disposition))
|
let stored: ActivationStored = serde_json::from_value(outcome.response)
|
||||||
|
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||||
|
let lease = stored
|
||||||
|
.lease
|
||||||
|
.ok_or_else(|| AttemptError::Invalid("accepted activation omitted its lease".into()))?;
|
||||||
|
Ok(ActivationMutationOutcome::Activated {
|
||||||
|
disposition: outcome.disposition,
|
||||||
|
lease,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn renew(
|
pub async fn renew(
|
||||||
|
|
@ -269,6 +303,103 @@ impl AttemptService {
|
||||||
Ok(RenewalOutcome::Renewed)
|
Ok(RenewalOutcome::Renewed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn renew_operation(
|
||||||
|
&self,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
operation_id: &OpaqueId,
|
||||||
|
) -> Result<RenewalMutationOutcome, AttemptError> {
|
||||||
|
validate_authority(authority)?;
|
||||||
|
let request_fingerprint = fingerprint(&ActivationRequest {
|
||||||
|
job_id: &authority.job_id,
|
||||||
|
attempt_id: &authority.attempt_id,
|
||||||
|
fence: authority.fence,
|
||||||
|
worker_incarnation: &authority.worker_incarnation,
|
||||||
|
})
|
||||||
|
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||||
|
let eligibility = authority_binding(authority);
|
||||||
|
let authority = authority.clone();
|
||||||
|
let coordination = self.coordination.clone();
|
||||||
|
let faults = self.faults.clone();
|
||||||
|
let outcome = execute_mutation(
|
||||||
|
&self.pool,
|
||||||
|
MutationScope {
|
||||||
|
subject: &self.operation_subject,
|
||||||
|
operation: "attempt.mutation",
|
||||||
|
key: operation_id.as_str(),
|
||||||
|
},
|
||||||
|
"renew_attempt",
|
||||||
|
&eligibility,
|
||||||
|
request_fingerprint,
|
||||||
|
move |tx| {
|
||||||
|
Box::pin(async move {
|
||||||
|
let Some(row) = lock_authority(tx, &authority.attempt_id).await? else {
|
||||||
|
return Ok((
|
||||||
|
MutationStatus::Rejected,
|
||||||
|
serde_json::to_value(RenewalMutationOutcome::AuthorityLost {
|
||||||
|
disposition: None,
|
||||||
|
})
|
||||||
|
.map_err(DurableError::effect)?,
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if row.state != AttemptState::Executing
|
||||||
|
|| !row.matches(
|
||||||
|
&authority.job_id,
|
||||||
|
authority.fence,
|
||||||
|
&authority.worker_incarnation,
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Ok((
|
||||||
|
MutationStatus::Rejected,
|
||||||
|
serde_json::to_value(RenewalMutationOutcome::AuthorityLost {
|
||||||
|
disposition: None,
|
||||||
|
})
|
||||||
|
.map_err(DurableError::effect)?,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
faults.hit(AttemptFaultPoint::RenewalBeforeLease);
|
||||||
|
let expected = lease_owner(&authority).map_err(DurableError::effect)?;
|
||||||
|
let renewed = coordination
|
||||||
|
.renew(authority.job_id.as_str(), &expected)
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::effect)?;
|
||||||
|
faults.hit(AttemptFaultPoint::RenewalAfterLease);
|
||||||
|
let live = coordination
|
||||||
|
.get_live_lease(authority.job_id.as_str())
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::effect)?;
|
||||||
|
let response = match live
|
||||||
|
.filter(|lease| renewed.ok() && lease.record.owner() == expected)
|
||||||
|
{
|
||||||
|
Some(lease) => RenewalMutationOutcome::Renewed {
|
||||||
|
lease: LeaseProjection {
|
||||||
|
expires_at_ms: lease.record.expires_at,
|
||||||
|
observed_at_ms: lease.now_ms,
|
||||||
|
},
|
||||||
|
disposition: None,
|
||||||
|
},
|
||||||
|
None => RenewalMutationOutcome::AuthorityLost { disposition: None },
|
||||||
|
};
|
||||||
|
let status = if matches!(response, RenewalMutationOutcome::Renewed { .. }) {
|
||||||
|
MutationStatus::Accepted
|
||||||
|
} else {
|
||||||
|
MutationStatus::Rejected
|
||||||
|
};
|
||||||
|
Ok((
|
||||||
|
status,
|
||||||
|
serde_json::to_value(response).map_err(DurableError::effect)?,
|
||||||
|
))
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if outcome.disposition == MutationDisposition::Conflict {
|
||||||
|
return Err(AttemptError::OperationConflict);
|
||||||
|
}
|
||||||
|
let response: RenewalMutationOutcome = serde_json::from_value(outcome.response)
|
||||||
|
.map_err(|error| AttemptError::Invalid(error.to_string()))?;
|
||||||
|
Ok(response.with_disposition(outcome.disposition))
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn recover_inactive(
|
pub async fn recover_inactive(
|
||||||
&self,
|
&self,
|
||||||
authority: &AttemptAuthority,
|
authority: &AttemptAuthority,
|
||||||
|
|
@ -372,6 +503,45 @@ impl AttemptService {
|
||||||
.map_err(Into::into)
|
.map_err(Into::into)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn historical_authority(
|
||||||
|
&self,
|
||||||
|
attempt_id: &OpaqueId,
|
||||||
|
) -> Result<Option<AttemptAuthority>, AttemptError> {
|
||||||
|
load_historical_authority(&self.pool, attempt_id)
|
||||||
|
.await
|
||||||
|
.map_err(Into::into)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn execution_envelope(
|
||||||
|
&self,
|
||||||
|
attempt_id: &OpaqueId,
|
||||||
|
) -> Result<Option<ExecutionEnvelopeRecord>, AttemptError> {
|
||||||
|
let value = sqlx::query_scalar::<_, Option<Json<ExecutionEnvelopeRecord>>>(
|
||||||
|
"SELECT execution_envelope FROM attempts WHERE attempt_id = $1",
|
||||||
|
)
|
||||||
|
.bind(attempt_id.as_str())
|
||||||
|
.fetch_optional(&self.pool)
|
||||||
|
.await?
|
||||||
|
.flatten();
|
||||||
|
Ok(value.map(|value| value.0))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn live_lease(
|
||||||
|
&self,
|
||||||
|
authority: &AttemptAuthority,
|
||||||
|
) -> Result<Option<LeaseProjection>, AttemptError> {
|
||||||
|
let expected = lease_owner(authority)?;
|
||||||
|
Ok(self
|
||||||
|
.coordination
|
||||||
|
.get_live_lease(authority.job_id.as_str())
|
||||||
|
.await?
|
||||||
|
.filter(|lease| lease.record.owner() == expected)
|
||||||
|
.map(|lease| LeaseProjection {
|
||||||
|
expires_at_ms: lease.record.expires_at,
|
||||||
|
observed_at_ms: lease.now_ms,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn release_local(&self, job_id: &OpaqueId) -> bool {
|
pub fn release_local(&self, job_id: &OpaqueId) -> bool {
|
||||||
self.local_permits.release_all(job_id.as_str())
|
self.local_permits.release_all(job_id.as_str())
|
||||||
}
|
}
|
||||||
|
|
@ -451,6 +621,11 @@ struct ActivationRequest<'a> {
|
||||||
worker_incarnation: &'a OpaqueId,
|
worker_incarnation: &'a OpaqueId,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(serde::Deserialize, Serialize)]
|
||||||
|
struct ActivationStored {
|
||||||
|
lease: Option<LeaseProjection>,
|
||||||
|
}
|
||||||
|
|
||||||
struct AuthorityRow {
|
struct AuthorityRow {
|
||||||
job_id: OpaqueId,
|
job_id: OpaqueId,
|
||||||
fence: u64,
|
fence: u64,
|
||||||
|
|
@ -534,11 +709,14 @@ async fn authorize_attempt(
|
||||||
_ => 0,
|
_ => 0,
|
||||||
};
|
};
|
||||||
let now = OffsetDateTime::now_utc();
|
let now = OffsetDateTime::now_utc();
|
||||||
|
let envelope =
|
||||||
|
load_execution_envelope_record(tx, input.job_id, input.attempt_id, fence, input.worker)
|
||||||
|
.await?;
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO attempts (
|
"INSERT INTO attempts (
|
||||||
attempt_id, job_id, fence, worker_incarnation, state,
|
attempt_id, job_id, fence, worker_incarnation, state,
|
||||||
initial_progress, authorized_at, executing_at, reaped_at
|
initial_progress, authorized_at, executing_at, reaped_at, execution_envelope
|
||||||
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL)",
|
) VALUES ($1, $2, $3, $4, 'authorized', $5, $6, NULL, NULL, $7)",
|
||||||
)
|
)
|
||||||
.bind(input.attempt_id.as_str())
|
.bind(input.attempt_id.as_str())
|
||||||
.bind(input.job_id.as_str())
|
.bind(input.job_id.as_str())
|
||||||
|
|
@ -546,6 +724,7 @@ async fn authorize_attempt(
|
||||||
.bind(input.worker.as_str())
|
.bind(input.worker.as_str())
|
||||||
.bind(i64::try_from(initial_progress).map_err(DurableError::effect)?)
|
.bind(i64::try_from(initial_progress).map_err(DurableError::effect)?)
|
||||||
.bind(now)
|
.bind(now)
|
||||||
|
.bind(Json(&envelope))
|
||||||
.execute(tx.as_mut())
|
.execute(tx.as_mut())
|
||||||
.await?;
|
.await?;
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
|
|
@ -572,21 +751,79 @@ async fn authorize_attempt(
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn load_execution_envelope_record(
|
||||||
|
tx: &mut remora_durable::PgTx<'_>,
|
||||||
|
job_id: &OpaqueId,
|
||||||
|
attempt_id: &OpaqueId,
|
||||||
|
fence: u64,
|
||||||
|
worker: &OpaqueId,
|
||||||
|
) -> Result<ExecutionEnvelopeRecord, DurableError> {
|
||||||
|
type Row = (
|
||||||
|
Json<serde_json::Value>,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
Json<Vec<String>>,
|
||||||
|
String,
|
||||||
|
Option<String>,
|
||||||
|
Option<String>,
|
||||||
|
Option<Json<crate::DurableDescriptor>>,
|
||||||
|
Option<Json<CheckpointProjection>>,
|
||||||
|
);
|
||||||
|
let row = sqlx::query_as::<_, Row>(
|
||||||
|
"SELECT j.source_projection, j.sensitive_source_ref, j.target_store,
|
||||||
|
j.target_bucket, j.target_prefix, j.execution_grant_id,
|
||||||
|
g.credential_id, g.credential_version_id, j.resolved_source, j.checkpoint
|
||||||
|
FROM jobs j
|
||||||
|
JOIN execution_grants g ON g.grant_id = j.execution_grant_id
|
||||||
|
WHERE j.job_id = $1",
|
||||||
|
)
|
||||||
|
.bind(job_id.as_str())
|
||||||
|
.fetch_one(tx.as_mut())
|
||||||
|
.await?;
|
||||||
|
let grant_id = OpaqueId::parse(row.5).map_err(DurableError::effect)?;
|
||||||
|
Ok(ExecutionEnvelopeRecord {
|
||||||
|
id: grant_id.clone(),
|
||||||
|
job_id: job_id.clone(),
|
||||||
|
attempt_id: attempt_id.clone(),
|
||||||
|
fence,
|
||||||
|
worker_incarnation: worker.clone(),
|
||||||
|
source: row.0.0,
|
||||||
|
sensitive_source_ref: OpaqueId::parse(row.1).map_err(DurableError::effect)?,
|
||||||
|
credential_id: row
|
||||||
|
.6
|
||||||
|
.map(OpaqueId::parse)
|
||||||
|
.transpose()
|
||||||
|
.map_err(DurableError::effect)?,
|
||||||
|
credential_version_id: row
|
||||||
|
.7
|
||||||
|
.map(OpaqueId::parse)
|
||||||
|
.transpose()
|
||||||
|
.map_err(DurableError::effect)?,
|
||||||
|
resolved_source: row.8.map(|value| value.0),
|
||||||
|
target_store: row.2,
|
||||||
|
target_bucket: row.3,
|
||||||
|
target_prefix: row.4.0,
|
||||||
|
execution_grant_id: grant_id,
|
||||||
|
checkpoint: row.9.map(|value| value.0),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn activate_attempt(
|
async fn activate_attempt(
|
||||||
tx: &mut remora_durable::PgTx<'_>,
|
tx: &mut remora_durable::PgTx<'_>,
|
||||||
coordination: &CoordinationStore,
|
coordination: &CoordinationStore,
|
||||||
authority: &AttemptAuthority,
|
authority: &AttemptAuthority,
|
||||||
faults: &dyn AttemptFaultInjector,
|
faults: &dyn AttemptFaultInjector,
|
||||||
) -> Result<bool, DurableError> {
|
) -> Result<Option<LeaseProjection>, DurableError> {
|
||||||
let Some(row) = lock_authority(tx, &authority.attempt_id).await? else {
|
let Some(row) = lock_authority(tx, &authority.attempt_id).await? else {
|
||||||
return Ok(false);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
if !row.matches(
|
if !row.matches(
|
||||||
&authority.job_id,
|
&authority.job_id,
|
||||||
authority.fence,
|
authority.fence,
|
||||||
&authority.worker_incarnation,
|
&authority.worker_incarnation,
|
||||||
) {
|
) {
|
||||||
return Ok(false);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
if row.state == AttemptState::Executing {
|
if row.state == AttemptState::Executing {
|
||||||
let live = coordination
|
let live = coordination
|
||||||
|
|
@ -595,16 +832,28 @@ async fn activate_attempt(
|
||||||
.map_err(DurableError::effect)?;
|
.map_err(DurableError::effect)?;
|
||||||
let expected = lease_owner(authority).map_err(DurableError::effect)?;
|
let expected = lease_owner(authority).map_err(DurableError::effect)?;
|
||||||
if !live.is_some_and(|lease| lease.record.owner() == expected) {
|
if !live.is_some_and(|lease| lease.record.owner() == expected) {
|
||||||
return Ok(false);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
return Ok(coordination
|
if !coordination
|
||||||
.renew(authority.job_id.as_str(), &expected)
|
.renew(authority.job_id.as_str(), &expected)
|
||||||
.await
|
.await
|
||||||
.map_err(DurableError::effect)?
|
.map_err(DurableError::effect)?
|
||||||
.ok());
|
.ok()
|
||||||
|
{
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
return Ok(coordination
|
||||||
|
.get_live_lease(authority.job_id.as_str())
|
||||||
|
.await
|
||||||
|
.map_err(DurableError::effect)?
|
||||||
|
.filter(|lease| lease.record.owner() == expected)
|
||||||
|
.map(|lease| LeaseProjection {
|
||||||
|
expires_at_ms: lease.record.expires_at,
|
||||||
|
observed_at_ms: lease.now_ms,
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
if row.state != AttemptState::Authorized {
|
if row.state != AttemptState::Authorized {
|
||||||
return Ok(false);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
faults.hit(AttemptFaultPoint::ActivationBeforeLease);
|
faults.hit(AttemptFaultPoint::ActivationBeforeLease);
|
||||||
let owner = lease_owner(authority).map_err(DurableError::effect)?;
|
let owner = lease_owner(authority).map_err(DurableError::effect)?;
|
||||||
|
|
@ -620,7 +869,7 @@ async fn activate_attempt(
|
||||||
.await
|
.await
|
||||||
.map_err(DurableError::effect)?;
|
.map_err(DurableError::effect)?;
|
||||||
if !coordinated.ok() {
|
if !coordinated.ok() {
|
||||||
return Ok(false);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
faults.hit(AttemptFaultPoint::ActivationAfterLease);
|
faults.hit(AttemptFaultPoint::ActivationAfterLease);
|
||||||
let live = coordination
|
let live = coordination
|
||||||
|
|
@ -628,9 +877,9 @@ async fn activate_attempt(
|
||||||
.await
|
.await
|
||||||
.map_err(DurableError::effect)?;
|
.map_err(DurableError::effect)?;
|
||||||
let expected = lease_owner(authority).map_err(DurableError::effect)?;
|
let expected = lease_owner(authority).map_err(DurableError::effect)?;
|
||||||
if !live.is_some_and(|lease| lease.record.owner() == expected) {
|
let Some(live) = live.filter(|lease| lease.record.owner() == expected) else {
|
||||||
return Ok(false);
|
return Ok(None);
|
||||||
}
|
};
|
||||||
let now = OffsetDateTime::now_utc();
|
let now = OffsetDateTime::now_utc();
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"UPDATE attempts SET state = 'executing', executing_at = $2
|
"UPDATE attempts SET state = 'executing', executing_at = $2
|
||||||
|
|
@ -640,7 +889,10 @@ async fn activate_attempt(
|
||||||
.bind(now)
|
.bind(now)
|
||||||
.execute(tx.as_mut())
|
.execute(tx.as_mut())
|
||||||
.await?;
|
.await?;
|
||||||
Ok(true)
|
Ok(Some(LeaseProjection {
|
||||||
|
expires_at_ms: live.record.expires_at,
|
||||||
|
observed_at_ms: live.now_ms,
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn lock_authority(
|
async fn lock_authority(
|
||||||
|
|
@ -692,6 +944,30 @@ async fn load_authority(
|
||||||
.transpose()
|
.transpose()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn load_historical_authority(
|
||||||
|
pool: &PgPool,
|
||||||
|
attempt_id: &OpaqueId,
|
||||||
|
) -> Result<Option<AttemptAuthority>, DurableError> {
|
||||||
|
let row = sqlx::query_as::<_, (String, String, i64, String, String)>(
|
||||||
|
"SELECT job_id, attempt_id, fence, worker_incarnation, state
|
||||||
|
FROM attempts WHERE attempt_id = $1",
|
||||||
|
)
|
||||||
|
.bind(attempt_id.as_str())
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
row.map(|(job, attempt, fence, worker, state)| {
|
||||||
|
Ok(AttemptAuthority {
|
||||||
|
job_id: OpaqueId::parse(job).map_err(DurableError::effect)?,
|
||||||
|
attempt_id: OpaqueId::parse(attempt).map_err(DurableError::effect)?,
|
||||||
|
fence: u64::try_from(fence).map_err(DurableError::effect)?,
|
||||||
|
worker_incarnation: OpaqueId::parse(worker).map_err(DurableError::effect)?,
|
||||||
|
state: AttemptState::parse(&state)
|
||||||
|
.ok_or_else(|| DurableError::Config(format!("unknown attempt state '{state}'")))?,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.transpose()
|
||||||
|
}
|
||||||
|
|
||||||
fn authority_row(row: (String, i64, String, String)) -> Result<AuthorityRow, DurableError> {
|
fn authority_row(row: (String, i64, String, String)) -> Result<AuthorityRow, DurableError> {
|
||||||
Ok(AuthorityRow {
|
Ok(AuthorityRow {
|
||||||
job_id: OpaqueId::parse(row.0).map_err(DurableError::effect)?,
|
job_id: OpaqueId::parse(row.0).map_err(DurableError::effect)?,
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
use remora_domain::OpaqueId;
|
use remora_domain::{JobPhase, OpaqueId};
|
||||||
use remora_durable::MutationDisposition;
|
use remora_durable::MutationDisposition;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
|
@ -95,12 +95,83 @@ pub enum ActivationOutcome {
|
||||||
PreconditionFailed,
|
PreconditionFailed,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
pub enum ActivationMutationOutcome {
|
||||||
|
Activated {
|
||||||
|
disposition: MutationDisposition,
|
||||||
|
lease: LeaseProjection,
|
||||||
|
},
|
||||||
|
PreconditionFailed,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub enum RenewalOutcome {
|
pub enum RenewalOutcome {
|
||||||
Renewed,
|
Renewed,
|
||||||
AuthorityLost,
|
AuthorityLost,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
pub struct LeaseProjection {
|
||||||
|
pub expires_at_ms: u64,
|
||||||
|
pub observed_at_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
#[serde(tag = "status", rename_all = "snake_case")]
|
||||||
|
pub enum RenewalMutationOutcome {
|
||||||
|
Renewed {
|
||||||
|
lease: LeaseProjection,
|
||||||
|
#[serde(skip)]
|
||||||
|
disposition: Option<MutationDisposition>,
|
||||||
|
},
|
||||||
|
AuthorityLost {
|
||||||
|
#[serde(skip)]
|
||||||
|
disposition: Option<MutationDisposition>,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
|
pub enum OutcomePublication {
|
||||||
|
Accepted {
|
||||||
|
disposition: MutationDisposition,
|
||||||
|
phase: JobPhase,
|
||||||
|
},
|
||||||
|
Rejected {
|
||||||
|
reason: crate::MutationRejection,
|
||||||
|
disposition: MutationDisposition,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
pub struct ExecutionEnvelopeRecord {
|
||||||
|
pub id: OpaqueId,
|
||||||
|
pub job_id: OpaqueId,
|
||||||
|
pub attempt_id: OpaqueId,
|
||||||
|
pub fence: u64,
|
||||||
|
pub worker_incarnation: OpaqueId,
|
||||||
|
pub source: serde_json::Value,
|
||||||
|
pub sensitive_source_ref: OpaqueId,
|
||||||
|
pub credential_id: Option<OpaqueId>,
|
||||||
|
pub credential_version_id: Option<OpaqueId>,
|
||||||
|
pub resolved_source: Option<crate::DurableDescriptor>,
|
||||||
|
pub target_store: String,
|
||||||
|
pub target_bucket: String,
|
||||||
|
pub target_prefix: Vec<String>,
|
||||||
|
pub execution_grant_id: OpaqueId,
|
||||||
|
pub checkpoint: Option<crate::CheckpointProjection>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RenewalMutationOutcome {
|
||||||
|
pub(crate) fn with_disposition(mut self, value: MutationDisposition) -> Self {
|
||||||
|
match &mut self {
|
||||||
|
Self::Renewed { disposition, .. } | Self::AuthorityLost { disposition } => {
|
||||||
|
*disposition = Some(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||||
pub enum RecoveryOutcome {
|
pub enum RecoveryOutcome {
|
||||||
StillOwned,
|
StillOwned,
|
||||||
|
|
|
||||||
|
|
@ -315,6 +315,8 @@ pub enum SafeDetailValueDto {
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct ResultDescriptorDto {
|
pub struct ResultDescriptorDto {
|
||||||
pub kind: ResultKindDto,
|
pub kind: ResultKindDto,
|
||||||
|
pub manifest_id: OpaqueId,
|
||||||
|
pub schema_version: PositiveIntegerDto,
|
||||||
pub manifest: ObjectIdentityDto,
|
pub manifest: ObjectIdentityDto,
|
||||||
pub member_count: PositiveIntegerDto,
|
pub member_count: PositiveIntegerDto,
|
||||||
pub total_bytes: SafeIntegerDto,
|
pub total_bytes: SafeIntegerDto,
|
||||||
|
|
@ -339,6 +341,7 @@ pub struct ObjectIdentityDto {
|
||||||
)]
|
)]
|
||||||
pub version_id: Option<String>,
|
pub version_id: Option<String>,
|
||||||
pub checksum: String,
|
pub checksum: String,
|
||||||
|
pub size: SafeIntegerDto,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
|
|
||||||
|
|
@ -157,6 +157,9 @@ pub struct AttemptCheckpointRequestDto {
|
||||||
pub fence: PositiveIntegerDto,
|
pub fence: PositiveIntegerDto,
|
||||||
pub worker_incarnation: OpaqueId,
|
pub worker_incarnation: OpaqueId,
|
||||||
pub checkpoint_ref: CheckpointDescriptorRefDto,
|
pub checkpoint_ref: CheckpointDescriptorRefDto,
|
||||||
|
pub resolved_source_id: OpaqueId,
|
||||||
|
pub validated_bytes: SafeIntegerDto,
|
||||||
|
pub validated_members: SafeIntegerDto,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,12 @@
|
||||||
use std::{collections::BTreeSet, fs};
|
use std::{collections::BTreeSet, fs};
|
||||||
|
|
||||||
|
use remora_contracts::{
|
||||||
|
ActivationResponseDto, AttemptActivateRequestDto, AttemptCheckpointRequestDto,
|
||||||
|
AttemptOutcomeRequestDto, AttemptProgressRequestDto, AttemptRenewRequestDto,
|
||||||
|
AttemptResolvedSourceRequestDto, ClaimRequestDto, ClaimResponseDto, JobPhaseProjectionDto,
|
||||||
|
LeaseProjectionDto,
|
||||||
|
};
|
||||||
|
use serde::{Serialize, de::DeserializeOwned};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -48,6 +55,273 @@ fn api_contract_documents_parse_and_local_references_resolve() {
|
||||||
validate_openapi_corpus(&component_schema, &corpus, "JobFailure", "job_failures");
|
validate_openapi_corpus(&component_schema, &corpus, "JobFailure", "job_failures");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_routes_and_dtos_conform_to_the_mutable_openapi_contract() {
|
||||||
|
let openapi: Value =
|
||||||
|
serde_yaml_ng::from_str(&fs::read_to_string("../../docs/api/openapi.yaml").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
let component_schema = openapi_component_schema(&openapi);
|
||||||
|
let mutation = serde_json::json!({
|
||||||
|
"operation_id": "operation_0001",
|
||||||
|
"fence": 1,
|
||||||
|
"worker_incarnation": "worker_0001"
|
||||||
|
});
|
||||||
|
let descriptor = serde_json::json!({
|
||||||
|
"id": "descriptor_0001",
|
||||||
|
"kind": "resolved_source",
|
||||||
|
"schema_version": 1,
|
||||||
|
"byte_length": 64,
|
||||||
|
"item_count": 1,
|
||||||
|
"checksum": format!("sha256:{}", "22".repeat(32))
|
||||||
|
});
|
||||||
|
let execution = serde_json::json!({
|
||||||
|
"id": "grant_0001",
|
||||||
|
"job_id": "job_00000001",
|
||||||
|
"attempt_id": "attempt_0001",
|
||||||
|
"fence": 1,
|
||||||
|
"worker_incarnation": "worker_0001",
|
||||||
|
"source": {
|
||||||
|
"kind": "http",
|
||||||
|
"display_url": "https://example.invalid/artifact",
|
||||||
|
"result_path": "artifact.bin"
|
||||||
|
},
|
||||||
|
"sensitive_source_ref": "source_00001",
|
||||||
|
"credential_version": null,
|
||||||
|
"resolved_source_ref": null,
|
||||||
|
"target": {
|
||||||
|
"store": "primary",
|
||||||
|
"bucket": "remora-results",
|
||||||
|
"prefix": "users/alice"
|
||||||
|
},
|
||||||
|
"staging_target": {
|
||||||
|
"store": "primary",
|
||||||
|
"bucket": "remora-results",
|
||||||
|
"prefix": "_remora/a/attempt_0001/"
|
||||||
|
},
|
||||||
|
"execution_grant_ref": "grant_0001",
|
||||||
|
"checkpoint_ref": null
|
||||||
|
});
|
||||||
|
let authority = serde_json::json!({
|
||||||
|
"attempt_id": "attempt_0001",
|
||||||
|
"fence": 1,
|
||||||
|
"worker_incarnation": "worker_0001"
|
||||||
|
});
|
||||||
|
let lease = serde_json::json!({
|
||||||
|
"expires_at": "2026-07-17T00:01:00Z",
|
||||||
|
"renew_after": "2026-07-17T00:00:30Z"
|
||||||
|
});
|
||||||
|
|
||||||
|
let claim_request = serde_json::json!({
|
||||||
|
"operation_id": "operation_0001",
|
||||||
|
"worker_incarnation": "worker_0001"
|
||||||
|
});
|
||||||
|
assert_dto_and_schema::<ClaimRequestDto>(&component_schema, "ClaimRequest", claim_request);
|
||||||
|
let claim_response = serde_json::json!({
|
||||||
|
"authority": authority,
|
||||||
|
"execution": execution
|
||||||
|
});
|
||||||
|
assert_dto_and_schema::<ClaimResponseDto>(
|
||||||
|
&component_schema,
|
||||||
|
"ClaimResponse",
|
||||||
|
claim_response.clone(),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<AttemptRenewRequestDto>(
|
||||||
|
&component_schema,
|
||||||
|
"AttemptRenewRequest",
|
||||||
|
mutation.clone(),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<AttemptActivateRequestDto>(
|
||||||
|
&component_schema,
|
||||||
|
"AttemptActivateRequest",
|
||||||
|
mutation.clone(),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<ActivationResponseDto>(
|
||||||
|
&component_schema,
|
||||||
|
"ActivationResponse",
|
||||||
|
serde_json::json!({
|
||||||
|
"authority": claim_response["authority"],
|
||||||
|
"execution": claim_response["execution"],
|
||||||
|
"lease": lease.clone()
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<LeaseProjectionDto>(
|
||||||
|
&component_schema,
|
||||||
|
"LeaseProjection",
|
||||||
|
lease.clone(),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<AttemptResolvedSourceRequestDto>(
|
||||||
|
&component_schema,
|
||||||
|
"AttemptResolvedSourceRequest",
|
||||||
|
merged(
|
||||||
|
&mutation,
|
||||||
|
serde_json::json!({ "resolved_source_ref": descriptor.clone() }),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let mut checkpoint_descriptor = descriptor.clone();
|
||||||
|
checkpoint_descriptor["kind"] = Value::String("checkpoint".into());
|
||||||
|
assert_dto_and_schema::<AttemptCheckpointRequestDto>(
|
||||||
|
&component_schema,
|
||||||
|
"AttemptCheckpointRequest",
|
||||||
|
merged(
|
||||||
|
&mutation,
|
||||||
|
serde_json::json!({
|
||||||
|
"checkpoint_ref": checkpoint_descriptor,
|
||||||
|
"resolved_source_id": "descriptor_0001",
|
||||||
|
"validated_bytes": 1024,
|
||||||
|
"validated_members": 1
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<AttemptProgressRequestDto>(
|
||||||
|
&component_schema,
|
||||||
|
"AttemptProgressRequest",
|
||||||
|
merged(
|
||||||
|
&mutation,
|
||||||
|
serde_json::json!({
|
||||||
|
"progress": {
|
||||||
|
"completed_bytes": 1024,
|
||||||
|
"total_bytes": 4096,
|
||||||
|
"completed_members": 1,
|
||||||
|
"total_members": 1
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<AttemptOutcomeRequestDto>(
|
||||||
|
&component_schema,
|
||||||
|
"AttemptOutcomeRequest",
|
||||||
|
merged(
|
||||||
|
&mutation,
|
||||||
|
serde_json::json!({
|
||||||
|
"outcome": {
|
||||||
|
"kind": "failure",
|
||||||
|
"failure": {
|
||||||
|
"class": "transient_source",
|
||||||
|
"message": "temporarily unavailable"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert_dto_and_schema::<JobPhaseProjectionDto>(
|
||||||
|
&component_schema,
|
||||||
|
"JobPhaseProjection",
|
||||||
|
serde_json::json!({ "phase": "RetryWait" }),
|
||||||
|
);
|
||||||
|
|
||||||
|
let routes = [
|
||||||
|
(
|
||||||
|
"/internal/v1/jobs/{job_id}/claim",
|
||||||
|
"ClaimRequest",
|
||||||
|
"200",
|
||||||
|
Some("ClaimResponse"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/internal/v1/attempts/{attempt_id}/renew",
|
||||||
|
"AttemptRenewRequest",
|
||||||
|
"200",
|
||||||
|
Some("LeaseProjection"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/internal/v1/attempts/{attempt_id}/activate",
|
||||||
|
"AttemptActivateRequest",
|
||||||
|
"200",
|
||||||
|
Some("ActivationResponse"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/internal/v1/attempts/{attempt_id}/resolved-source",
|
||||||
|
"AttemptResolvedSourceRequest",
|
||||||
|
"204",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/internal/v1/attempts/{attempt_id}/checkpoint",
|
||||||
|
"AttemptCheckpointRequest",
|
||||||
|
"204",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/internal/v1/attempts/{attempt_id}/progress",
|
||||||
|
"AttemptProgressRequest",
|
||||||
|
"204",
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"/internal/v1/attempts/{attempt_id}/outcome",
|
||||||
|
"AttemptOutcomeRequest",
|
||||||
|
"200",
|
||||||
|
Some("JobPhaseProjection"),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
for (path, request, status, response) in routes {
|
||||||
|
assert_worker_route(&openapi, path, request, status, response);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_dto_and_schema<T>(component_schema: &Value, definition: &str, value: Value)
|
||||||
|
where
|
||||||
|
T: DeserializeOwned + Serialize,
|
||||||
|
{
|
||||||
|
let dto: T = serde_json::from_value(value.clone()).unwrap();
|
||||||
|
assert_eq!(serde_json::to_value(dto).unwrap(), value);
|
||||||
|
validate_openapi_value(component_schema, definition, &value);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_openapi_value(component_schema: &Value, definition: &str, value: &Value) {
|
||||||
|
let mut schema = component_schema.clone();
|
||||||
|
schema.as_object_mut().unwrap().insert(
|
||||||
|
"$ref".to_owned(),
|
||||||
|
Value::String(format!("#/$defs/{definition}")),
|
||||||
|
);
|
||||||
|
jsonschema::draft202012::options()
|
||||||
|
.should_validate_formats(true)
|
||||||
|
.build(&schema)
|
||||||
|
.unwrap()
|
||||||
|
.validate(value)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn merged(base: &Value, extension: Value) -> Value {
|
||||||
|
let mut value = base.clone();
|
||||||
|
value
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.extend(extension.as_object().unwrap().clone());
|
||||||
|
value
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_worker_route(
|
||||||
|
openapi: &Value,
|
||||||
|
path: &str,
|
||||||
|
request_schema: &str,
|
||||||
|
success_status: &str,
|
||||||
|
response_schema: Option<&str>,
|
||||||
|
) {
|
||||||
|
let operation = &openapi["paths"][path]["post"];
|
||||||
|
assert_eq!(
|
||||||
|
operation.pointer("/security/0/serviceAuth"),
|
||||||
|
Some(&serde_json::json!([]))
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
operation.pointer("/requestBody/content/application~1json/schema/$ref"),
|
||||||
|
Some(&Value::String(format!(
|
||||||
|
"#/components/schemas/{request_schema}"
|
||||||
|
)))
|
||||||
|
);
|
||||||
|
assert!(operation["responses"].get("409").is_some());
|
||||||
|
assert!(operation["responses"].get("default").is_some());
|
||||||
|
let success = &operation["responses"][success_status];
|
||||||
|
match response_schema {
|
||||||
|
Some(response_schema) => assert_eq!(
|
||||||
|
success.pointer("/content/application~1json/schema/$ref"),
|
||||||
|
Some(&Value::String(format!(
|
||||||
|
"#/components/schemas/{response_schema}"
|
||||||
|
)))
|
||||||
|
),
|
||||||
|
None => assert!(success.get("content").is_none()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn openapi_component_schema(openapi: &Value) -> Value {
|
fn openapi_component_schema(openapi: &Value) -> Value {
|
||||||
let mut definitions = openapi
|
let mut definitions = openapi
|
||||||
.pointer("/components/schemas")
|
.pointer("/components/schemas")
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ const PATH_SET: &AsciiSet = &CONTROLS
|
||||||
.add(b'}');
|
.add(b'}');
|
||||||
|
|
||||||
/// One version or delete marker listed under a key/prefix.
|
/// One version or delete marker listed under a key/prefix.
|
||||||
|
#[cfg(feature = "test-utils")]
|
||||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||||
pub struct ObjectVersion {
|
pub struct ObjectVersion {
|
||||||
pub key: String,
|
pub key: String,
|
||||||
|
|
@ -81,6 +82,7 @@ impl MultipartClient {
|
||||||
/// Enable bucket versioning when the peer supports it.
|
/// Enable bucket versioning when the peer supports it.
|
||||||
///
|
///
|
||||||
/// Peers that return NotImplemented are reported as `false` without error.
|
/// Peers that return NotImplemented are reported as `false` without error.
|
||||||
|
#[cfg(feature = "test-utils")]
|
||||||
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
|
pub async fn enable_bucket_versioning(&self) -> Result<bool, TargetError> {
|
||||||
let body = r#"<?xml version="1.0" encoding="UTF-8"?>
|
let body = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||||
|
|
@ -341,6 +343,7 @@ impl MultipartClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// List object versions and delete markers under an optional key prefix.
|
/// List object versions and delete markers under an optional key prefix.
|
||||||
|
#[cfg(feature = "test-utils")]
|
||||||
pub async fn list_object_versions(
|
pub async fn list_object_versions(
|
||||||
&self,
|
&self,
|
||||||
prefix: &str,
|
prefix: &str,
|
||||||
|
|
@ -410,6 +413,7 @@ impl MultipartClient {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete one exact object version or delete marker.
|
/// Delete one exact object version or delete marker.
|
||||||
|
#[cfg(feature = "test-utils")]
|
||||||
pub async fn delete_object_version(
|
pub async fn delete_object_version(
|
||||||
&self,
|
&self,
|
||||||
key: &str,
|
key: &str,
|
||||||
|
|
@ -556,6 +560,7 @@ struct CompletePart {
|
||||||
etag: String,
|
etag: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "test-utils")]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct ListVersionsResult {
|
struct ListVersionsResult {
|
||||||
#[serde(rename = "Version", default)]
|
#[serde(rename = "Version", default)]
|
||||||
|
|
@ -570,6 +575,7 @@ struct ListVersionsResult {
|
||||||
next_version_id_marker: Option<String>,
|
next_version_id_marker: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "test-utils")]
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct ListedVersion {
|
struct ListedVersion {
|
||||||
#[serde(rename = "Key", default)]
|
#[serde(rename = "Key", default)]
|
||||||
|
|
|
||||||
|
|
@ -551,7 +551,7 @@ paths:
|
||||||
content:
|
content:
|
||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
$ref: '#/components/schemas/Job'
|
$ref: '#/components/schemas/JobPhaseProjection'
|
||||||
'409':
|
'409':
|
||||||
$ref: '#/components/responses/PreconditionFailed'
|
$ref: '#/components/responses/PreconditionFailed'
|
||||||
default:
|
default:
|
||||||
|
|
@ -1109,6 +1109,13 @@ components:
|
||||||
JobPhase:
|
JobPhase:
|
||||||
type: string
|
type: string
|
||||||
enum: [Queued, Running, RetryWait, Cancelling, Succeeded, Failed, Cancelled]
|
enum: [Queued, Running, RetryWait, Cancelling, Succeeded, Failed, Cancelled]
|
||||||
|
JobPhaseProjection:
|
||||||
|
type: object
|
||||||
|
additionalProperties: false
|
||||||
|
required: [phase]
|
||||||
|
properties:
|
||||||
|
phase:
|
||||||
|
$ref: '#/components/schemas/JobPhase'
|
||||||
TerminalJobPhase:
|
TerminalJobPhase:
|
||||||
type: string
|
type: string
|
||||||
enum: [Succeeded, Failed, Cancelled]
|
enum: [Succeeded, Failed, Cancelled]
|
||||||
|
|
@ -1247,10 +1254,15 @@ components:
|
||||||
ResultDescriptor:
|
ResultDescriptor:
|
||||||
type: object
|
type: object
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
required: [kind, manifest, member_count, total_bytes]
|
required: [kind, manifest_id, schema_version, manifest, member_count, total_bytes]
|
||||||
properties:
|
properties:
|
||||||
kind:
|
kind:
|
||||||
const: manifest
|
const: manifest
|
||||||
|
manifest_id:
|
||||||
|
$ref: '#/components/schemas/OpaqueId'
|
||||||
|
schema_version:
|
||||||
|
type: integer
|
||||||
|
minimum: 1
|
||||||
manifest:
|
manifest:
|
||||||
$ref: '#/components/schemas/ObjectIdentity'
|
$ref: '#/components/schemas/ObjectIdentity'
|
||||||
member_count:
|
member_count:
|
||||||
|
|
@ -1262,7 +1274,7 @@ components:
|
||||||
ObjectIdentity:
|
ObjectIdentity:
|
||||||
type: object
|
type: object
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
required: [store, bucket, key, checksum]
|
required: [store, bucket, key, checksum, size]
|
||||||
properties:
|
properties:
|
||||||
store:
|
store:
|
||||||
type: string
|
type: string
|
||||||
|
|
@ -1274,6 +1286,9 @@ components:
|
||||||
type: string
|
type: string
|
||||||
checksum:
|
checksum:
|
||||||
$ref: '#/components/schemas/Sha256Digest'
|
$ref: '#/components/schemas/Sha256Digest'
|
||||||
|
size:
|
||||||
|
type: integer
|
||||||
|
minimum: 0
|
||||||
Sha256Digest:
|
Sha256Digest:
|
||||||
type: string
|
type: string
|
||||||
pattern: '^sha256:[a-f0-9]{64}$'
|
pattern: '^sha256:[a-f0-9]{64}$'
|
||||||
|
|
@ -1657,7 +1672,7 @@ components:
|
||||||
allOf:
|
allOf:
|
||||||
- $ref: '#/components/schemas/AttemptMutation'
|
- $ref: '#/components/schemas/AttemptMutation'
|
||||||
- type: object
|
- type: object
|
||||||
required: [checkpoint_ref]
|
required: [checkpoint_ref, resolved_source_id, validated_bytes, validated_members]
|
||||||
properties:
|
properties:
|
||||||
checkpoint_ref:
|
checkpoint_ref:
|
||||||
allOf:
|
allOf:
|
||||||
|
|
@ -1666,6 +1681,14 @@ components:
|
||||||
properties:
|
properties:
|
||||||
kind:
|
kind:
|
||||||
const: checkpoint
|
const: checkpoint
|
||||||
|
resolved_source_id:
|
||||||
|
$ref: '#/components/schemas/OpaqueId'
|
||||||
|
validated_bytes:
|
||||||
|
type: integer
|
||||||
|
minimum: 0
|
||||||
|
validated_members:
|
||||||
|
type: integer
|
||||||
|
minimum: 0
|
||||||
AttemptProgressRequest:
|
AttemptProgressRequest:
|
||||||
unevaluatedProperties: false
|
unevaluatedProperties: false
|
||||||
allOf:
|
allOf:
|
||||||
|
|
|
||||||
50
tests/kind/run-worker-api-job.sh
Executable file
50
tests/kind/run-worker-api-job.sh
Executable file
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||||
|
source "$ROOT/tests/kind/lib.sh"
|
||||||
|
|
||||||
|
USER_KUBECONFIG_BEFORE="$(user_kubeconfig_digest)"
|
||||||
|
SUCCEEDED=false
|
||||||
|
OWNS_FIXTURE=false
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
local status=$?
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
if [[ "${REMORA_TESTBED_RETAIN:-0}" = 1 ]] && [[ "$SUCCEEDED" != true ]]; then
|
||||||
|
printf 'retained kubeconfig: %s\n' "$KUBECONFIG_PATH"
|
||||||
|
printf 'retained context: %s\n' "$CONTEXT"
|
||||||
|
exit "$status"
|
||||||
|
fi
|
||||||
|
if [[ "$OWNS_FIXTURE" = true ]]; then
|
||||||
|
destroy_fixture || status=1
|
||||||
|
fi
|
||||||
|
if [[ "$(user_kubeconfig_digest)" != "$USER_KUBECONFIG_BEFORE" ]]; then
|
||||||
|
printf 'user kubeconfig changed unexpectedly\n' >&2
|
||||||
|
status=1
|
||||||
|
fi
|
||||||
|
exit "$status"
|
||||||
|
}
|
||||||
|
|
||||||
|
trap cleanup EXIT
|
||||||
|
trap 'exit 130' INT
|
||||||
|
trap 'exit 143' TERM
|
||||||
|
|
||||||
|
acquire_fixture_lock
|
||||||
|
REMORA_REAL_NERDCTL="$(command -v nerdctl)"
|
||||||
|
export REMORA_REAL_NERDCTL
|
||||||
|
"$REMORA_REAL_NERDCTL" --namespace default build \
|
||||||
|
--file "$ROOT/Dockerfile.test" \
|
||||||
|
--target kind-worker-api-runner \
|
||||||
|
--tag remora-kind-worker-api:local \
|
||||||
|
"$ROOT"
|
||||||
|
OWNS_FIXTURE=true
|
||||||
|
setup_fixture durable coordination
|
||||||
|
load_local_image remora-kind-worker-api:local
|
||||||
|
run_job_matrix \
|
||||||
|
worker-api \
|
||||||
|
"$TESTBED/worker-api-job.yaml.in" \
|
||||||
|
remora-kind-worker-api:local \
|
||||||
|
durable coordination
|
||||||
|
SUCCEEDED=true
|
||||||
|
printf 'worker API package test Jobs passed\n'
|
||||||
34
tests/kind/worker-api-job.yaml.in
Normal file
34
tests/kind/worker-api-job.yaml.in
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
apiVersion: batch/v1
|
||||||
|
kind: Job
|
||||||
|
metadata:
|
||||||
|
name: @@JOB_NAME@@
|
||||||
|
namespace: remora-testbed
|
||||||
|
spec:
|
||||||
|
activeDeadlineSeconds: 600
|
||||||
|
backoffLimit: 0
|
||||||
|
ttlSecondsAfterFinished: 300
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
automountServiceAccountToken: false
|
||||||
|
enableServiceLinks: false
|
||||||
|
restartPolicy: Never
|
||||||
|
containers:
|
||||||
|
- name: tests
|
||||||
|
image: @@TEST_IMAGE@@
|
||||||
|
imagePullPolicy: Never
|
||||||
|
envFrom:
|
||||||
|
- configMapRef: {name: @@PROFILE_NAME@@}
|
||||||
|
env:
|
||||||
|
- name: REMORA_WORKER_API_REQUIRE
|
||||||
|
value: "1"
|
||||||
|
volumeMounts:
|
||||||
|
- {name: fixture-profile, mountPath: /run/remora-fixture, readOnly: true}
|
||||||
|
resources:
|
||||||
|
requests: {cpu: 250m, memory: 256Mi}
|
||||||
|
limits: {cpu: "2", memory: 1Gi}
|
||||||
|
volumes:
|
||||||
|
- name: fixture-profile
|
||||||
|
configMap:
|
||||||
|
name: @@PROFILE_NAME@@-file
|
||||||
|
items:
|
||||||
|
- {key: profile.env, path: profile.env}
|
||||||
Loading…
Reference in a new issue