Implement fail-closed ignored workspace routing
This commit is contained in:
parent
ba6a075839
commit
95104fa4e7
21 changed files with 6024 additions and 6 deletions
1
.cargo/config.toml
Normal file
1
.cargo/config.toml
Normal file
|
|
@ -0,0 +1 @@
|
|||
paths = [".lazaret/vendor/fuser-0.17.0"]
|
||||
316
Cargo.lock
generated
316
Cargo.lock
generated
|
|
@ -152,6 +152,17 @@ dependencies = [
|
|||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.20.3"
|
||||
|
|
@ -614,6 +625,26 @@ version = "1.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fuser"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80a5eca878900c2e39e9e52fd797954b7fc39eeefc8558257114bfea6a698fcf"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"libc",
|
||||
"log",
|
||||
"memchr",
|
||||
"nix 0.30.1",
|
||||
"num_enum",
|
||||
"page_size",
|
||||
"parking_lot",
|
||||
"pkg-config",
|
||||
"ref-cast",
|
||||
"smallvec",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.32"
|
||||
|
|
@ -722,6 +753,68 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gix-features"
|
||||
version = "0.48.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc"
|
||||
dependencies = [
|
||||
"gix-trace",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gix-glob"
|
||||
version = "0.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bstr",
|
||||
"gix-features",
|
||||
"gix-path",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gix-ignore"
|
||||
version = "0.21.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d"
|
||||
dependencies = [
|
||||
"bstr",
|
||||
"gix-glob",
|
||||
"gix-path",
|
||||
"gix-trace",
|
||||
"unicode-bom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gix-path"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6"
|
||||
dependencies = [
|
||||
"bstr",
|
||||
"gix-trace",
|
||||
"gix-validate",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gix-trace"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678"
|
||||
|
||||
[[package]]
|
||||
name = "gix-validate"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3"
|
||||
dependencies = [
|
||||
"bstr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.17.1"
|
||||
|
|
@ -1166,6 +1259,20 @@ dependencies = [
|
|||
"x509-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazaret-ignored-workspace"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"bstr",
|
||||
"fuser",
|
||||
"gix-ignore",
|
||||
"libc",
|
||||
"nix 0.31.3",
|
||||
"rustix",
|
||||
"serde_json",
|
||||
"sha2 0.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazaret-linux"
|
||||
version = "0.0.0"
|
||||
|
|
@ -1261,6 +1368,15 @@ version = "0.8.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.33"
|
||||
|
|
@ -1279,6 +1395,15 @@ version = "2.8.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
version = "0.2.1"
|
||||
|
|
@ -1306,6 +1431,31 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"memoffset",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.31.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
|
|
@ -1350,6 +1500,28 @@ dependencies = [
|
|||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
|
||||
dependencies = [
|
||||
"num_enum_derive",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_enum_derive"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oci-client"
|
||||
version = "0.17.0"
|
||||
|
|
@ -1443,6 +1615,39 @@ version = "0.2.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||
|
||||
[[package]]
|
||||
name = "page_size"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
|
|
@ -1498,6 +1703,15 @@ version = "0.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-crate"
|
||||
version = "3.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
|
||||
dependencies = [
|
||||
"toml_edit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
|
|
@ -1636,6 +1850,35 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.0"
|
||||
|
|
@ -1862,6 +2105,12 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "3.7.0"
|
||||
|
|
@ -2100,9 +2349,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
|||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.118"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2302,6 +2551,18 @@ dependencies = [
|
|||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.25.12+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"toml_datetime",
|
||||
"toml_parser",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
|
|
@ -2412,6 +2673,12 @@ version = "2.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bom"
|
||||
version = "2.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
|
|
@ -2591,6 +2858,22 @@ dependencies = [
|
|||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
|
|
@ -2600,6 +2883,12 @@ dependencies = [
|
|||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
|
|
@ -2746,6 +3035,9 @@ name = "winnow"
|
|||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
|
|
@ -2814,6 +3106,26 @@ dependencies = [
|
|||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.8"
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ members = [
|
|||
"crates/lazaret-core",
|
||||
"crates/lazaret-escalation-client",
|
||||
"crates/lazaret-host-coordinator",
|
||||
"crates/lazaret-ignored-workspace",
|
||||
"crates/lazaret-linux",
|
||||
"crates/lazaret-network-sidecar",
|
||||
"crates/lazaret-prepared-lower",
|
||||
|
|
@ -12,6 +13,7 @@ members = [
|
|||
"crates/lazaret-storage-catalog",
|
||||
]
|
||||
resolver = "3"
|
||||
exclude = [".lazaret/vendor/fuser-0.17.0"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.0.0"
|
||||
|
|
@ -21,14 +23,19 @@ license = "Apache-2.0"
|
|||
|
||||
[workspace.dependencies]
|
||||
anyhow = "=1.0.103"
|
||||
bstr = "=1.13.0"
|
||||
ed25519-dalek = { version = "=2.2.0", default-features = false, features = ["std"] }
|
||||
flate2 = "=1.1.9"
|
||||
fuser = { version = "=0.17.0", default-features = false, features = ["abi-7-31"] }
|
||||
gix-ignore = "=0.21.1"
|
||||
getrandom = "=0.4.3"
|
||||
oci-client = { version = "=0.17.0", default-features = false, features = ["rustls-tls"] }
|
||||
oci-spec = { version = "=0.10.0", default-features = false, features = ["runtime"] }
|
||||
postcard = { version = "=1.1.3", default-features = false, features = ["use-std"] }
|
||||
rcgen = { version = "=0.14.8", default-features = false, features = ["aws_lc_rs", "x509-parser"] }
|
||||
redb = "=4.1.0"
|
||||
libc = "=0.2.186"
|
||||
nix = { version = "=0.31.3", features = ["fs"] }
|
||||
rustls = { version = "=0.23.42", default-features = false, features = ["aws_lc_rs", "std"] }
|
||||
serde = { version = "=1.0.228", features = ["derive"] }
|
||||
serde_json = "=1.0.150"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use std::fs::{self, File, OpenOptions};
|
|||
use std::net::{Ipv4Addr, SocketAddrV4, TcpListener, TcpStream, ToSocketAddrs};
|
||||
use std::os::linux::net::SocketAddrExt;
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::os::unix::fs::symlink;
|
||||
use std::os::unix::net::{SocketAddr, UnixListener, UnixStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
|
|
@ -46,6 +47,10 @@ const SKIP: u8 = 77;
|
|||
const HOST_COORDINATOR_DRIVER: &str = "--runtime-composition-host-coordinator";
|
||||
const SANDBOX_NETWORK_PROBE: &str = "--sandbox-network-probe";
|
||||
const NETWORK_MATRIX: &str = "--network-matrix";
|
||||
const BENCHMARK_DRIVER: &str = "--runtime-composition-benchmark-command";
|
||||
const BENCHMARK_GUEST: &str = "--runtime-composition-benchmark-guest";
|
||||
const BENCHMARK_COMPLETE: &str = "benchmark-complete";
|
||||
const BENCHMARK_ERROR: &str = "benchmark-error";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
|
||||
|
|
@ -55,6 +60,12 @@ fn main() -> ExitCode {
|
|||
if let Some(result) = dispatch_private_runtime_role(&arguments) {
|
||||
return exit(result);
|
||||
}
|
||||
if arguments.first().and_then(|argument| argument.to_str()) == Some(BENCHMARK_GUEST) {
|
||||
return exit(run_benchmark_guest(&arguments[1..]));
|
||||
}
|
||||
if arguments.first().and_then(|argument| argument.to_str()) == Some(BENCHMARK_DRIVER) {
|
||||
return exit(run_benchmark_driver(&arguments[1..]));
|
||||
}
|
||||
if arguments.first().and_then(|argument| argument.to_str()) == Some(HOST_COORDINATOR_DRIVER) {
|
||||
return exit(run_host_coordinator_driver(&arguments[1..]));
|
||||
}
|
||||
|
|
@ -93,6 +104,89 @@ fn main() -> ExitCode {
|
|||
}
|
||||
}
|
||||
|
||||
fn run_benchmark_driver(arguments: &[std::ffi::OsString]) -> std::io::Result<()> {
|
||||
let separator = arguments
|
||||
.iter()
|
||||
.position(|argument| argument == "--")
|
||||
.ok_or_else(|| std::io::Error::other("benchmark command separator is missing"))?;
|
||||
if separator != 8 || arguments.len() == separator + 1 {
|
||||
return Err(std::io::Error::other(
|
||||
"benchmark composition arguments are malformed",
|
||||
));
|
||||
}
|
||||
let paths = arguments[..separator]
|
||||
.iter()
|
||||
.map(PathBuf::from)
|
||||
.collect::<Vec<_>>();
|
||||
let fixture = Fixture::build_benchmark(
|
||||
&paths[0],
|
||||
&paths[1],
|
||||
&paths[2],
|
||||
&paths[3],
|
||||
&paths[4],
|
||||
&paths[5],
|
||||
&paths[6],
|
||||
&paths[7],
|
||||
&arguments[separator + 1..],
|
||||
)
|
||||
.map_err(matrix_io)?;
|
||||
let coordinator = SandboxCoordinator::new(&fixture.store);
|
||||
let runtime = fixture.launch_committed().map_err(matrix_io)?;
|
||||
let results = fixture
|
||||
.benchmark_results
|
||||
.as_ref()
|
||||
.ok_or_else(|| std::io::Error::other("benchmark composition result authority is absent"))?;
|
||||
let complete = results.join(BENCHMARK_COMPLETE);
|
||||
let error = results.join(BENCHMARK_ERROR);
|
||||
for _ in 0..24_000 {
|
||||
if complete.is_file() || error.is_file() {
|
||||
break;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(5));
|
||||
}
|
||||
let outcome = if complete.is_file() {
|
||||
Ok(())
|
||||
} else if error.is_file() {
|
||||
Err(std::io::Error::other(fs::read_to_string(&error)?))
|
||||
} else {
|
||||
Err(std::io::Error::other("benchmark command timed out"))
|
||||
};
|
||||
runtime
|
||||
.stop(&coordinator)
|
||||
.map_err(|error| std::io::Error::other(error.to_string()))?;
|
||||
outcome
|
||||
}
|
||||
|
||||
#[allow(clippy::disallowed_methods, clippy::disallowed_types)]
|
||||
fn run_benchmark_guest(arguments: &[std::ffi::OsString]) -> std::io::Result<()> {
|
||||
if arguments.len() < 3 {
|
||||
return Err(std::io::Error::other(
|
||||
"benchmark guest arguments are malformed",
|
||||
));
|
||||
}
|
||||
let results = PathBuf::from(&arguments[0]);
|
||||
let output = std::process::Command::new(&arguments[1])
|
||||
.args(&arguments[2..])
|
||||
.output();
|
||||
match output {
|
||||
Ok(output) if output.status.success() => {
|
||||
fs::write(results.join(BENCHMARK_COMPLETE), b"ok")?;
|
||||
}
|
||||
Ok(output) => fs::write(
|
||||
results.join(BENCHMARK_ERROR),
|
||||
format!(
|
||||
"benchmark command exited with {}: {}",
|
||||
output.status,
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
),
|
||||
)?,
|
||||
Err(error) => fs::write(results.join(BENCHMARK_ERROR), error.to_string())?,
|
||||
}
|
||||
loop {
|
||||
thread::park();
|
||||
}
|
||||
}
|
||||
|
||||
fn exit(result: std::io::Result<()>) -> ExitCode {
|
||||
match result {
|
||||
Ok(()) => ExitCode::SUCCESS,
|
||||
|
|
@ -412,6 +506,7 @@ struct Fixture {
|
|||
store: HostCoordinatorDomainStore,
|
||||
bundle: SandboxBundle,
|
||||
activity: PathBuf,
|
||||
benchmark_results: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
|
|
@ -425,6 +520,111 @@ impl Fixture {
|
|||
Self::build(root, None)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn build_benchmark(
|
||||
root: &Path,
|
||||
workspace: &Path,
|
||||
host_root: &Path,
|
||||
output: &Path,
|
||||
repository: &Path,
|
||||
cargo_home: &Path,
|
||||
rustup_home: &Path,
|
||||
uv_bin: &Path,
|
||||
sample_arguments: &[std::ffi::OsString],
|
||||
) -> Result<Self, MatrixError> {
|
||||
let configuration_directory = root.join("configuration");
|
||||
let runtime = root.join("runtime");
|
||||
for directory in [&configuration_directory, &runtime] {
|
||||
fs::create_dir_all(directory)?;
|
||||
}
|
||||
let configuration_path = configuration_directory.join("lazaret.toml");
|
||||
let paths = [
|
||||
root,
|
||||
workspace,
|
||||
host_root,
|
||||
output,
|
||||
repository,
|
||||
cargo_home,
|
||||
rustup_home,
|
||||
uv_bin,
|
||||
];
|
||||
if paths.iter().any(|path| {
|
||||
path.to_str()
|
||||
.is_none_or(|value| value.contains(['\n', '\r', '"']))
|
||||
}) {
|
||||
return Err(failure("benchmark composition path is not TOML-safe"));
|
||||
}
|
||||
fs::write(
|
||||
&configuration_path,
|
||||
format!(
|
||||
"schema = \"lazaret-trusted-host-config-v1\"\n\n[storage]\ndata_root = \"{}\"\n\n[profiles.default]\nimage = \"example.invalid/runtime-fixture:v1\"\npersistent_dirs = [\"/tooling\", \"/fixtures\"]\nenv = {{ PATH = \"env:BENCHMARK_PATH\", CARGO_HOME = \"env:BENCHMARK_CARGO_HOME\", RUSTUP_HOME = \"env:BENCHMARK_RUSTUP_HOME\", CARGO_BUILD_JOBS = \"env:BENCHMARK_CARGO_BUILD_JOBS\" }}\nvolumes = [\n {{ host = \"{}\", container = \"/tooling/usr\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/host\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/fixtures/output\", kind = \"directory\", access = \"read-write\" }},\n {{ host = \"{}\", container = \"/fixtures/repository\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/cargo\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/rustup\", kind = \"directory\", access = \"read-only\" }},\n {{ host = \"{}\", container = \"/tooling/uv-bin\", kind = \"directory\", access = \"read-only\" }},\n]\n",
|
||||
root.join("data").display(),
|
||||
Path::new("/usr").display(),
|
||||
host_root.display(),
|
||||
output.display(),
|
||||
repository.display(),
|
||||
cargo_home.display(),
|
||||
rustup_home.display(),
|
||||
uv_bin.display(),
|
||||
),
|
||||
)?;
|
||||
let environment = BTreeMap::from([
|
||||
(
|
||||
"BENCHMARK_PATH".into(),
|
||||
"/tooling/cargo/bin:/tooling/uv-bin:/usr/bin:/bin".into(),
|
||||
),
|
||||
("BENCHMARK_CARGO_HOME".into(), "/tooling/cargo".into()),
|
||||
("BENCHMARK_RUSTUP_HOME".into(), "/tooling/rustup".into()),
|
||||
("BENCHMARK_CARGO_BUILD_JOBS".into(), "2".into()),
|
||||
]);
|
||||
let configuration = load_trusted_configuration(ConfigurationLoadRequest {
|
||||
explicit_configuration_path: Some(configuration_path),
|
||||
selected_profile: "default".into(),
|
||||
workspace: workspace.to_path_buf(),
|
||||
home: root.join("home"),
|
||||
xdg_config_home: None,
|
||||
xdg_data_home: None,
|
||||
xdg_runtime_dir: runtime,
|
||||
environment,
|
||||
sandbox_layout: production_sandbox_layout().map_err(failure)?,
|
||||
})
|
||||
.map_err(failure)?;
|
||||
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
|
||||
let results = output.to_path_buf();
|
||||
stage_benchmark_rootfs_and_control(&runtime_domain)?;
|
||||
let mut guest_arguments = vec![
|
||||
BENCHMARK_GUEST.to_owned(),
|
||||
"/fixtures/output".to_owned(),
|
||||
"/usr/bin/python3".to_owned(),
|
||||
"/fixtures/repository/tools/ignored_workspace_benchmark.py".to_owned(),
|
||||
"sample".to_owned(),
|
||||
];
|
||||
guest_arguments.extend(
|
||||
sample_arguments
|
||||
.iter()
|
||||
.map(|argument| argument.to_string_lossy().into_owned()),
|
||||
);
|
||||
let root_process =
|
||||
RootProcess::new("/bin/runtime-composition", guest_arguments).map_err(failure)?;
|
||||
let namespace =
|
||||
lazaret_linux::filesystem::namespace_identity(Path::new(NETWORK_NAMESPACE_PATH))?;
|
||||
let bundle = build_sandbox_bundle(SandboxBuildRequest {
|
||||
configuration: &configuration,
|
||||
container_id: ContainerId::new([6; 32]),
|
||||
expected_network_namespace: namespace,
|
||||
root_process,
|
||||
terminal: None,
|
||||
})
|
||||
.map_err(failure)?;
|
||||
Self::finish_build(
|
||||
None,
|
||||
configuration,
|
||||
bundle,
|
||||
workspace.join("activity"),
|
||||
Some(results),
|
||||
)
|
||||
}
|
||||
|
||||
fn build(root: &Path, tree: Option<TempDir>) -> Result<Self, MatrixError> {
|
||||
let configuration_directory = root.join("configuration");
|
||||
let workspace = root.join("workspace");
|
||||
|
|
@ -473,6 +673,24 @@ impl Fixture {
|
|||
terminal: None,
|
||||
})
|
||||
.map_err(failure)?;
|
||||
Self::finish_build(
|
||||
tree,
|
||||
configuration,
|
||||
bundle,
|
||||
workspace.join("activity"),
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
fn finish_build(
|
||||
tree: Option<TempDir>,
|
||||
configuration: ResolvedDomainConfiguration,
|
||||
bundle: SandboxBundle,
|
||||
activity: PathBuf,
|
||||
benchmark_results: Option<PathBuf>,
|
||||
) -> Result<Self, MatrixError> {
|
||||
let runtime_domain = host_path(configuration.runtime_domain_root().canonical_label());
|
||||
let container_id = ContainerId::new([6; 32]);
|
||||
let bundle_path = runtime_domain
|
||||
.join("bundles")
|
||||
.join(hex(container_id.as_bytes()));
|
||||
|
|
@ -500,8 +718,7 @@ impl Fixture {
|
|||
durable_decision: DurableDecision::None,
|
||||
})])
|
||||
.map_err(failure)?;
|
||||
let coordinator = SandboxCoordinator::new(&store);
|
||||
coordinator
|
||||
SandboxCoordinator::new(&store)
|
||||
.begin_intent(
|
||||
GenerationId::new(7),
|
||||
container_id,
|
||||
|
|
@ -515,7 +732,8 @@ impl Fixture {
|
|||
configuration,
|
||||
store,
|
||||
bundle,
|
||||
activity: workspace.join("activity"),
|
||||
activity,
|
||||
benchmark_results,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -864,6 +1082,37 @@ fn stage_rootfs_and_control(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn stage_benchmark_rootfs_and_control(runtime_domain: &Path) -> Result<(), MatrixError> {
|
||||
let rootfs = runtime_domain.join("state-anchor/merged");
|
||||
for directory in [
|
||||
"bin",
|
||||
"tooling/usr",
|
||||
"tooling/cargo",
|
||||
"tooling/rustup",
|
||||
"tooling/uv-bin",
|
||||
"fixtures/host",
|
||||
"fixtures/output",
|
||||
"fixtures/repository",
|
||||
] {
|
||||
fs::create_dir_all(rootfs.join(directory))?;
|
||||
}
|
||||
fs::copy(
|
||||
std::env::current_exe()?,
|
||||
rootfs.join("bin/runtime-composition"),
|
||||
)?;
|
||||
symlink("tooling/usr", rootfs.join("usr"))?;
|
||||
symlink("usr/lib", rootfs.join("lib"))?;
|
||||
symlink("usr/lib64", rootfs.join("lib64"))?;
|
||||
symlink("usr/sbin", rootfs.join("sbin"))?;
|
||||
|
||||
let control = runtime_domain.join("sandbox-control").join(hex(&[6; 32]));
|
||||
for directory in ["credentials", "mailbox/requests", "mailbox/results", "bin"] {
|
||||
fs::create_dir_all(control.join(directory))?;
|
||||
}
|
||||
fs::copy("/bin/true", control.join("bin/lazaret-escalate"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn sandbox_participant(store: &HostCoordinatorDomainStore) -> lazaret_core::records::Participant {
|
||||
store
|
||||
.snapshot()
|
||||
|
|
|
|||
21
crates/lazaret-ignored-workspace/Cargo.toml
Normal file
21
crates/lazaret-ignored-workspace/Cargo.toml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
[package]
|
||||
name = "lazaret-ignored-workspace"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
bstr.workspace = true
|
||||
fuser.workspace = true
|
||||
gix-ignore.workspace = true
|
||||
libc.workspace = true
|
||||
nix.workspace = true
|
||||
rustix.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
339
crates/lazaret-ignored-workspace/src/backing.rs
Normal file
339
crates/lazaret-ignored-workspace/src/backing.rs
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
use std::ffi::{OsStr, OsString};
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::io::Read;
|
||||
use std::os::fd::AsFd;
|
||||
use std::os::unix::ffi::{OsStrExt, OsStringExt};
|
||||
use std::path::{Component, Path};
|
||||
|
||||
use rustix::fs::{
|
||||
AtFlags, Mode, OFlags, RawDir, RenameFlags, ResolveFlags, Stat, Timestamps, linkat, mkdirat,
|
||||
openat, openat2, readlinkat, renameat_with, statat, symlinkat, unlinkat, utimensat,
|
||||
};
|
||||
|
||||
const RESOLVE: ResolveFlags = ResolveFlags::BENEATH
|
||||
.union(ResolveFlags::NO_MAGICLINKS)
|
||||
.union(ResolveFlags::NO_SYMLINKS);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct BackingRoot {
|
||||
directory: File,
|
||||
}
|
||||
|
||||
impl BackingRoot {
|
||||
pub(crate) fn open(path: &Path) -> io::Result<Self> {
|
||||
let directory = File::from(openat2(
|
||||
rustix::fs::CWD,
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS,
|
||||
)?);
|
||||
Ok(Self { directory })
|
||||
}
|
||||
|
||||
pub(crate) fn from_file(directory: File) -> io::Result<Self> {
|
||||
let stat = rustix::fs::fstat(directory.as_fd())?;
|
||||
if stat.st_mode & libc::S_IFMT != libc::S_IFDIR {
|
||||
return Err(io::Error::other("backing authority is not a directory"));
|
||||
}
|
||||
Ok(Self { directory })
|
||||
}
|
||||
|
||||
pub(crate) fn identity(&self) -> io::Result<(u64, u64)> {
|
||||
let stat = rustix::fs::fstat(self.directory.as_fd())?;
|
||||
Ok((stat.st_dev, stat.st_ino))
|
||||
}
|
||||
|
||||
pub(crate) fn duplicate(&self) -> io::Result<File> {
|
||||
self.directory.try_clone()
|
||||
}
|
||||
|
||||
pub(crate) fn stat(&self, relative: &Path) -> io::Result<Stat> {
|
||||
if relative.as_os_str().is_empty() {
|
||||
return Ok(rustix::fs::fstat(self.directory.as_fd())?);
|
||||
}
|
||||
let (parent, name) = split(relative)?;
|
||||
let parent = self.open_directory(parent)?;
|
||||
Ok(statat(parent.as_fd(), name, AtFlags::SYMLINK_NOFOLLOW)?)
|
||||
}
|
||||
|
||||
pub(crate) fn open_entry(
|
||||
&self,
|
||||
relative: &Path,
|
||||
flags: OFlags,
|
||||
mode: Mode,
|
||||
) -> io::Result<File> {
|
||||
if relative.as_os_str().is_empty() {
|
||||
return self.duplicate();
|
||||
}
|
||||
let (parent, name) = split(relative)?;
|
||||
let parent = self.open_directory(parent)?;
|
||||
Ok(File::from(openat(
|
||||
parent.as_fd(),
|
||||
name,
|
||||
flags | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
mode,
|
||||
)?))
|
||||
}
|
||||
|
||||
pub(crate) fn open_directory(&self, relative: &Path) -> io::Result<File> {
|
||||
validate_relative(relative)?;
|
||||
let path = if relative.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
relative
|
||||
};
|
||||
Ok(File::from(openat2(
|
||||
self.directory.as_fd(),
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
RESOLVE,
|
||||
)?))
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_parent_directories(&self, relative: &Path) -> io::Result<()> {
|
||||
let parent = relative.parent().unwrap_or_else(|| Path::new(""));
|
||||
validate_relative(parent)?;
|
||||
let mut current = self.duplicate()?;
|
||||
for component in parent.components() {
|
||||
let Component::Normal(name) = component else {
|
||||
return Err(invalid_path());
|
||||
};
|
||||
match mkdirat(current.as_fd(), name, Mode::from_raw_mode(0o700)) {
|
||||
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
current = File::from(openat(
|
||||
current.as_fd(),
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn mkdir(&self, relative: &Path, mode: Mode) -> io::Result<()> {
|
||||
let (parent, name) = split(relative)?;
|
||||
let parent = self.open_directory(parent)?;
|
||||
Ok(mkdirat(parent.as_fd(), name, mode)?)
|
||||
}
|
||||
|
||||
pub(crate) fn unlink(&self, relative: &Path, directory: bool) -> io::Result<()> {
|
||||
let (parent, name) = split(relative)?;
|
||||
let parent = self.open_directory(parent)?;
|
||||
let flags = if directory {
|
||||
AtFlags::REMOVEDIR
|
||||
} else {
|
||||
AtFlags::empty()
|
||||
};
|
||||
Ok(unlinkat(parent.as_fd(), name, flags)?)
|
||||
}
|
||||
|
||||
pub(crate) fn rename(
|
||||
&self,
|
||||
source: &Path,
|
||||
destination: &Path,
|
||||
flags: RenameFlags,
|
||||
) -> io::Result<()> {
|
||||
let (source_parent, source_name) = split(source)?;
|
||||
let (destination_parent, destination_name) = split(destination)?;
|
||||
let source_parent = self.open_directory(source_parent)?;
|
||||
let destination_parent = self.open_directory(destination_parent)?;
|
||||
Ok(renameat_with(
|
||||
source_parent.as_fd(),
|
||||
source_name,
|
||||
destination_parent.as_fd(),
|
||||
destination_name,
|
||||
flags,
|
||||
)?)
|
||||
}
|
||||
|
||||
pub(crate) fn link(&self, source: &Path, destination: &Path) -> io::Result<()> {
|
||||
let (source_parent, source_name) = split(source)?;
|
||||
let (destination_parent, destination_name) = split(destination)?;
|
||||
let source_parent = self.open_directory(source_parent)?;
|
||||
let destination_parent = self.open_directory(destination_parent)?;
|
||||
Ok(linkat(
|
||||
source_parent.as_fd(),
|
||||
source_name,
|
||||
destination_parent.as_fd(),
|
||||
destination_name,
|
||||
AtFlags::empty(),
|
||||
)?)
|
||||
}
|
||||
|
||||
pub(crate) fn symlink(&self, target: &Path, relative: &Path) -> io::Result<()> {
|
||||
let (parent, name) = split(relative)?;
|
||||
let parent = self.open_directory(parent)?;
|
||||
Ok(symlinkat(target, parent.as_fd(), name)?)
|
||||
}
|
||||
|
||||
pub(crate) fn readlink(&self, relative: &Path) -> io::Result<Vec<u8>> {
|
||||
let (parent, name) = split(relative)?;
|
||||
let parent = self.open_directory(parent)?;
|
||||
Ok(readlinkat(parent.as_fd(), name, Vec::new())?.into_bytes())
|
||||
}
|
||||
|
||||
pub(crate) fn set_symlink_times(
|
||||
&self,
|
||||
relative: &Path,
|
||||
timestamps: &Timestamps,
|
||||
) -> io::Result<()> {
|
||||
let (parent, name) = split(relative)?;
|
||||
let parent = self.open_directory(parent)?;
|
||||
Ok(utimensat(
|
||||
parent.as_fd(),
|
||||
name,
|
||||
timestamps,
|
||||
AtFlags::SYMLINK_NOFOLLOW,
|
||||
)?)
|
||||
}
|
||||
|
||||
pub(crate) fn entries(&self, relative: &Path) -> io::Result<Vec<OsString>> {
|
||||
Ok(self
|
||||
.typed_entries(relative)?
|
||||
.into_iter()
|
||||
.map(|(name, _)| name)
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(crate) fn typed_entries(
|
||||
&self,
|
||||
relative: &Path,
|
||||
) -> io::Result<Vec<(OsString, rustix::fs::FileType)>> {
|
||||
let directory = self.open_directory(relative)?;
|
||||
let mut buffer = vec![std::mem::MaybeUninit::<u8>::uninit(); 32 * 1024];
|
||||
let mut entries = RawDir::new(directory, &mut buffer);
|
||||
let mut names = Vec::new();
|
||||
while let Some(entry) = entries.next() {
|
||||
let entry = entry?;
|
||||
let bytes = entry.file_name().to_bytes();
|
||||
if bytes != b"." && bytes != b".." {
|
||||
names.push((OsString::from_vec(bytes.to_vec()), entry.file_type()));
|
||||
}
|
||||
}
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
pub(crate) fn read_file(&self, relative: &Path, maximum: usize) -> io::Result<Vec<u8>> {
|
||||
let mut file =
|
||||
self.open_entry(relative, OFlags::RDONLY | OFlags::NONBLOCK, Mode::empty())?;
|
||||
let metadata = file.metadata()?;
|
||||
let length = usize::try_from(metadata.len())
|
||||
.map_err(|_| io::Error::other("file length is out of range"))?;
|
||||
if !metadata.file_type().is_file() || length > maximum {
|
||||
return Err(io::Error::other(
|
||||
"classification input is not a bounded regular file",
|
||||
));
|
||||
}
|
||||
let mut bytes = Vec::with_capacity(length.saturating_add(1));
|
||||
Read::by_ref(&mut file)
|
||||
.take(u64::try_from(maximum).unwrap_or(u64::MAX).saturating_add(1))
|
||||
.read_to_end(&mut bytes)?;
|
||||
if bytes.len() != length {
|
||||
return Err(io::Error::other(
|
||||
"classification input changed while being read",
|
||||
));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_relative(path: &Path) -> io::Result<()> {
|
||||
if path.is_absolute()
|
||||
|| path
|
||||
.components()
|
||||
.any(|component| !matches!(component, Component::Normal(_)))
|
||||
|| path.as_os_str().as_bytes().contains(&0)
|
||||
{
|
||||
Err(invalid_path())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate_name(name: &OsStr) -> io::Result<()> {
|
||||
let bytes = name.as_bytes();
|
||||
if bytes.is_empty()
|
||||
|| bytes == b"."
|
||||
|| bytes == b".."
|
||||
|| bytes.contains(&0)
|
||||
|| bytes.contains(&b'/')
|
||||
{
|
||||
Err(invalid_path())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn split(relative: &Path) -> io::Result<(&Path, &OsStr)> {
|
||||
validate_relative(relative)?;
|
||||
let name = relative.file_name().ok_or_else(invalid_path)?;
|
||||
validate_name(name)?;
|
||||
Ok((relative.parent().unwrap_or_else(|| Path::new("")), name))
|
||||
}
|
||||
|
||||
fn invalid_path() -> io::Error {
|
||||
io::Error::from_raw_os_error(libc::EINVAL)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::os::unix::fs::symlink;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustix::fs::{RenameFlags, renameat_with};
|
||||
|
||||
use super::BackingRoot;
|
||||
|
||||
#[test]
|
||||
fn descriptor_traversal_race_never_follows_backing_symlinks() {
|
||||
let base = std::env::current_dir()
|
||||
.unwrap()
|
||||
.join("target/ignored-workspace-tests")
|
||||
.join(format!("confinement-{}", std::process::id()));
|
||||
let root = base.join("root");
|
||||
let outside = base.join("outside");
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
std::fs::create_dir_all(&outside).unwrap();
|
||||
std::fs::write(outside.join("sentinel"), b"outside").unwrap();
|
||||
let raced = root.join("raced");
|
||||
let candidate = root.join("candidate");
|
||||
std::fs::create_dir(&raced).unwrap();
|
||||
std::fs::write(raced.join("sentinel"), b"inside").unwrap();
|
||||
symlink(&outside, &candidate).unwrap();
|
||||
|
||||
let authority = Arc::new(BackingRoot::open(&root).unwrap());
|
||||
let exchange_thread = std::thread::spawn(move || {
|
||||
for _ in 0..128 {
|
||||
renameat_with(
|
||||
rustix::fs::CWD,
|
||||
&raced,
|
||||
rustix::fs::CWD,
|
||||
&candidate,
|
||||
RenameFlags::EXCHANGE,
|
||||
)
|
||||
.unwrap();
|
||||
renameat_with(
|
||||
rustix::fs::CWD,
|
||||
&raced,
|
||||
rustix::fs::CWD,
|
||||
&candidate,
|
||||
RenameFlags::EXCHANGE,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
for _ in 0..256 {
|
||||
if let Ok(bytes) = authority.read_file(std::path::Path::new("raced/sentinel"), 64) {
|
||||
assert_eq!(bytes, b"inside");
|
||||
}
|
||||
}
|
||||
exchange_thread.join().unwrap();
|
||||
assert_eq!(std::fs::read(outside.join("sentinel")).unwrap(), b"outside");
|
||||
std::fs::remove_dir_all(base).unwrap();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
use std::env;
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lazaret_ignored_workspace::{GitSentry, GitSentryConfiguration};
|
||||
|
||||
fn main() {
|
||||
if let Err(error) = run() {
|
||||
eprintln!("lazaret Git Sentry refused: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> io::Result<()> {
|
||||
let arguments = env::args_os().skip(1).collect::<Vec<_>>();
|
||||
if arguments.len() != 3 {
|
||||
return Err(io::Error::other(
|
||||
"usage: lazaret-git-sentry MOUNTED_WORKSPACE HOST_GIT_COMMON_DIRECTORY TRANSIENT_HOOKS",
|
||||
));
|
||||
}
|
||||
let sentry = GitSentry::install(&GitSentryConfiguration {
|
||||
mounted_workspace: PathBuf::from(&arguments[0]),
|
||||
host_git_common_directory: PathBuf::from(&arguments[1]),
|
||||
transient_hooks: PathBuf::from(&arguments[2]),
|
||||
})?;
|
||||
println!("ready");
|
||||
io::stdout().flush()?;
|
||||
io::stdin().read_to_end(&mut Vec::new())?;
|
||||
sentry.unmount()
|
||||
}
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::io::{self, Read};
|
||||
use std::os::unix::ffi::{OsStrExt, OsStringExt};
|
||||
use std::os::unix::fs::{FileTypeExt, MetadataExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use rustix::fs::{AtFlags, StatxFlags, statx};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const FIXTURE_CONTEXT: &[u8] = b"lazaret-ignored-workspace-fixture-v1\0";
|
||||
|
||||
fn main() {
|
||||
if let Err(error) = run() {
|
||||
eprintln!("lazaret workspace benchmark refused: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> io::Result<()> {
|
||||
let arguments = env::args_os().skip(1).collect::<Vec<_>>();
|
||||
let root = parse_metadata_arguments(&arguments)?;
|
||||
let result = measure_metadata(&root)?;
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::json!({
|
||||
"content_bytes": result.content_bytes,
|
||||
"fixture_identity": hex(&result.fixture_identity),
|
||||
"regular_files": result.regular_files,
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_metadata_arguments(arguments: &[OsString]) -> io::Result<PathBuf> {
|
||||
if arguments.len() != 6
|
||||
|| arguments[0] != "metadata"
|
||||
|| arguments[1] != "--root"
|
||||
|| arguments[3] != "--bytewise-order"
|
||||
|| arguments[4] != "--statx-mask"
|
||||
|| arguments[5] != "basic-stats"
|
||||
{
|
||||
return Err(io::Error::other(
|
||||
"usage: lazaret-workspace-benchmark metadata --root ROOT --bytewise-order --statx-mask basic-stats",
|
||||
));
|
||||
}
|
||||
Ok(PathBuf::from(&arguments[2]))
|
||||
}
|
||||
|
||||
fn measure_metadata(root: &Path) -> io::Result<MetadataResult> {
|
||||
let mut entries = Vec::new();
|
||||
collect_entries(root, Path::new(""), &mut entries)?;
|
||||
entries.sort_by(|left, right| {
|
||||
left.relative
|
||||
.as_os_str()
|
||||
.as_bytes()
|
||||
.cmp(right.relative.as_os_str().as_bytes())
|
||||
});
|
||||
|
||||
let mut stream = Vec::from(FIXTURE_CONTEXT);
|
||||
stream.extend_from_slice(
|
||||
&u32::try_from(entries.len())
|
||||
.map_err(|_| io::Error::other("fixture contains too many entries"))?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
let mut regular_files = 0_u64;
|
||||
let mut content_bytes = 0_u64;
|
||||
for entry in entries {
|
||||
let path_bytes = entry.relative.as_os_str().as_bytes();
|
||||
stream.extend_from_slice(
|
||||
&u32::try_from(path_bytes.len())
|
||||
.map_err(|_| io::Error::other("fixture path is too long"))?
|
||||
.to_be_bytes(),
|
||||
);
|
||||
stream.extend_from_slice(path_bytes);
|
||||
stream.push(entry.kind);
|
||||
stream.extend_from_slice(&(entry.metadata.mode() & 0o7777).to_be_bytes());
|
||||
let payload_length = u64::try_from(entry.payload.len())
|
||||
.map_err(|_| io::Error::other("fixture payload is too large"))?;
|
||||
stream.extend_from_slice(&payload_length.to_be_bytes());
|
||||
stream.extend_from_slice(&Sha256::digest(&entry.payload));
|
||||
if entry.kind == 2 {
|
||||
regular_files = regular_files.saturating_add(1);
|
||||
content_bytes = content_bytes.saturating_add(payload_length);
|
||||
}
|
||||
}
|
||||
Ok(MetadataResult {
|
||||
regular_files,
|
||||
content_bytes,
|
||||
fixture_identity: Sha256::digest(stream).into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn collect_entries(
|
||||
root: &Path,
|
||||
relative: &Path,
|
||||
entries: &mut Vec<FixtureEntry>,
|
||||
) -> io::Result<()> {
|
||||
let mut children = fs::read_dir(root.join(relative))?
|
||||
.map(|entry| entry.map(|entry| entry.file_name()))
|
||||
.collect::<io::Result<Vec<_>>>()?;
|
||||
children.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
|
||||
for name in children {
|
||||
let child_relative = relative.join(name);
|
||||
let path = root.join(&child_relative);
|
||||
let metadata = fs::symlink_metadata(&path)?;
|
||||
statx(
|
||||
rustix::fs::CWD,
|
||||
&path,
|
||||
AtFlags::SYMLINK_NOFOLLOW,
|
||||
StatxFlags::BASIC_STATS,
|
||||
)?;
|
||||
let file_type = metadata.file_type();
|
||||
let (kind, payload) = if file_type.is_dir() {
|
||||
(1, Vec::new())
|
||||
} else if file_type.is_file() {
|
||||
let mut payload = Vec::new();
|
||||
fs::File::open(&path)?.read_to_end(&mut payload)?;
|
||||
(2, payload)
|
||||
} else if file_type.is_symlink() {
|
||||
(3, fs::read_link(&path)?.into_os_string().into_vec())
|
||||
} else if file_type.is_socket()
|
||||
|| file_type.is_fifo()
|
||||
|| file_type.is_block_device()
|
||||
|| file_type.is_char_device()
|
||||
{
|
||||
return Err(io::Error::other(
|
||||
"fixture contains an unsupported entry kind",
|
||||
));
|
||||
} else {
|
||||
return Err(io::Error::other("fixture entry kind is unknown"));
|
||||
};
|
||||
entries.push(FixtureEntry {
|
||||
relative: child_relative.clone(),
|
||||
metadata,
|
||||
kind,
|
||||
payload,
|
||||
});
|
||||
if kind == 1 {
|
||||
collect_entries(root, &child_relative, entries)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
const DIGITS: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut encoded = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
encoded.push(char::from(DIGITS[usize::from(byte >> 4)]));
|
||||
encoded.push(char::from(DIGITS[usize::from(byte & 0x0f)]));
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
struct FixtureEntry {
|
||||
relative: PathBuf,
|
||||
metadata: fs::Metadata,
|
||||
kind: u8,
|
||||
payload: Vec<u8>,
|
||||
}
|
||||
|
||||
struct MetadataResult {
|
||||
regular_files: u64,
|
||||
content_bytes: u64,
|
||||
fixture_identity: [u8; 32],
|
||||
}
|
||||
959
crates/lazaret-ignored-workspace/src/classifier.rs
Normal file
959
crates/lazaret-ignored-workspace/src/classifier.rs
Normal file
|
|
@ -0,0 +1,959 @@
|
|||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::io;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::os::unix::ffi::OsStringExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use bstr::BStr;
|
||||
|
||||
use crate::backing::{BackingRoot, validate_relative};
|
||||
|
||||
const MAXIMUM_POLICY_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum Classification {
|
||||
Host,
|
||||
Isolated,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ClassificationSnapshot {
|
||||
inner: Arc<SnapshotInner>,
|
||||
}
|
||||
|
||||
impl ClassificationSnapshot {
|
||||
#[must_use]
|
||||
pub fn classify(&self, relative: &Path, is_directory: bool) -> Classification {
|
||||
self.inner.classify(relative, is_directory)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn generation(&self) -> u64 {
|
||||
self.inner.generation
|
||||
}
|
||||
|
||||
pub(crate) fn is_tracked(&self, relative: &Path) -> bool {
|
||||
self.inner.is_tracked(relative)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GitClassifier {
|
||||
workspace: PathBuf,
|
||||
root: BackingRoot,
|
||||
global_excludes: Option<PathBuf>,
|
||||
current: Mutex<Arc<SnapshotInner>>,
|
||||
}
|
||||
|
||||
impl GitClassifier {
|
||||
/// # Errors
|
||||
/// Refuses a non-repository workspace or unreadable Git policy input.
|
||||
pub fn open(workspace: &Path, global_excludes: Option<PathBuf>) -> io::Result<Self> {
|
||||
let workspace = workspace.canonicalize()?;
|
||||
let root = BackingRoot::open(&workspace)?;
|
||||
Self::from_root(workspace, root, global_excludes)
|
||||
}
|
||||
|
||||
pub(crate) fn from_root(
|
||||
workspace: PathBuf,
|
||||
root: BackingRoot,
|
||||
global_excludes: Option<PathBuf>,
|
||||
) -> io::Result<Self> {
|
||||
let current = Arc::new(SnapshotInner::load(
|
||||
&workspace,
|
||||
&root,
|
||||
global_excludes.as_deref(),
|
||||
1,
|
||||
)?);
|
||||
Ok(Self {
|
||||
workspace,
|
||||
root,
|
||||
global_excludes,
|
||||
current: Mutex::new(current),
|
||||
})
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn snapshot(&self) -> ClassificationSnapshot {
|
||||
self.snapshot_for(Path::new(""))
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn snapshot_for(&self, relative: &Path) -> ClassificationSnapshot {
|
||||
self.snapshot_for_paths(&[relative])
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub(crate) fn snapshot_for_paths(&self, relatives: &[&Path]) -> ClassificationSnapshot {
|
||||
let mut current = self
|
||||
.current
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if relatives
|
||||
.iter()
|
||||
.any(|relative| current.relevant_input_changed(&self.root, relative))
|
||||
{
|
||||
let generation = current.generation.saturating_add(1);
|
||||
if let Ok(next) = SnapshotInner::load(
|
||||
&self.workspace,
|
||||
&self.root,
|
||||
self.global_excludes.as_deref(),
|
||||
generation,
|
||||
) {
|
||||
*current = Arc::new(next);
|
||||
}
|
||||
}
|
||||
ClassificationSnapshot {
|
||||
inner: Arc::clone(¤t),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SnapshotInner {
|
||||
generation: u64,
|
||||
repositories: Vec<RepositoryPolicy>,
|
||||
workspace_inputs: BTreeMap<PathBuf, Option<InputStamp>>,
|
||||
external_inputs: BTreeMap<PathBuf, Option<InputStamp>>,
|
||||
}
|
||||
|
||||
impl SnapshotInner {
|
||||
fn load(
|
||||
workspace: &Path,
|
||||
root: &BackingRoot,
|
||||
global_excludes: Option<&Path>,
|
||||
generation: u64,
|
||||
) -> io::Result<Self> {
|
||||
let mut repository_roots = vec![PathBuf::new()];
|
||||
let mut ignore_files = Vec::new();
|
||||
let mut workspace_inputs = BTreeMap::new();
|
||||
scan_workspace(
|
||||
root,
|
||||
Path::new(""),
|
||||
&mut Vec::new(),
|
||||
&mut repository_roots,
|
||||
&mut ignore_files,
|
||||
&mut workspace_inputs,
|
||||
)?;
|
||||
repository_roots.sort_by_key(|path| path.components().count());
|
||||
repository_roots.dedup();
|
||||
let all_repository_roots = repository_roots.clone();
|
||||
|
||||
let mut external_inputs = BTreeMap::new();
|
||||
let mut repositories = Vec::new();
|
||||
for prefix in repository_roots {
|
||||
let metadata = repository_metadata(workspace, root, &prefix)?;
|
||||
for path in [&metadata.index, &metadata.config, &metadata.info_exclude] {
|
||||
external_inputs.insert(path.clone(), input_stamp(path)?);
|
||||
}
|
||||
let applicable_ignores = ignore_files
|
||||
.iter()
|
||||
.filter(|path| {
|
||||
all_repository_roots
|
||||
.iter()
|
||||
.filter(|root| path.starts_with(root))
|
||||
.max_by_key(|root| root.components().count())
|
||||
== Some(&prefix)
|
||||
})
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
repositories.push(RepositoryPolicy::load(
|
||||
workspace,
|
||||
root,
|
||||
prefix,
|
||||
&metadata,
|
||||
applicable_ignores,
|
||||
global_excludes,
|
||||
)?);
|
||||
}
|
||||
if let Some(path) = global_excludes {
|
||||
external_inputs.insert(path.to_path_buf(), input_stamp(path)?);
|
||||
}
|
||||
for (path, expected) in &workspace_inputs {
|
||||
if workspace_input_stamp(root, path)? != *expected {
|
||||
return Err(io::Error::other(
|
||||
"classification input changed while building a snapshot",
|
||||
));
|
||||
}
|
||||
}
|
||||
for (path, expected) in &external_inputs {
|
||||
if input_stamp(path)? != *expected {
|
||||
return Err(io::Error::other(
|
||||
"classification input changed while building a snapshot",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
generation,
|
||||
repositories,
|
||||
workspace_inputs,
|
||||
external_inputs,
|
||||
})
|
||||
}
|
||||
|
||||
fn classify(&self, relative: &Path, is_directory: bool) -> Classification {
|
||||
if relative.as_os_str().is_empty()
|
||||
|| relative
|
||||
.components()
|
||||
.next()
|
||||
.is_some_and(|component| component.as_os_str() == OsStr::new(".git"))
|
||||
{
|
||||
return Classification::Host;
|
||||
}
|
||||
let repository = self
|
||||
.repositories
|
||||
.iter()
|
||||
.filter(|repository| relative.starts_with(&repository.prefix))
|
||||
.max_by_key(|repository| repository.prefix.components().count());
|
||||
repository.map_or(Classification::Host, |repository| {
|
||||
repository.classify(relative, is_directory)
|
||||
})
|
||||
}
|
||||
|
||||
fn is_tracked(&self, relative: &Path) -> bool {
|
||||
self.repositories
|
||||
.iter()
|
||||
.filter(|repository| relative.starts_with(&repository.prefix))
|
||||
.max_by_key(|repository| repository.prefix.components().count())
|
||||
.is_some_and(|repository| repository.is_tracked(relative))
|
||||
}
|
||||
|
||||
fn relevant_input_changed(&self, root: &BackingRoot, relative: &Path) -> bool {
|
||||
if validate_relative(relative).is_err() {
|
||||
return false;
|
||||
}
|
||||
let mut directory = if root
|
||||
.stat(relative)
|
||||
.is_ok_and(|stat| stat.st_mode & libc::S_IFMT == libc::S_IFDIR)
|
||||
{
|
||||
relative
|
||||
} else {
|
||||
relative.parent().unwrap_or_else(|| Path::new(""))
|
||||
};
|
||||
loop {
|
||||
for name in [OsStr::new(".gitignore"), OsStr::new(".git")] {
|
||||
let path = directory.join(name);
|
||||
let Ok(current) = workspace_input_stamp(root, &path) else {
|
||||
return true;
|
||||
};
|
||||
if self.workspace_inputs.get(&path).copied().unwrap_or(None) != current {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
let Some(parent) = directory.parent() else {
|
||||
break;
|
||||
};
|
||||
if parent == directory {
|
||||
break;
|
||||
}
|
||||
directory = parent;
|
||||
}
|
||||
self.external_inputs
|
||||
.iter()
|
||||
.any(|(path, expected)| match input_stamp(path) {
|
||||
Ok(current) => current != *expected,
|
||||
Err(_) => true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct RepositoryPolicy {
|
||||
prefix: PathBuf,
|
||||
tracked: BTreeSet<Vec<u8>>,
|
||||
tracked_directories: BTreeSet<Vec<u8>>,
|
||||
search: gix_ignore::Search,
|
||||
case: gix_ignore::glob::pattern::Case,
|
||||
}
|
||||
|
||||
impl RepositoryPolicy {
|
||||
fn load(
|
||||
workspace: &Path,
|
||||
root: &BackingRoot,
|
||||
prefix: PathBuf,
|
||||
metadata: &RepositoryMetadata,
|
||||
mut ignore_files: Vec<PathBuf>,
|
||||
global_excludes: Option<&Path>,
|
||||
) -> io::Result<Self> {
|
||||
let repository = workspace.join(&prefix);
|
||||
let tracked = match read_external_optional(&metadata.index)? {
|
||||
Some(index) => parse_index_paths(&index)?,
|
||||
None => BTreeSet::new(),
|
||||
};
|
||||
let mut tracked_directories = BTreeSet::new();
|
||||
for path in &tracked {
|
||||
for separator in path
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, byte)| (*byte == b'/').then_some(index))
|
||||
{
|
||||
tracked_directories.insert(path[..separator].to_vec());
|
||||
}
|
||||
}
|
||||
|
||||
let mut search = gix_ignore::Search::default();
|
||||
if let Some(global) = global_excludes
|
||||
&& let Some(bytes) = read_external_optional(global)?
|
||||
{
|
||||
search.add_patterns_buffer(
|
||||
&bytes,
|
||||
global,
|
||||
Some(&repository),
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = read_external_optional(&metadata.info_exclude)? {
|
||||
search.add_patterns_buffer(
|
||||
&bytes,
|
||||
&metadata.info_exclude,
|
||||
None,
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
}
|
||||
ignore_files.sort_by_key(|path| path.components().count());
|
||||
for path in ignore_files {
|
||||
let bytes = root.read_file(&path, MAXIMUM_POLICY_BYTES)?;
|
||||
let source = workspace.join(&path);
|
||||
search.add_patterns_buffer(
|
||||
&bytes,
|
||||
source,
|
||||
Some(&repository),
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
}
|
||||
let ignore_case = read_external_optional(&metadata.config)?
|
||||
.is_some_and(|bytes| config_boolean(&bytes, b"core", b"ignorecase"));
|
||||
let case = if ignore_case {
|
||||
gix_ignore::glob::pattern::Case::Fold
|
||||
} else {
|
||||
gix_ignore::glob::pattern::Case::Sensitive
|
||||
};
|
||||
Ok(Self {
|
||||
prefix,
|
||||
tracked,
|
||||
tracked_directories,
|
||||
search,
|
||||
case,
|
||||
})
|
||||
}
|
||||
|
||||
fn classify(&self, relative: &Path, is_directory: bool) -> Classification {
|
||||
let repository_relative = relative
|
||||
.strip_prefix(&self.prefix)
|
||||
.unwrap_or(relative)
|
||||
.as_os_str()
|
||||
.as_bytes();
|
||||
if self.tracked.contains(repository_relative)
|
||||
|| self.tracked_directories.contains(repository_relative)
|
||||
{
|
||||
return Classification::Host;
|
||||
}
|
||||
let ignored_ancestor = repository_relative
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, byte)| (*byte == b'/').then_some(&repository_relative[..index]))
|
||||
.any(|ancestor| {
|
||||
self.search
|
||||
.pattern_matching_relative_path(BStr::new(ancestor), Some(true), self.case)
|
||||
.is_some_and(|matched| !matched.pattern.is_negative())
|
||||
});
|
||||
let ignored = ignored_ancestor
|
||||
|| self
|
||||
.search
|
||||
.pattern_matching_relative_path(
|
||||
BStr::new(repository_relative),
|
||||
Some(is_directory),
|
||||
self.case,
|
||||
)
|
||||
.is_some_and(|matched| !matched.pattern.is_negative());
|
||||
if ignored {
|
||||
Classification::Isolated
|
||||
} else {
|
||||
Classification::Host
|
||||
}
|
||||
}
|
||||
|
||||
fn is_tracked(&self, relative: &Path) -> bool {
|
||||
let repository_relative = relative
|
||||
.strip_prefix(&self.prefix)
|
||||
.unwrap_or(relative)
|
||||
.as_os_str()
|
||||
.as_bytes();
|
||||
self.tracked.contains(repository_relative)
|
||||
}
|
||||
}
|
||||
|
||||
struct RepositoryMetadata {
|
||||
index: PathBuf,
|
||||
config: PathBuf,
|
||||
info_exclude: PathBuf,
|
||||
}
|
||||
|
||||
fn repository_metadata(
|
||||
workspace: &Path,
|
||||
root: &BackingRoot,
|
||||
prefix: &Path,
|
||||
) -> io::Result<RepositoryMetadata> {
|
||||
let dot_git = prefix.join(".git");
|
||||
let stat = root.stat(&dot_git)?;
|
||||
let repository = workspace.join(prefix);
|
||||
let git_dir = if stat.st_mode & libc::S_IFMT == libc::S_IFDIR {
|
||||
workspace.join(&dot_git)
|
||||
} else if stat.st_mode & libc::S_IFMT == libc::S_IFREG {
|
||||
let bytes = root.read_file(&dot_git, 64 * 1024)?;
|
||||
let value = bytes
|
||||
.strip_prefix(b"gitdir:")
|
||||
.ok_or_else(|| io::Error::other("worktree Git pointer has no gitdir prefix"))?;
|
||||
let value = trim_ascii(value);
|
||||
let path = PathBuf::from(OsString::from_vec(value.to_vec()));
|
||||
if path.is_absolute() {
|
||||
path
|
||||
} else {
|
||||
repository.join(path)
|
||||
}
|
||||
} else {
|
||||
return Err(io::Error::other(
|
||||
"repository .git entry has unsupported type",
|
||||
));
|
||||
};
|
||||
let git_root = BackingRoot::open(&git_dir)?;
|
||||
let common_dir = match git_root.read_file(Path::new("commondir"), 64 * 1024) {
|
||||
Ok(bytes) => {
|
||||
let path = PathBuf::from(OsString::from_vec(trim_ascii(&bytes).to_vec()));
|
||||
if path.is_absolute() {
|
||||
path
|
||||
} else {
|
||||
git_dir.join(path)
|
||||
}
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => git_dir.clone(),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
Ok(RepositoryMetadata {
|
||||
index: git_dir.join("index"),
|
||||
config: common_dir.join("config"),
|
||||
info_exclude: common_dir.join("info/exclude"),
|
||||
})
|
||||
}
|
||||
|
||||
fn scan_workspace(
|
||||
root: &BackingRoot,
|
||||
directory: &Path,
|
||||
patterns: &mut Vec<(PathBuf, Vec<u8>)>,
|
||||
repositories: &mut Vec<PathBuf>,
|
||||
ignore_files: &mut Vec<PathBuf>,
|
||||
inputs: &mut BTreeMap<PathBuf, Option<InputStamp>>,
|
||||
) -> io::Result<()> {
|
||||
let local_ignore = directory.join(".gitignore");
|
||||
let has_local_ignore = match root.read_file(&local_ignore, MAXIMUM_POLICY_BYTES) {
|
||||
Ok(bytes) => {
|
||||
let stat = root.stat(&local_ignore)?;
|
||||
inputs.insert(local_ignore.clone(), Some(InputStamp::from_stat(stat)));
|
||||
ignore_files.push(local_ignore.clone());
|
||||
patterns.push((local_ignore.clone(), bytes));
|
||||
true
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => false,
|
||||
Err(error) if error.raw_os_error() == Some(libc::EISDIR) => false,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
for (name, entry_type) in root.typed_entries(directory)? {
|
||||
let path = directory.join(&name);
|
||||
if name == OsStr::new(".gitignore") {
|
||||
continue;
|
||||
}
|
||||
if name == OsStr::new(".git") {
|
||||
let stat = root.stat(&path)?;
|
||||
inputs.insert(path, Some(InputStamp::from_stat(stat)));
|
||||
if !directory.as_os_str().is_empty() {
|
||||
repositories.push(directory.to_path_buf());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let directory_entry = entry_type == rustix::fs::FileType::Directory
|
||||
|| (entry_type == rustix::fs::FileType::Unknown
|
||||
&& root
|
||||
.stat(&path)
|
||||
.is_ok_and(|stat| stat.st_mode & libc::S_IFMT == libc::S_IFDIR));
|
||||
if directory_entry && !patterns_ignore(patterns, &path) {
|
||||
scan_workspace(root, &path, patterns, repositories, ignore_files, inputs)?;
|
||||
}
|
||||
}
|
||||
let missing_ignore = directory.join(".gitignore");
|
||||
inputs.entry(missing_ignore).or_insert(None);
|
||||
let missing_git = directory.join(".git");
|
||||
inputs.entry(missing_git).or_insert(None);
|
||||
if has_local_ignore {
|
||||
patterns.pop();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn patterns_ignore(patterns: &[(PathBuf, Vec<u8>)], path: &Path) -> bool {
|
||||
let mut search = gix_ignore::Search::default();
|
||||
for (source, bytes) in patterns {
|
||||
search.add_patterns_buffer(
|
||||
bytes,
|
||||
source,
|
||||
Some(Path::new("")),
|
||||
gix_ignore::search::Ignore::default(),
|
||||
);
|
||||
}
|
||||
search
|
||||
.pattern_matching_relative_path(
|
||||
BStr::new(path.as_os_str().as_bytes()),
|
||||
Some(true),
|
||||
gix_ignore::glob::pattern::Case::Sensitive,
|
||||
)
|
||||
.is_some_and(|matched| !matched.pattern.is_negative())
|
||||
}
|
||||
|
||||
fn read_external_optional(path: &Path) -> io::Result<Option<Vec<u8>>> {
|
||||
let Some(parent) = path.parent() else {
|
||||
return Err(io::Error::other("classification input has no parent"));
|
||||
};
|
||||
let Some(name) = path.file_name() else {
|
||||
return Err(io::Error::other("classification input has no name"));
|
||||
};
|
||||
let root = match BackingRoot::open(parent) {
|
||||
Ok(root) => root,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
match root.read_file(Path::new(name), MAXIMUM_POLICY_BYTES) {
|
||||
Ok(bytes) => Ok(Some(bytes)),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_index_paths(bytes: &[u8]) -> io::Result<BTreeSet<Vec<u8>>> {
|
||||
match (
|
||||
parse_index_paths_with_oid(bytes, 20),
|
||||
parse_index_paths_with_oid(bytes, 32),
|
||||
) {
|
||||
(Ok(paths), Err(_)) | (Err(_), Ok(paths)) => Ok(paths),
|
||||
(Ok(first), Ok(second)) if first == second => Ok(first),
|
||||
(Ok(_), Ok(_)) => Err(io::Error::other("Git index object format is ambiguous")),
|
||||
(Err(first), Err(_)) => Err(first),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_index_paths_with_oid(bytes: &[u8], oid_bytes: usize) -> io::Result<BTreeSet<Vec<u8>>> {
|
||||
if bytes.len() < 12 || &bytes[..4] != b"DIRC" {
|
||||
return Err(io::Error::other("Git index header is malformed"));
|
||||
}
|
||||
let version = read_u32(bytes, 4)?;
|
||||
if !(2..=4).contains(&version) {
|
||||
return Err(io::Error::other("Git index version is unsupported"));
|
||||
}
|
||||
let count = usize::try_from(read_u32(bytes, 8)?)
|
||||
.map_err(|_| io::Error::other("Git index entry count is out of range"))?;
|
||||
let mut offset = 12usize;
|
||||
let mut previous = Vec::new();
|
||||
let mut paths = BTreeSet::new();
|
||||
for _ in 0..count {
|
||||
let entry_start = offset;
|
||||
let flags_offset = offset
|
||||
.checked_add(40)
|
||||
.and_then(|offset| offset.checked_add(oid_bytes))
|
||||
.ok_or_else(index_overflow)?;
|
||||
let flags = read_u16(bytes, flags_offset)?;
|
||||
offset = flags_offset.checked_add(2).ok_or_else(index_overflow)?;
|
||||
if flags & 0x4000 != 0 {
|
||||
offset = offset.checked_add(2).ok_or_else(index_overflow)?;
|
||||
}
|
||||
let path = if version == 4 {
|
||||
let (removed, next) = read_index_varint(bytes, offset)?;
|
||||
offset = next;
|
||||
if removed > previous.len() {
|
||||
return Err(io::Error::other("Git index v4 prefix is malformed"));
|
||||
}
|
||||
let end = find_nul(bytes, offset)?;
|
||||
let mut path = previous[..previous.len() - removed].to_vec();
|
||||
path.extend_from_slice(&bytes[offset..end]);
|
||||
offset = end.checked_add(1).ok_or_else(index_overflow)?;
|
||||
path
|
||||
} else {
|
||||
let end = find_nul(bytes, offset)?;
|
||||
let path = bytes[offset..end].to_vec();
|
||||
offset = end.checked_add(1).ok_or_else(index_overflow)?;
|
||||
let length = offset.checked_sub(entry_start).ok_or_else(index_overflow)?;
|
||||
let padding = (8 - length % 8) % 8;
|
||||
offset = offset.checked_add(padding).ok_or_else(index_overflow)?;
|
||||
path
|
||||
};
|
||||
if path.is_empty() || path.contains(&0) || !paths.insert(path.clone()) {
|
||||
return Err(io::Error::other("Git index path is empty or duplicated"));
|
||||
}
|
||||
previous = path;
|
||||
}
|
||||
validate_index_tail(bytes, offset, oid_bytes)?;
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
pub(crate) fn tracked_paths_from_index(bytes: &[u8]) -> io::Result<BTreeSet<Vec<u8>>> {
|
||||
parse_index_paths(bytes)
|
||||
}
|
||||
|
||||
fn validate_index_tail(bytes: &[u8], mut offset: usize, oid_bytes: usize) -> io::Result<()> {
|
||||
let checksum_start = bytes
|
||||
.len()
|
||||
.checked_sub(oid_bytes)
|
||||
.ok_or_else(|| io::Error::other("Git index checksum is absent"))?;
|
||||
while offset < checksum_start {
|
||||
let signature_end = offset.checked_add(4).ok_or_else(index_overflow)?;
|
||||
let signature = bytes
|
||||
.get(offset..signature_end)
|
||||
.ok_or_else(|| io::Error::other("Git index extension is truncated"))?;
|
||||
if !signature.iter().all(u8::is_ascii_uppercase) {
|
||||
return Err(io::Error::other(
|
||||
"Git index extension signature is malformed",
|
||||
));
|
||||
}
|
||||
let length = usize::try_from(read_u32(bytes, signature_end)?)
|
||||
.map_err(|_| io::Error::other("Git index extension size is out of range"))?;
|
||||
offset = signature_end
|
||||
.checked_add(4)
|
||||
.and_then(|offset| offset.checked_add(length))
|
||||
.ok_or_else(index_overflow)?;
|
||||
if offset > checksum_start {
|
||||
return Err(io::Error::other(
|
||||
"Git index extension exceeds the checksum boundary",
|
||||
));
|
||||
}
|
||||
}
|
||||
if offset == checksum_start {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::other("Git index tail is malformed"))
|
||||
}
|
||||
}
|
||||
|
||||
fn read_index_varint(bytes: &[u8], mut offset: usize) -> io::Result<(usize, usize)> {
|
||||
let mut value = 0usize;
|
||||
loop {
|
||||
let byte = *bytes
|
||||
.get(offset)
|
||||
.ok_or_else(|| io::Error::other("Git index v4 varint is truncated"))?;
|
||||
offset = offset.checked_add(1).ok_or_else(index_overflow)?;
|
||||
value = value
|
||||
.checked_shl(7)
|
||||
.and_then(|value| value.checked_add(usize::from(byte & 0x7f)))
|
||||
.ok_or_else(index_overflow)?;
|
||||
if byte & 0x80 == 0 {
|
||||
return Ok((value, offset));
|
||||
}
|
||||
value = value.checked_add(1).ok_or_else(index_overflow)?;
|
||||
}
|
||||
}
|
||||
|
||||
fn read_u32(bytes: &[u8], offset: usize) -> io::Result<u32> {
|
||||
let end = offset.checked_add(4).ok_or_else(index_overflow)?;
|
||||
let array: [u8; 4] = bytes
|
||||
.get(offset..end)
|
||||
.ok_or_else(|| io::Error::other("Git index is truncated"))?
|
||||
.try_into()
|
||||
.map_err(|_| io::Error::other("Git index field is malformed"))?;
|
||||
Ok(u32::from_be_bytes(array))
|
||||
}
|
||||
|
||||
fn read_u16(bytes: &[u8], offset: usize) -> io::Result<u16> {
|
||||
let end = offset.checked_add(2).ok_or_else(index_overflow)?;
|
||||
let array: [u8; 2] = bytes
|
||||
.get(offset..end)
|
||||
.ok_or_else(|| io::Error::other("Git index is truncated"))?
|
||||
.try_into()
|
||||
.map_err(|_| io::Error::other("Git index field is malformed"))?;
|
||||
Ok(u16::from_be_bytes(array))
|
||||
}
|
||||
|
||||
fn find_nul(bytes: &[u8], offset: usize) -> io::Result<usize> {
|
||||
bytes
|
||||
.get(offset..)
|
||||
.and_then(|tail| tail.iter().position(|byte| *byte == 0))
|
||||
.and_then(|position| offset.checked_add(position))
|
||||
.ok_or_else(|| io::Error::other("Git index pathname is unterminated"))
|
||||
}
|
||||
|
||||
fn index_overflow() -> io::Error {
|
||||
io::Error::other("Git index offset is out of range")
|
||||
}
|
||||
|
||||
fn config_boolean(bytes: &[u8], section: &[u8], key: &[u8]) -> bool {
|
||||
let mut current = Vec::new();
|
||||
for line in bytes.split(|byte| *byte == b'\n') {
|
||||
let line = trim_ascii(line);
|
||||
if line.starts_with(b"[") && line.ends_with(b"]") {
|
||||
current = line[1..line.len() - 1]
|
||||
.split(u8::is_ascii_whitespace)
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
} else if current == section {
|
||||
let mut fields = line.splitn(2, |byte| *byte == b'=');
|
||||
let name = trim_ascii(fields.next().unwrap_or_default()).to_ascii_lowercase();
|
||||
let value = trim_ascii(fields.next().unwrap_or(b"true"));
|
||||
if name == key {
|
||||
return value.eq_ignore_ascii_case(b"true")
|
||||
|| value.eq_ignore_ascii_case(b"yes")
|
||||
|| value == b"1";
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn trim_ascii(mut bytes: &[u8]) -> &[u8] {
|
||||
while bytes.first().is_some_and(u8::is_ascii_whitespace) {
|
||||
bytes = &bytes[1..];
|
||||
}
|
||||
while bytes.last().is_some_and(u8::is_ascii_whitespace) {
|
||||
bytes = &bytes[..bytes.len() - 1];
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
fn input_stamp(path: &Path) -> io::Result<Option<InputStamp>> {
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("classification input has no parent"))?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::other("classification input has no name"))?;
|
||||
let root = match BackingRoot::open(parent) {
|
||||
Ok(root) => root,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
match root.stat(Path::new(name)) {
|
||||
Ok(stat) => Ok(Some(InputStamp::from_stat(stat))),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn workspace_input_stamp(root: &BackingRoot, path: &Path) -> io::Result<Option<InputStamp>> {
|
||||
match root.stat(path) {
|
||||
Ok(stat) => Ok(Some(InputStamp::from_stat(stat))),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct InputStamp {
|
||||
device: u64,
|
||||
inode: u64,
|
||||
size: u64,
|
||||
modified_seconds: i64,
|
||||
modified_nanoseconds: i64,
|
||||
changed_seconds: i64,
|
||||
changed_nanoseconds: i64,
|
||||
}
|
||||
|
||||
impl InputStamp {
|
||||
fn from_stat(stat: rustix::fs::Stat) -> Self {
|
||||
Self {
|
||||
device: stat.st_dev,
|
||||
inode: stat.st_ino,
|
||||
size: u64::try_from(stat.st_size).unwrap_or(0),
|
||||
modified_seconds: stat.st_mtime,
|
||||
modified_nanoseconds: i64::try_from(stat.st_mtime_nsec).unwrap_or(i64::MAX),
|
||||
changed_seconds: stat.st_ctime,
|
||||
changed_nanoseconds: i64::try_from(stat.st_ctime_nsec).unwrap_or(i64::MAX),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use super::{Classification, GitClassifier, parse_index_paths};
|
||||
|
||||
static SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[test]
|
||||
fn index_v2_paths_are_parsed_without_utf8_assumptions() {
|
||||
let bytes = index_v2(&[b"src/lib.rs", b"target/\xffapp"]);
|
||||
let paths = parse_index_paths(&bytes).unwrap();
|
||||
assert!(paths.contains(b"src/lib.rs".as_slice()));
|
||||
assert!(paths.contains(b"target/\xffapp".as_slice()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_index_is_refused() {
|
||||
assert!(parse_index_paths(b"DIRC\0\0\0\x05\0\0\0\0").is_err());
|
||||
let mut truncated = index_v2(&[b"tracked"]);
|
||||
truncated.truncate(40);
|
||||
assert!(parse_index_paths(&truncated).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repository_without_an_index_has_no_tracked_paths() {
|
||||
let fixture = Fixture::new("absent-index");
|
||||
repository(&fixture.workspace, &[]);
|
||||
fs::remove_file(fixture.workspace.join(".git/index")).unwrap();
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"target/\n").unwrap();
|
||||
|
||||
let classifier = GitClassifier::open(&fixture.workspace, None).unwrap();
|
||||
assert_eq!(
|
||||
classifier
|
||||
.snapshot_for(std::path::Path::new("target/output"))
|
||||
.classify(std::path::Path::new("target/output"), false),
|
||||
Classification::Isolated
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_exclude_is_root_scoped_with_source_provenance() {
|
||||
let fixture = Fixture::new("info-exclude");
|
||||
repository(&fixture.workspace, &[]);
|
||||
let source = fixture.workspace.join(".git/info/exclude");
|
||||
fs::write(&source, b"local.secret\n").unwrap();
|
||||
|
||||
let classifier = GitClassifier::open(&fixture.workspace, None).unwrap();
|
||||
let snapshot = classifier.snapshot_for(std::path::Path::new("local.secret"));
|
||||
assert_eq!(
|
||||
snapshot.classify(std::path::Path::new("local.secret"), false),
|
||||
Classification::Isolated
|
||||
);
|
||||
assert!(
|
||||
snapshot.inner.repositories[0]
|
||||
.search
|
||||
.patterns
|
||||
.iter()
|
||||
.any(
|
||||
|patterns| patterns.source.as_deref() == Some(source.as_path())
|
||||
&& patterns.base.is_none()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn linked_worktree_git_directory_refuses_symlink_traversal() {
|
||||
let fixture = Fixture::new("linked-symlink");
|
||||
let actual = fixture.root.join("actual-git");
|
||||
fs::create_dir_all(actual.join("info")).unwrap();
|
||||
fs::write(actual.join("config"), b"[core]\nignoreCase = false\n").unwrap();
|
||||
fs::write(actual.join("index"), index_v2(&[])).unwrap();
|
||||
let alias = fixture.root.join("git-alias");
|
||||
std::os::unix::fs::symlink(&actual, &alias).unwrap();
|
||||
fs::write(
|
||||
fixture.workspace.join(".git"),
|
||||
format!("gitdir: {}\n", alias.display()),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(GitClassifier::open(&fixture.workspace, None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshots_route_tracked_ordinary_ignored_and_live_policy_changes() {
|
||||
let fixture = Fixture::new("classification");
|
||||
repository(&fixture.workspace, &[b"tracked.cache"]);
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"*.cache\n").unwrap();
|
||||
fs::write(fixture.workspace.join("tracked.cache"), b"tracked").unwrap();
|
||||
|
||||
let classifier = GitClassifier::open(&fixture.workspace, None).unwrap();
|
||||
let first = classifier.snapshot_for(std::path::Path::new("ignored.cache"));
|
||||
assert_eq!(
|
||||
first.classify(std::path::Path::new("tracked.cache"), false),
|
||||
Classification::Host
|
||||
);
|
||||
assert_eq!(
|
||||
first.classify(std::path::Path::new("ignored.cache"), false),
|
||||
Classification::Isolated
|
||||
);
|
||||
assert_eq!(
|
||||
first.classify(std::path::Path::new("ordinary"), false),
|
||||
Classification::Host
|
||||
);
|
||||
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"*.different\n").unwrap();
|
||||
let second = classifier.snapshot_for(std::path::Path::new("ignored.cache"));
|
||||
assert!(second.generation() > first.generation());
|
||||
assert_eq!(
|
||||
second.classify(std::path::Path::new("ignored.cache"), false),
|
||||
Classification::Host
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_ignore_case_and_nested_repository_policy_are_selected() {
|
||||
let fixture = Fixture::new("nested");
|
||||
repository(&fixture.workspace, &[]);
|
||||
fs::write(
|
||||
fixture.workspace.join(".git/config"),
|
||||
b"[core]\nignoreCase = true\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(fixture.workspace.join(".gitignore"), b"build.log\n").unwrap();
|
||||
let nested = fixture.workspace.join("vendor/nested");
|
||||
fs::create_dir_all(&nested).unwrap();
|
||||
repository(&nested, &[]);
|
||||
fs::write(nested.join(".gitignore"), b"target/\n").unwrap();
|
||||
|
||||
let classifier = GitClassifier::open(&fixture.workspace, None).unwrap();
|
||||
let snapshot = classifier.snapshot_for(std::path::Path::new("BUILD.LOG"));
|
||||
assert_eq!(
|
||||
snapshot.classify(std::path::Path::new("BUILD.LOG"), false),
|
||||
Classification::Isolated
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.classify(std::path::Path::new("vendor/nested/target/app"), false),
|
||||
Classification::Isolated
|
||||
);
|
||||
}
|
||||
|
||||
fn repository(root: &std::path::Path, tracked: &[&[u8]]) {
|
||||
fs::create_dir_all(root.join(".git/info")).unwrap();
|
||||
fs::create_dir_all(root.join(".git/hooks")).unwrap();
|
||||
fs::write(root.join(".git/config"), b"[core]\nignoreCase = false\n").unwrap();
|
||||
fs::write(root.join(".git/info/exclude"), b"").unwrap();
|
||||
fs::write(root.join(".git/index"), index_v2(tracked)).unwrap();
|
||||
}
|
||||
|
||||
fn index_v2(paths: &[&[u8]]) -> Vec<u8> {
|
||||
let mut bytes = b"DIRC".to_vec();
|
||||
bytes.extend_from_slice(&2u32.to_be_bytes());
|
||||
bytes.extend_from_slice(&u32::try_from(paths.len()).unwrap().to_be_bytes());
|
||||
for path in paths {
|
||||
let start = bytes.len();
|
||||
bytes.extend_from_slice(&[0; 60]);
|
||||
bytes.extend_from_slice(&u16::try_from(path.len().min(0x0fff)).unwrap().to_be_bytes());
|
||||
bytes.extend_from_slice(path);
|
||||
bytes.push(0);
|
||||
while !(bytes.len() - start).is_multiple_of(8) {
|
||||
bytes.push(0);
|
||||
}
|
||||
}
|
||||
bytes.extend_from_slice(&[0; 20]);
|
||||
bytes
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
root: std::path::PathBuf,
|
||||
workspace: std::path::PathBuf,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn new(name: &str) -> Self {
|
||||
let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed);
|
||||
let root = std::env::current_dir()
|
||||
.unwrap()
|
||||
.join("target/ignored-workspace-tests")
|
||||
.join(format!("{name}-{}-{sequence}", std::process::id()));
|
||||
let workspace = root.join("workspace");
|
||||
fs::create_dir_all(&workspace).unwrap();
|
||||
Self { root, workspace }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
fs::remove_dir_all(&self.root).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
1742
crates/lazaret-ignored-workspace/src/filesystem.rs
Normal file
1742
crates/lazaret-ignored-workspace/src/filesystem.rs
Normal file
File diff suppressed because it is too large
Load diff
8
crates/lazaret-ignored-workspace/src/lib.rs
Normal file
8
crates/lazaret-ignored-workspace/src/lib.rs
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
mod backing;
|
||||
mod classifier;
|
||||
mod filesystem;
|
||||
mod sentry;
|
||||
|
||||
pub use classifier::{Classification, ClassificationSnapshot, GitClassifier};
|
||||
pub use filesystem::{IgnoredWorkspace, MountConfiguration};
|
||||
pub use sentry::{GitSentry, GitSentryConfiguration};
|
||||
57
crates/lazaret-ignored-workspace/src/main.rs
Normal file
57
crates/lazaret-ignored-workspace/src/main.rs
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
use std::ffi::OsString;
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lazaret_ignored_workspace::{IgnoredWorkspace, MountConfiguration};
|
||||
|
||||
fn main() {
|
||||
if let Err(error) = run() {
|
||||
eprintln!("lazaret ignored workspace refused: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> io::Result<()> {
|
||||
let arguments = std::env::args_os().skip(1).collect::<Vec<_>>();
|
||||
let configuration = parse(&arguments)?;
|
||||
IgnoredWorkspace::mount(configuration)
|
||||
}
|
||||
|
||||
fn parse(arguments: &[OsString]) -> io::Result<MountConfiguration> {
|
||||
if !(3..=4).contains(&arguments.len()) {
|
||||
return Err(io::Error::other(
|
||||
"usage: lazaret-ignored-workspace WORKSPACE ISOLATED_STATE MOUNTPOINT [GLOBAL_EXCLUDES]",
|
||||
));
|
||||
}
|
||||
Ok(MountConfiguration {
|
||||
workspace: PathBuf::from(&arguments[0]),
|
||||
isolated_state: PathBuf::from(&arguments[1]),
|
||||
mountpoint: PathBuf::from(&arguments[2]),
|
||||
global_excludes: arguments.get(3).map(PathBuf::from),
|
||||
threads: std::thread::available_parallelism()
|
||||
.map(std::num::NonZero::get)
|
||||
.unwrap_or(1)
|
||||
.min(8),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::ffi::OsString;
|
||||
|
||||
#[test]
|
||||
fn parser_refuses_incomplete_and_excess_arguments() {
|
||||
assert!(super::parse(&[]).is_err());
|
||||
assert!(super::parse(&[OsString::from("one"), OsString::from("two")]).is_err());
|
||||
assert!(
|
||||
super::parse(&[
|
||||
OsString::from("one"),
|
||||
OsString::from("two"),
|
||||
OsString::from("three"),
|
||||
OsString::from("four"),
|
||||
OsString::from("five"),
|
||||
])
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
99
crates/lazaret-ignored-workspace/src/sentry.rs
Normal file
99
crates/lazaret-ignored-workspace/src/sentry.rs
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::os::fd::{AsRawFd, RawFd};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use rustix::fs::{Mode, OFlags};
|
||||
use rustix::mount::{MountFlags, UnmountFlags, mount_bind, mount_remount, unmount};
|
||||
|
||||
use crate::backing::BackingRoot;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GitSentryConfiguration {
|
||||
pub mounted_workspace: PathBuf,
|
||||
pub host_git_common_directory: PathBuf,
|
||||
pub transient_hooks: PathBuf,
|
||||
}
|
||||
|
||||
pub struct GitSentry {
|
||||
config_target: File,
|
||||
hook_target: File,
|
||||
}
|
||||
|
||||
impl GitSentry {
|
||||
/// # Errors
|
||||
/// Refuses linked or missing sources and targets and incomplete protective mounts.
|
||||
pub fn install(configuration: &GitSentryConfiguration) -> io::Result<Self> {
|
||||
let _workspace = BackingRoot::open(&configuration.mounted_workspace)?;
|
||||
let common = BackingRoot::open(&configuration.host_git_common_directory)?;
|
||||
let transient = BackingRoot::open(&configuration.transient_hooks)?;
|
||||
|
||||
let hook_source = transient.duplicate()?;
|
||||
let hook_target = common.open_entry(
|
||||
Path::new("hooks"),
|
||||
OFlags::RDONLY | OFlags::DIRECTORY,
|
||||
Mode::empty(),
|
||||
)?;
|
||||
descriptor_bind(&hook_source, &hook_target)?;
|
||||
let hook_target = common.open_entry(
|
||||
Path::new("hooks"),
|
||||
OFlags::RDONLY | OFlags::DIRECTORY,
|
||||
Mode::empty(),
|
||||
)?;
|
||||
|
||||
let config_source =
|
||||
common.open_entry(Path::new("config"), OFlags::RDONLY, Mode::empty())?;
|
||||
let config_target =
|
||||
common.open_entry(Path::new("config"), OFlags::RDONLY, Mode::empty())?;
|
||||
descriptor_bind(&config_source, &config_target)?;
|
||||
let config_target =
|
||||
common.open_entry(Path::new("config"), OFlags::RDONLY, Mode::empty())?;
|
||||
if let Err(error) = descriptor_remount_read_only(&config_target) {
|
||||
let _ = descriptor_unmount(&hook_target);
|
||||
return Err(error);
|
||||
}
|
||||
Ok(Self {
|
||||
config_target,
|
||||
hook_target,
|
||||
})
|
||||
}
|
||||
|
||||
/// # Errors
|
||||
/// Returns an error unless both exact protective mountpoints are detached.
|
||||
pub fn unmount(self) -> io::Result<()> {
|
||||
let config = descriptor_unmount(&self.config_target);
|
||||
let hooks = descriptor_unmount(&self.hook_target);
|
||||
config.and(hooks)
|
||||
}
|
||||
}
|
||||
|
||||
fn descriptor_bind(source: &File, target: &File) -> io::Result<()> {
|
||||
Ok(mount_bind(
|
||||
proc_fd(source.as_raw_fd()),
|
||||
proc_fd(target.as_raw_fd()),
|
||||
)?)
|
||||
}
|
||||
|
||||
fn descriptor_remount_read_only(target: &File) -> io::Result<()> {
|
||||
let result = mount_remount(
|
||||
proc_fd(target.as_raw_fd()),
|
||||
MountFlags::BIND
|
||||
| MountFlags::RDONLY
|
||||
| MountFlags::NOSUID
|
||||
| MountFlags::NODEV
|
||||
| MountFlags::NOEXEC,
|
||||
"",
|
||||
);
|
||||
if result.is_err() {
|
||||
let _ = unmount(proc_fd(target.as_raw_fd()), UnmountFlags::DETACH);
|
||||
}
|
||||
Ok(result?)
|
||||
}
|
||||
|
||||
fn descriptor_unmount(target: &File) -> io::Result<()> {
|
||||
Ok(unmount(proc_fd(target.as_raw_fd()), UnmountFlags::DETACH)?)
|
||||
}
|
||||
|
||||
fn proc_fd(fd: RawFd) -> PathBuf {
|
||||
PathBuf::from(format!("/proc/self/fd/{fd}"))
|
||||
}
|
||||
163
patches/fuser-0.17.0-lock-flags.patch
Normal file
163
patches/fuser-0.17.0-lock-flags.patch
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
diff --git a/src/lib.rs b/src/lib.rs
|
||||
index 063f9a0..5b91dae 100644
|
||||
--- a/src/lib.rs
|
||||
+++ b/src/lib.rs
|
||||
@@ -40,6 +40,7 @@ pub use crate::ll::flags::fattr_flags::FattrFlags;
|
||||
pub use crate::ll::flags::fopen_flags::FopenFlags;
|
||||
pub use crate::ll::flags::init_flags::InitFlags;
|
||||
pub use crate::ll::flags::ioctl_flags::IoctlFlags;
|
||||
+pub use crate::ll::flags::lock_flags::LockFlags;
|
||||
pub use crate::ll::flags::poll_flags::PollFlags;
|
||||
pub use crate::ll::flags::write_flags::WriteFlags;
|
||||
pub use crate::ll::fuse_abi::consts;
|
||||
@@ -867,11 +868,12 @@ pub trait Filesystem {
|
||||
end: u64,
|
||||
typ: i32,
|
||||
pid: u32,
|
||||
+ flags: LockFlags,
|
||||
reply: ReplyLock,
|
||||
) {
|
||||
warn!(
|
||||
"[Not Implemented] getlk(ino: {ino:#x?}, fh: {fh}, lock_owner: {lock_owner}, \
|
||||
- start: {start}, end: {end}, typ: {typ}, pid: {pid})"
|
||||
+ start: {start}, end: {end}, typ: {typ}, pid: {pid}, flags: {flags:?})"
|
||||
);
|
||||
reply.error(Errno::ENOSYS);
|
||||
}
|
||||
@@ -894,11 +896,12 @@ pub trait Filesystem {
|
||||
typ: i32,
|
||||
pid: u32,
|
||||
sleep: bool,
|
||||
+ flags: LockFlags,
|
||||
reply: ReplyEmpty,
|
||||
) {
|
||||
warn!(
|
||||
"[Not Implemented] setlk(ino: {ino:#x?}, fh: {fh}, lock_owner: {lock_owner}, \
|
||||
- start: {start}, end: {end}, typ: {typ}, pid: {pid}, sleep: {sleep})"
|
||||
+ start: {start}, end: {end}, typ: {typ}, pid: {pid}, sleep: {sleep}, flags: {flags:?})"
|
||||
);
|
||||
reply.error(Errno::ENOSYS);
|
||||
}
|
||||
diff --git a/src/ll/flags/lock_flags.rs b/src/ll/flags/lock_flags.rs
|
||||
new file mode 100644
|
||||
index 0000000..6f797d4
|
||||
--- /dev/null
|
||||
+++ b/src/ll/flags/lock_flags.rs
|
||||
@@ -0,0 +1,10 @@
|
||||
+use bitflags::bitflags;
|
||||
+
|
||||
+bitflags! {
|
||||
+ /// Flags describing a FUSE file-lock request.
|
||||
+ #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
+ pub struct LockFlags: u32 {
|
||||
+ /// The request is a BSD `flock` rather than a POSIX record lock.
|
||||
+ const FUSE_LK_FLOCK = 1 << 0;
|
||||
+ }
|
||||
+}
|
||||
diff --git a/src/ll/flags/mod.rs b/src/ll/flags/mod.rs
|
||||
index d9fbf92..54e4dc0 100644
|
||||
--- a/src/ll/flags/mod.rs
|
||||
+++ b/src/ll/flags/mod.rs
|
||||
@@ -8,6 +8,7 @@ pub(crate) mod getattr_flags;
|
||||
pub(crate) mod getattr_flags;
|
||||
pub(crate) mod init_flags;
|
||||
pub(crate) mod ioctl_flags;
|
||||
+pub(crate) mod lock_flags;
|
||||
pub(crate) mod poll_flags;
|
||||
pub(crate) mod read_flags;
|
||||
pub(crate) mod release_flags;
|
||||
diff --git a/src/ll/request.rs b/src/ll/request.rs
|
||||
index f572cc2..da80ff8 100644
|
||||
--- a/src/ll/request.rs
|
||||
+++ b/src/ll/request.rs
|
||||
@@ -264,6 +264,7 @@ mod operation {
|
||||
use crate::ll::flags::fsync_flags::FsyncFlags;
|
||||
use crate::ll::flags::getattr_flags::GetattrFlags;
|
||||
use crate::ll::flags::init_flags::InitFlags;
|
||||
+ use crate::ll::flags::lock_flags::LockFlags;
|
||||
use crate::ll::flags::read_flags::ReadFlags;
|
||||
use crate::ll::flags::release_flags::ReleaseFlags;
|
||||
use crate::ll::request::FileHandle;
|
||||
@@ -1170,6 +1171,9 @@ mod operation {
|
||||
pub(crate) fn lock_owner(&self) -> LockOwner {
|
||||
LockOwner(self.arg.owner)
|
||||
}
|
||||
+ pub(crate) fn lock_flags(&self) -> LockFlags {
|
||||
+ LockFlags::from_bits_retain(self.arg.lk_flags)
|
||||
+ }
|
||||
}
|
||||
|
||||
/// Acquire, modify or release a POSIX file lock.
|
||||
@@ -1200,6 +1204,9 @@ mod operation {
|
||||
pub(crate) fn lock_owner(&self) -> LockOwner {
|
||||
LockOwner(self.arg.owner)
|
||||
}
|
||||
+ pub(crate) fn lock_flags(&self) -> LockFlags {
|
||||
+ LockFlags::from_bits_retain(self.arg.lk_flags)
|
||||
+ }
|
||||
/// If true, block until the lock can be acquired (SETLKW).
|
||||
/// If false, return immediately if the lock cannot be acquired (SETLK).
|
||||
pub(crate) fn sleep(&self) -> bool {
|
||||
@@ -2219,6 +2226,8 @@ mod tests {
|
||||
mod tests {
|
||||
use std::ffi::OsStr;
|
||||
|
||||
+ use crate::LockFlags;
|
||||
+ use crate::ll::fuse_abi::{consts, fuse_opcode};
|
||||
use crate::ll::request::*;
|
||||
use crate::ll::test::AlignedData;
|
||||
|
||||
@@ -2314,6 +2323,33 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
+ #[test]
|
||||
+ fn lock_request_preserves_kind_flags() {
|
||||
+ let mut bytes = AlignedData([0_u8; 88]);
|
||||
+ bytes[0..4].copy_from_slice(&88_u32.to_ne_bytes());
|
||||
+ bytes[4..8].copy_from_slice(&(fuse_opcode::FUSE_GETLK as u32).to_ne_bytes());
|
||||
+ bytes[40..48].copy_from_slice(&11_u64.to_ne_bytes());
|
||||
+ bytes[48..56].copy_from_slice(&22_u64.to_ne_bytes());
|
||||
+ bytes[56..64].copy_from_slice(&3_u64.to_ne_bytes());
|
||||
+ bytes[64..72].copy_from_slice(&8_u64.to_ne_bytes());
|
||||
+ bytes[72..76].copy_from_slice(&libc::F_WRLCK.to_ne_bytes());
|
||||
+ bytes[76..80].copy_from_slice(&44_u32.to_ne_bytes());
|
||||
+ bytes[80..84].copy_from_slice(&consts::FUSE_LK_FLOCK.to_ne_bytes());
|
||||
+
|
||||
+ match AnyRequest::try_from(&bytes[..])
|
||||
+ .unwrap()
|
||||
+ .operation()
|
||||
+ .unwrap()
|
||||
+ {
|
||||
+ Operation::GetLk(lock) => {
|
||||
+ assert_eq!(lock.file_handle(), FileHandle(11));
|
||||
+ assert_eq!(lock.lock_owner(), LockOwner(22));
|
||||
+ assert_eq!(lock.lock_flags(), LockFlags::FUSE_LK_FLOCK);
|
||||
+ }
|
||||
+ _ => panic!("unexpected request operation"),
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
#[test]
|
||||
fn mknod() {
|
||||
let req = AnyRequest::try_from(&MKNOD_REQUEST[..]).unwrap();
|
||||
diff --git a/src/request.rs b/src/request.rs
|
||||
index 82cddf5..6bb5566 100644
|
||||
--- a/src/request.rs
|
||||
+++ b/src/request.rs
|
||||
@@ -395,6 +395,7 @@ impl AnyRequest<'_> {
|
||||
x.lock().range.1,
|
||||
x.lock().typ,
|
||||
x.lock().pid,
|
||||
+ x.lock_flags(),
|
||||
self.reply(),
|
||||
);
|
||||
}
|
||||
@@ -409,6 +410,7 @@ impl AnyRequest<'_> {
|
||||
x.lock().typ,
|
||||
x.lock().pid,
|
||||
x.sleep(),
|
||||
+ x.lock_flags(),
|
||||
self.reply(),
|
||||
);
|
||||
}
|
||||
|
|
@ -168,7 +168,7 @@
|
|||
"path_hex": "6275696c642f737061727365",
|
||||
"setup": ["isolated-sparse-source", "isolated-destination", "host-destination"],
|
||||
"actions": ["seek-hole-data", "punch-hole", "same-route-copy-range", "cross-route-copy-range"],
|
||||
"expect": ["sparse-extents-preserved", "same-route-bytes-match", "cross-route-EXDEV"]
|
||||
"expect": ["same-route-returned-count-equals-request", "same-route-source-and-destination-offsets-advance", "same-route-size-and-bytes-match", "cross-route-returned-count-equals-request", "cross-route-destination-range-equals-source", "cross-route-source-unchanged", "cross-route-unrelated-objects-unchanged", "cross-route-handles-remain-pinned", "cross-route-sparse-layout-not-asserted"]
|
||||
},
|
||||
{
|
||||
"id": "symlink-confinement",
|
||||
|
|
|
|||
88
tests/tooling/test_ignored_workspace_benchmark.py
Normal file
88
tests/tooling/test_ignored_workspace_benchmark.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"ignored_workspace_benchmark",
|
||||
ROOT / "tools/ignored_workspace_benchmark.py",
|
||||
)
|
||||
BENCHMARK = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(BENCHMARK)
|
||||
|
||||
|
||||
class BenchmarkAdmissionTests(unittest.TestCase):
|
||||
def test_storage_model_accepts_null_and_uses_last_nonblank_ancestor(self) -> None:
|
||||
blockdevices = {
|
||||
"blockdevices": [
|
||||
{
|
||||
"path": "/dev/nvme0n1",
|
||||
"type": "disk",
|
||||
"model": "Reference Storage",
|
||||
"tran": "nvme",
|
||||
"children": [
|
||||
{
|
||||
"path": "/dev/mapper/benchmark",
|
||||
"type": "crypt",
|
||||
"model": None,
|
||||
"tran": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
with (
|
||||
mock.patch.object(BENCHMARK, "version_output", return_value=json.dumps(blockdevices)),
|
||||
mock.patch.object(BENCHMARK.os.path, "realpath", side_effect=lambda path: path),
|
||||
):
|
||||
information = BENCHMARK.storage_information("/dev/mapper/benchmark")
|
||||
|
||||
self.assertEqual(information["storage_model"], "Reference Storage")
|
||||
self.assertEqual(information["storage_transport"], "nvme")
|
||||
self.assertEqual(information["storage_stack"], ["disk", "crypt"])
|
||||
|
||||
def test_current_cgroup_swap_max_reads_root_and_nested_v2_memberships(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
temporary = Path(temporary)
|
||||
cgroup = temporary / "cgroup"
|
||||
cgroup.mkdir()
|
||||
membership = temporary / "membership"
|
||||
|
||||
membership.write_text("0::/\n")
|
||||
(cgroup / "memory.swap.max").write_text("max\n")
|
||||
self.assertEqual(BENCHMARK.current_cgroup_swap_max(membership, cgroup), "max")
|
||||
|
||||
nested = cgroup / "user.slice/benchmark.scope"
|
||||
nested.mkdir(parents=True)
|
||||
(nested / "memory.swap.max").write_text("0\n")
|
||||
membership.write_text("0::/user.slice/benchmark.scope\n")
|
||||
self.assertEqual(BENCHMARK.current_cgroup_swap_max(membership, cgroup), "0")
|
||||
|
||||
def test_current_cgroup_swap_max_refuses_malformed_and_escaping_paths(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
temporary = Path(temporary)
|
||||
cgroup = temporary / "cgroup"
|
||||
cgroup.mkdir()
|
||||
membership = temporary / "membership"
|
||||
|
||||
for value in ("2:memory:/benchmark\n", "0::/../../outside\n", "0::/ok\n1:name:/other\n"):
|
||||
with self.subTest(value=value):
|
||||
membership.write_text(value)
|
||||
with self.assertRaises(AssertionError):
|
||||
BENCHMARK.current_cgroup_swap_max(membership, cgroup)
|
||||
|
||||
outside = temporary / "outside"
|
||||
outside.mkdir()
|
||||
(outside / "memory.swap.max").write_text("0\n")
|
||||
(cgroup / "escape").symlink_to(outside, target_is_directory=True)
|
||||
membership.write_text("0::/escape\n")
|
||||
with self.assertRaisesRegex(AssertionError, "escapes cgroup root"):
|
||||
BENCHMARK.current_cgroup_swap_max(membership, cgroup)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
943
tools/ignored_workspace_benchmark.py
Normal file
943
tools/ignored_workspace_benchmark.py
Normal file
|
|
@ -0,0 +1,943 @@
|
|||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import stat
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONTRACT_PATH = ROOT / "tests/fixtures/ignored-workspace-v1/benchmark.json"
|
||||
FIXTURE_CONTEXT = b"lazaret-ignored-workspace-fixture-v1\0"
|
||||
RESULT_CONTEXT = b"lazaret-ignored-workspace-result-v1\0"
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
def load_contract(path):
|
||||
contract = json.loads(Path(path).read_text())
|
||||
require(contract["schema"] == "lazaret-ignored-workspace-benchmark-v1", "unsupported benchmark contract")
|
||||
return contract
|
||||
|
||||
|
||||
def workload(contract, identifier):
|
||||
matches = [candidate for candidate in contract["workloads"] if candidate["id"] == identifier]
|
||||
require(len(matches) == 1, f"unknown workload: {identifier}")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def derived_bytes(seed, path, length=64):
|
||||
digest = hashlib.sha256(seed.encode() + b"/" + os.fsencode(path)).digest()
|
||||
return (digest * ((length + len(digest) - 1) // len(digest)))[:length]
|
||||
|
||||
|
||||
def write(path, data, mode=0o644):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
path.chmod(mode)
|
||||
|
||||
|
||||
def backing_paths(arguments):
|
||||
visible = Path(arguments.visible_root)
|
||||
if arguments.host_root is None:
|
||||
require(arguments.isolated_root is None, "isolated root requires a host root")
|
||||
visible.mkdir(parents=True, exist_ok=False)
|
||||
return visible, visible, visible
|
||||
host = Path(arguments.host_root)
|
||||
isolated = Path(arguments.isolated_root)
|
||||
require(host != isolated, "host and isolated roots must differ")
|
||||
host.mkdir(parents=True, exist_ok=False)
|
||||
isolated.mkdir(parents=True, exist_ok=False)
|
||||
return visible, host, isolated
|
||||
|
||||
|
||||
def initialize_routed_workspace(arguments, host):
|
||||
host.mkdir(parents=True, exist_ok=False)
|
||||
git(arguments, host, "init", "--quiet", "--initial-branch=benchmark")
|
||||
patterns = [f"/bench/metadata/d{index:03d}/" for index in range(1, 100, 2)]
|
||||
patterns.extend(["/bench/uv/.venv/", "/bench/cargo/target/"])
|
||||
write(host / ".gitignore", ("\n".join(patterns) + "\n").encode())
|
||||
|
||||
|
||||
def materialize_metadata(specification, host, isolated, direct):
|
||||
relative = Path("bench/metadata")
|
||||
seed = specification["fixture_seed"]
|
||||
for directory_index in range(100):
|
||||
directory = Path(f"d{directory_index:03d}")
|
||||
backing = host if direct or directory_index % 2 == 0 else isolated
|
||||
for file_index in range(100):
|
||||
file = directory / f"f{file_index:03d}"
|
||||
write(backing / relative / file, derived_bytes(seed, file))
|
||||
return relative
|
||||
|
||||
|
||||
def git(arguments, root, *values, check=True):
|
||||
environment = dict(os.environ)
|
||||
environment.update({
|
||||
"GIT_CONFIG_NOSYSTEM": "1",
|
||||
"GIT_AUTHOR_NAME": "Lazaret Benchmark",
|
||||
"GIT_AUTHOR_EMAIL": "[email protected]",
|
||||
"GIT_COMMITTER_NAME": "Lazaret Benchmark",
|
||||
"GIT_COMMITTER_EMAIL": "[email protected]",
|
||||
"GIT_AUTHOR_DATE": "1700000000 +0000",
|
||||
"GIT_COMMITTER_DATE": "1700000000 +0000",
|
||||
})
|
||||
return subprocess.run(
|
||||
[arguments.git, *values], cwd=root, env=environment, check=check,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
|
||||
def materialize_git(arguments, specification, host, isolated, direct):
|
||||
root = host / "bench/git"
|
||||
root.mkdir(parents=True)
|
||||
git(arguments, root, "init", "--quiet", "--initial-branch=benchmark")
|
||||
git(arguments, root, "config", "user.name", "Lazaret Benchmark")
|
||||
git(arguments, root, "config", "user.email", "[email protected]")
|
||||
write(root / ".gitignore", b"ignored/\n")
|
||||
seed = specification["fixture_seed"]
|
||||
for index in range(1000):
|
||||
path = Path("tracked") / f"t{index:04d}"
|
||||
write(root / path, derived_bytes(seed, path))
|
||||
git(arguments, root, "add", "--", ".gitignore", "tracked")
|
||||
git(arguments, root, "commit", "--quiet", "-m", "fixture")
|
||||
for index in range(500):
|
||||
ordinary = Path("ordinary") / f"u{index:04d}"
|
||||
ignored = Path("ignored") / f"i{index:04d}"
|
||||
write(root / ordinary, derived_bytes(seed, ordinary))
|
||||
if direct:
|
||||
write(root / ignored, b"isolated:" + derived_bytes(seed, ignored, 32))
|
||||
else:
|
||||
write(root / ignored, b"host-sentinel:" + derived_bytes(seed, ignored, 32))
|
||||
write(isolated / "bench/git" / ignored, b"isolated:" + derived_bytes(seed, ignored, 32))
|
||||
return Path("bench/git")
|
||||
|
||||
|
||||
def wheel_record_line(path, payload):
|
||||
digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=").decode()
|
||||
return f"{path},sha256={digest},{len(payload)}"
|
||||
|
||||
|
||||
def build_wheel(wheelhouse, package_index, seed):
|
||||
distribution = f"lazaret_bench_pkg_{package_index:02d}"
|
||||
module = distribution
|
||||
version = "1.0.0"
|
||||
dist_info = f"{distribution}-{version}.dist-info"
|
||||
module_path = f"{module}/__init__.py"
|
||||
value = hashlib.sha256(seed.encode() + distribution.replace("_", "-").encode()).hexdigest()
|
||||
module_bytes = f'VALUE = "{value}"\n'.encode()
|
||||
requirements = "" if package_index else "".join(
|
||||
f"Requires-Dist: lazaret-bench-pkg-{dependency:02d} == 1.0.0\n" for dependency in range(1, 32)
|
||||
)
|
||||
metadata = (
|
||||
"Metadata-Version: 2.1\n"
|
||||
f"Name: lazaret-bench-pkg-{package_index:02d}\n"
|
||||
f"Version: {version}\n"
|
||||
f"{requirements}\n"
|
||||
).encode()
|
||||
wheel = b"Wheel-Version: 1.0\nGenerator: lazaret\nRoot-Is-Purelib: true\nTag: py3-none-any\n"
|
||||
entries = {
|
||||
module_path: module_bytes,
|
||||
f"{dist_info}/METADATA": metadata,
|
||||
f"{dist_info}/WHEEL": wheel,
|
||||
}
|
||||
record_path = f"{dist_info}/RECORD"
|
||||
records = [wheel_record_line(path, payload) for path, payload in sorted(entries.items())]
|
||||
records.append(f"{record_path},,")
|
||||
entries[record_path] = ("\n".join(records) + "\n").encode()
|
||||
destination = wheelhouse / f"{distribution}-{version}-py3-none-any.whl"
|
||||
with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
|
||||
for path, payload in sorted(entries.items()):
|
||||
info = zipfile.ZipInfo(path, (2020, 1, 1, 0, 0, 0))
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o100644 << 16
|
||||
archive.writestr(info, payload)
|
||||
|
||||
|
||||
def materialize_uv(arguments, specification, host, isolated, direct):
|
||||
root = host / "bench/uv"
|
||||
wheelhouse = root / "wheelhouse"
|
||||
wheelhouse.mkdir(parents=True)
|
||||
for index in range(32):
|
||||
build_wheel(wheelhouse, index, specification["fixture_seed"])
|
||||
venv = root / ".venv" if direct else isolated / "bench/uv/.venv"
|
||||
venv.parent.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(
|
||||
[arguments.uv, "venv", "--offline", "--no-project", "--python", sys.executable, venv],
|
||||
check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
)
|
||||
return Path("bench/uv")
|
||||
|
||||
|
||||
def cargo_package_name(index):
|
||||
return f"lazaret-bench-crate-{index:02d}"
|
||||
|
||||
|
||||
def materialize_cargo(arguments, specification, host, isolated, direct):
|
||||
root = host / "bench/cargo"
|
||||
root.mkdir(parents=True)
|
||||
members = [cargo_package_name(index) for index in range(32)]
|
||||
write(root / "Cargo.toml", ("[workspace]\nresolver = \"2\"\nmembers = [\n" + "".join(f' "{member}",\n' for member in members) + "]\n").encode())
|
||||
seed = specification["fixture_seed"].encode()
|
||||
for index, name in enumerate(members):
|
||||
package = root / name
|
||||
dependencies = ""
|
||||
if index == 0:
|
||||
dependencies = "\n[dependencies]\n" + "".join(
|
||||
f'{cargo_package_name(value).replace("-", "_")} = {{ package = "{cargo_package_name(value)}", path = "../{cargo_package_name(value)}", version = "=1.0.0" }}\n'
|
||||
for value in range(1, 32)
|
||||
)
|
||||
write(package / "Cargo.toml", (f'[package]\nname = "{name}"\nversion = "1.0.0"\nedition = "2024"\n{dependencies}').encode())
|
||||
if index == 0:
|
||||
expression = " ^ ".join(f"{cargo_package_name(value).replace('-', '_')}::VALUE" for value in range(1, 32))
|
||||
write(package / "src/main.rs", f"fn main() {{ println!(\"{{}}\", {expression}); }}\n".encode())
|
||||
else:
|
||||
value = int.from_bytes(hashlib.sha256(seed + name.encode()).digest()[:8], "big")
|
||||
write(package / "src/lib.rs", f"pub const VALUE: u64 = {value};\n".encode())
|
||||
subprocess.run(
|
||||
[arguments.cargo, "generate-lockfile", "--offline"], cwd=root, check=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
)
|
||||
target = root / "target" if direct else isolated / "bench/cargo/target"
|
||||
target.mkdir(parents=True)
|
||||
return Path("bench/cargo")
|
||||
|
||||
|
||||
def materialize(arguments):
|
||||
contract = load_contract(arguments.contract)
|
||||
specification = workload(contract, arguments.workload)
|
||||
visible = Path(arguments.visible_root)
|
||||
if arguments.host_root is None:
|
||||
visible, host, isolated = backing_paths(arguments)
|
||||
else:
|
||||
require(arguments.isolated_root is not None, "routed materialization requires an isolated root")
|
||||
host = Path(arguments.host_root)
|
||||
isolated = Path(arguments.isolated_root)
|
||||
require(host != isolated, "host and isolated roots must differ")
|
||||
initialize_routed_workspace(arguments, host)
|
||||
isolated.mkdir(parents=True, exist_ok=False)
|
||||
direct = arguments.host_root is None
|
||||
if specification["category"] == "metadata":
|
||||
relative = materialize_metadata(specification, host, isolated, direct)
|
||||
elif specification["category"] == "git":
|
||||
relative = materialize_git(arguments, specification, host, isolated, direct)
|
||||
elif specification["category"] == "package-manager":
|
||||
relative = materialize_uv(arguments, specification, host, isolated, direct)
|
||||
elif specification["category"] == "build":
|
||||
relative = materialize_cargo(arguments, specification, host, isolated, direct)
|
||||
else:
|
||||
raise AssertionError("unsupported workload category")
|
||||
manifest = {
|
||||
"schema": "lazaret-ignored-workspace-materialization-v1",
|
||||
"workload": arguments.workload,
|
||||
"placement": "direct" if direct else "routed",
|
||||
"visible_workload_root": os.fspath(visible / relative),
|
||||
}
|
||||
Path(arguments.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(arguments.output).write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def fixture_identity(root, excluded_roots=()):
|
||||
records = []
|
||||
excluded = {os.fsencode(value) for value in excluded_roots}
|
||||
for directory, directories, files in os.walk(root, topdown=True, followlinks=False):
|
||||
directories.sort(key=os.fsencode)
|
||||
files.sort(key=os.fsencode)
|
||||
relative_directory = Path(directory).relative_to(root)
|
||||
if relative_directory == Path("."):
|
||||
directories[:] = [name for name in directories if os.fsencode(name) not in excluded]
|
||||
files[:] = [name for name in files if os.fsencode(name) not in excluded]
|
||||
for name in [*directories, *files]:
|
||||
path = Path(directory) / name
|
||||
relative = relative_directory / name
|
||||
metadata = os.lstat(path)
|
||||
if stat.S_ISDIR(metadata.st_mode):
|
||||
kind, payload = 1, b""
|
||||
elif stat.S_ISREG(metadata.st_mode):
|
||||
kind, payload = 2, path.read_bytes()
|
||||
elif stat.S_ISLNK(metadata.st_mode):
|
||||
kind, payload = 3, os.fsencode(os.readlink(path))
|
||||
else:
|
||||
raise AssertionError(f"unsupported fixture entry: {relative}")
|
||||
records.append((os.fsencode(relative), kind, metadata.st_mode & 0o7777, payload))
|
||||
records.sort(key=lambda record: record[0])
|
||||
stream = bytearray(FIXTURE_CONTEXT)
|
||||
stream.extend(struct.pack(">I", len(records)))
|
||||
for path, kind, mode, payload in records:
|
||||
stream.extend(struct.pack(">I", len(path)))
|
||||
stream.extend(path)
|
||||
stream.extend(struct.pack(">BIQ", kind, mode, len(payload)))
|
||||
stream.extend(hashlib.sha256(payload).digest())
|
||||
return hashlib.sha256(stream).digest()
|
||||
|
||||
|
||||
def workload_commands(arguments, identifier, root):
|
||||
if identifier == "metadata-10000":
|
||||
return [[arguments.metadata_benchmark, "metadata", "--root", root, "--bytewise-order", "--statx-mask", "basic-stats"]]
|
||||
if identifier == "git-2000-status-add-reset":
|
||||
return [
|
||||
[arguments.git, "status", "--porcelain=v2", "--untracked-files=all"],
|
||||
[arguments.git, "add", "--", "ordinary"],
|
||||
[arguments.git, "reset", "--quiet", "--mixed"],
|
||||
[arguments.git, "status", "--porcelain=v2", "--untracked-files=all"],
|
||||
]
|
||||
if identifier == "uv-offline-32-wheels":
|
||||
return [[arguments.uv, "pip", "install", "--offline", "--no-index", "--find-links", root / "wheelhouse", "--python", root / ".venv/bin/python", "lazaret-bench-pkg-00==1.0.0"]]
|
||||
if identifier == "cargo-offline-32-crates":
|
||||
return [[arguments.cargo, "check", "--offline", "--locked", "--workspace", "--target-dir", root / "target"]]
|
||||
raise AssertionError(f"unsupported workload: {identifier}")
|
||||
|
||||
|
||||
def workload_state(arguments, specification, root):
|
||||
if specification["category"] == "git":
|
||||
index = root / ".git/index"
|
||||
state = {"index_sha256": hashlib.sha256(index.read_bytes()).hexdigest()}
|
||||
if arguments.host_root is not None:
|
||||
sentinels = Path(arguments.host_root) / "bench/git/ignored"
|
||||
state["ignored_host_sentinels_identity"] = fixture_identity(sentinels).hex()
|
||||
return state
|
||||
if specification["category"] == "build":
|
||||
return {
|
||||
"source_identity": fixture_identity(root, ("target",)).hex(),
|
||||
"lock_sha256": hashlib.sha256((root / "Cargo.lock").read_bytes()).hexdigest(),
|
||||
}
|
||||
if specification["category"] == "package-manager":
|
||||
candidates = list((root / ".venv").glob("lib/python*/site-packages"))
|
||||
if not candidates:
|
||||
return {"installed_distribution_count": 0, "installed_values_identity": None}
|
||||
site_packages = candidates[0]
|
||||
distributions = sorted(site_packages.glob("lazaret_bench_pkg_*-1.0.0.dist-info"))
|
||||
values = []
|
||||
for index in range(32):
|
||||
module = site_packages / f"lazaret_bench_pkg_{index:02d}/__init__.py"
|
||||
if module.exists():
|
||||
values.append(module.read_bytes())
|
||||
return {
|
||||
"installed_distribution_count": len(distributions),
|
||||
"installed_values_identity": hashlib.sha256(b"".join(values)).hexdigest() if len(values) == 32 else None,
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def verify_uv_installation(root, seed):
|
||||
site_packages = next((root / ".venv").glob("lib/python*/site-packages"), None)
|
||||
require(site_packages is not None, "installed virtual environment has no site-packages")
|
||||
distributions = list(site_packages.glob("lazaret_bench_pkg_*-1.0.0.dist-info"))
|
||||
require(len(distributions) == 32, "offline installation did not install all 32 distributions")
|
||||
for index in range(32):
|
||||
normalized = f"lazaret-bench-pkg-{index:02d}"
|
||||
expected = hashlib.sha256(seed.encode() + normalized.encode()).hexdigest()
|
||||
module = site_packages / normalized.replace("-", "_") / "__init__.py"
|
||||
require(module.read_text() == f'VALUE = "{expected}"\n', f"installed module {normalized} differs")
|
||||
|
||||
|
||||
def invoke(arguments, specification, capture):
|
||||
root = Path(arguments.root)
|
||||
initial_identity = fixture_identity(root)
|
||||
initial_state = workload_state(arguments, specification, root)
|
||||
records = []
|
||||
started = time.clock_gettime_ns(time.CLOCK_MONOTONIC_RAW)
|
||||
timeout_seconds = 4 * specification["absolute_median_ms"] / 1000
|
||||
for argv in workload_commands(arguments, arguments.workload, root):
|
||||
elapsed_seconds = (time.clock_gettime_ns(time.CLOCK_MONOTONIC_RAW) - started) / 1_000_000_000
|
||||
remaining = timeout_seconds - elapsed_seconds
|
||||
require(remaining > 0, "workload exceeded its sample timeout")
|
||||
result = subprocess.run(
|
||||
[os.fspath(value) for value in argv], cwd=root, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, timeout=remaining,
|
||||
)
|
||||
require(result.returncode == 0, f"workload command failed: {result.stderr.decode(errors='replace')}")
|
||||
if capture:
|
||||
records.append({"status": result.returncode, "stdout": result.stdout, "stderr": result.stderr})
|
||||
elapsed = time.clock_gettime_ns(time.CLOCK_MONOTONIC_RAW) - started
|
||||
post_identity = fixture_identity(root)
|
||||
post_state = workload_state(arguments, specification, root)
|
||||
if specification["category"] == "package-manager":
|
||||
verify_uv_installation(root, specification["fixture_seed"])
|
||||
if not capture:
|
||||
return None
|
||||
stream = bytearray(RESULT_CONTEXT)
|
||||
workload_bytes = arguments.workload.encode()
|
||||
stream.extend(struct.pack(">I", len(workload_bytes)))
|
||||
stream.extend(workload_bytes)
|
||||
stream.extend(initial_identity)
|
||||
stream.extend(struct.pack(">I", len(records)))
|
||||
for record in records:
|
||||
stream.extend(struct.pack(">iQ", record["status"], len(record["stdout"])))
|
||||
stream.extend(record["stdout"])
|
||||
stream.extend(struct.pack(">Q", len(record["stderr"])))
|
||||
stream.extend(record["stderr"])
|
||||
stream.extend(post_identity)
|
||||
return {
|
||||
"elapsed_ns": elapsed,
|
||||
"initial_fixture_identity": initial_identity.hex(),
|
||||
"post_fixture_identity": post_identity.hex(),
|
||||
"initial_workload_state": initial_state,
|
||||
"post_workload_state": post_state,
|
||||
"result_identity": hashlib.sha256(stream).hexdigest(),
|
||||
"commands": [
|
||||
{
|
||||
"status": record["status"],
|
||||
"stdout_base64": base64.b64encode(record["stdout"]).decode(),
|
||||
"stderr_base64": base64.b64encode(record["stderr"]).decode(),
|
||||
}
|
||||
for record in records
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def cpu_snapshot():
|
||||
fields = Path("/proc/stat").read_text().splitlines()[0].split()[1:]
|
||||
usage = resource.getrusage(resource.RUSAGE_SELF)
|
||||
children = resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
return {
|
||||
"system_ticks": sum(int(value) for value in fields),
|
||||
"idle_ticks": int(fields[3]) + int(fields[4]),
|
||||
"owned_seconds": usage.ru_utime + usage.ru_stime + children.ru_utime + children.ru_stime,
|
||||
}
|
||||
|
||||
|
||||
def memory_pressure_snapshot():
|
||||
pressure = {}
|
||||
path = Path("/sys/fs/cgroup/memory.pressure")
|
||||
if path.exists():
|
||||
for line in path.read_text().splitlines():
|
||||
category, *fields = line.split()
|
||||
pressure[category] = {key: int(value) for key, value in (field.split("=") for field in fields) if key == "total"}["total"]
|
||||
return pressure
|
||||
|
||||
|
||||
def run_sample(arguments):
|
||||
contract = load_contract(arguments.contract)
|
||||
specification = workload(contract, arguments.workload)
|
||||
require(arguments.preparation_state in {"unprimed", "primed"}, "invalid preparation state")
|
||||
sequence_started_ns = time.clock_gettime_ns(time.CLOCK_MONOTONIC_RAW)
|
||||
if arguments.preparation_state == "primed":
|
||||
invoke(arguments, specification, False)
|
||||
cpu_before = cpu_snapshot()
|
||||
pressure_before = memory_pressure_snapshot()
|
||||
sample = invoke(arguments, specification, True)
|
||||
pressure_after = memory_pressure_snapshot()
|
||||
cpu_after = cpu_snapshot()
|
||||
elapsed_seconds = sample["elapsed_ns"] / 1_000_000_000
|
||||
ticks_per_second = os.sysconf("SC_CLK_TCK")
|
||||
total_busy_seconds = (
|
||||
(cpu_after["system_ticks"] - cpu_before["system_ticks"])
|
||||
- (cpu_after["idle_ticks"] - cpu_before["idle_ticks"])
|
||||
) / ticks_per_second
|
||||
owned_seconds = max(0.0, cpu_after["owned_seconds"] - cpu_before["owned_seconds"])
|
||||
external_seconds = max(0.0, total_busy_seconds - owned_seconds)
|
||||
pressure_delta = {key: pressure_after.get(key, 0) - pressure_before.get(key, 0) for key in pressure_after.keys() | pressure_before.keys()}
|
||||
sample.update({
|
||||
"schema": "lazaret-ignored-workspace-benchmark-sample-v1",
|
||||
"workload": arguments.workload,
|
||||
"placement": arguments.placement,
|
||||
"preparation_state": arguments.preparation_state,
|
||||
"independent_run": arguments.independent_run,
|
||||
"sample_kind": arguments.sample_kind,
|
||||
"sample_index": arguments.sample_index,
|
||||
"pair_index": arguments.pair_index,
|
||||
"fixture_id": arguments.fixture_id,
|
||||
"sequence_started_ns": sequence_started_ns,
|
||||
"background_load": {
|
||||
"external_cpu_seconds": external_seconds,
|
||||
"reference_elapsed_seconds": elapsed_seconds,
|
||||
"memory_pressure_total_delta": pressure_delta,
|
||||
},
|
||||
})
|
||||
Path(arguments.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(arguments.output).write_text(json.dumps(sample, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def plan(arguments):
|
||||
contract = load_contract(arguments.contract)
|
||||
sampling = contract["sampling"]
|
||||
entries = []
|
||||
for independent_run in range(sampling["independent_runs"]):
|
||||
for specification in contract["workloads"]:
|
||||
for preparation_state in specification["preparation_states"]:
|
||||
for sample_kind, count in (
|
||||
("warmup", sampling["warmup_samples_per_cell"]),
|
||||
("measured", sampling["measured_samples_per_cell"]),
|
||||
):
|
||||
for sample_index in range(count):
|
||||
pair_index = len(entries) // 2
|
||||
placements = ["direct", "routed"] if pair_index % 2 == 0 else ["routed", "direct"]
|
||||
for placement in placements:
|
||||
entries.append({
|
||||
"workload": specification["id"],
|
||||
"preparation_state": preparation_state,
|
||||
"independent_run": independent_run,
|
||||
"sample_kind": sample_kind,
|
||||
"sample_index": sample_index,
|
||||
"pair_index": pair_index,
|
||||
"placement": placement,
|
||||
"fixture_id": f"run-{independent_run}-{specification['id']}-{preparation_state}-{sample_kind}-{sample_index}-{placement}",
|
||||
})
|
||||
output = {"schema": "lazaret-ignored-workspace-benchmark-plan-v1", "samples": entries}
|
||||
Path(arguments.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(arguments.output).write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def version_output(argv):
|
||||
result = subprocess.run(argv, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def cpu_information():
|
||||
values = {}
|
||||
processors = []
|
||||
current = {}
|
||||
for line in Path("/proc/cpuinfo").read_text().splitlines() + [""]:
|
||||
if not line:
|
||||
if current:
|
||||
processors.append(current)
|
||||
current = {}
|
||||
continue
|
||||
key, separator, value = line.partition(":")
|
||||
if separator:
|
||||
current[key.strip()] = value.strip()
|
||||
allowed = sorted(os.sched_getaffinity(0))
|
||||
topology = []
|
||||
for cpu in allowed:
|
||||
base = Path(f"/sys/devices/system/cpu/cpu{cpu}")
|
||||
topology.append({
|
||||
"logical_cpu": cpu,
|
||||
"package": int((base / "topology/physical_package_id").read_text()),
|
||||
"core": int((base / "topology/core_id").read_text()),
|
||||
"governor": (base / "cpufreq/scaling_governor").read_text().strip(),
|
||||
})
|
||||
first = processors[0]
|
||||
values["cpu_model"] = first.get("model name", "")
|
||||
values["cpu_microcode"] = first.get("microcode", "")
|
||||
values["allocated_cpu_topology"] = topology
|
||||
return values
|
||||
|
||||
|
||||
def decode_mount_path(value):
|
||||
return value.replace("\\040", " ").replace("\\011", "\t").replace("\\012", "\n").replace("\\134", "\\")
|
||||
|
||||
|
||||
def mount_information(root):
|
||||
root = Path(root).resolve()
|
||||
candidates = []
|
||||
for line in Path("/proc/self/mountinfo").read_text().splitlines():
|
||||
left, right = line.split(" - ", 1)
|
||||
fields = left.split()
|
||||
mountpoint = Path(decode_mount_path(fields[4]))
|
||||
try:
|
||||
root.relative_to(mountpoint)
|
||||
except ValueError:
|
||||
continue
|
||||
right_fields = right.split()
|
||||
candidates.append((len(mountpoint.parts), {
|
||||
"mountpoint": os.fspath(mountpoint),
|
||||
"source": decode_mount_path(right_fields[1]),
|
||||
"filesystem_type": right_fields[0],
|
||||
"filesystem_mount_options": sorted(set(fields[5].split(",")) | set(right_fields[2].split(","))),
|
||||
}))
|
||||
require(candidates, "benchmark root has no mountinfo entry")
|
||||
return max(candidates, key=lambda candidate: candidate[0])[1]
|
||||
|
||||
|
||||
def storage_information(source):
|
||||
require(source.startswith("/dev/"), "qualification storage is not a local block device")
|
||||
tree = json.loads(version_output(["lsblk", "--json", "--output", "PATH,TYPE,MODEL,TRAN"]))["blockdevices"]
|
||||
flattened = []
|
||||
|
||||
def visit(device, parents):
|
||||
flattened.append((device, parents))
|
||||
for child in device.get("children", []):
|
||||
visit(child, [*parents, device])
|
||||
|
||||
for device in tree:
|
||||
visit(device, [])
|
||||
source_real = os.path.realpath(source)
|
||||
matches = [(device, parents) for device, parents in flattened if os.path.realpath(device.get("path", "")) == source_real]
|
||||
require(matches, f"cannot identify storage device for {source}")
|
||||
device, parents = matches[0]
|
||||
stack = [*parents, device]
|
||||
models = [model.strip() for entry in stack if (model := entry.get("model")) and model.strip()]
|
||||
transports = [entry.get("tran", "") for entry in stack if entry.get("tran")]
|
||||
return {
|
||||
"storage_model": models[-1] if models else "unreported",
|
||||
"storage_transport": transports[-1] if transports else "unreported",
|
||||
"storage_stack": [entry.get("type", "unknown") for entry in stack],
|
||||
}
|
||||
|
||||
|
||||
def current_cgroup_swap_max(
|
||||
membership_path=Path("/proc/self/cgroup"),
|
||||
cgroup_root=Path("/sys/fs/cgroup"),
|
||||
):
|
||||
memberships = Path(membership_path).read_text().splitlines()
|
||||
require(len(memberships) == 1, "current cgroup data is not unified cgroup v2")
|
||||
fields = memberships[0].split(":", 2)
|
||||
require(
|
||||
len(fields) == 3 and fields[0] == "0" and fields[1] == "" and fields[2].startswith("/"),
|
||||
"current cgroup data is not unified cgroup v2",
|
||||
)
|
||||
membership = PurePosixPath(fields[2])
|
||||
require(
|
||||
all(component not in {"", ".", ".."} for component in membership.parts[1:]),
|
||||
"current cgroup path is malformed",
|
||||
)
|
||||
root = Path(cgroup_root).resolve(strict=True)
|
||||
directory = root.joinpath(*membership.parts[1:]).resolve(strict=True)
|
||||
try:
|
||||
directory.relative_to(root)
|
||||
except ValueError as error:
|
||||
raise AssertionError("current cgroup path escapes cgroup root") from error
|
||||
value = (directory / "memory.swap.max").read_text().strip()
|
||||
require(value == "max" or value.isascii() and value.isdigit(), "invalid memory.swap.max value")
|
||||
return value
|
||||
|
||||
|
||||
def qualify(arguments):
|
||||
mount = mount_information(arguments.root)
|
||||
filesystem = os.statvfs(arguments.root)
|
||||
cpu = cpu_information()
|
||||
memory_kib = int(next(line.split()[1] for line in Path("/proc/meminfo").read_text().splitlines() if line.startswith("MemTotal:")))
|
||||
memory_max_path = Path("/sys/fs/cgroup/memory.max")
|
||||
memory_max = memory_max_path.read_text().strip() if memory_max_path.exists() else "max"
|
||||
memory_bytes = memory_kib * 1024
|
||||
if memory_max != "max":
|
||||
memory_bytes = min(memory_bytes, int(memory_max))
|
||||
swap_max = current_cgroup_swap_max()
|
||||
xfs = version_output([arguments.xfs_info, arguments.root]) if mount["filesystem_type"].lower() == "xfs" else ""
|
||||
result = {
|
||||
"schema": "lazaret-ignored-workspace-qualification-v1",
|
||||
"architecture": platform.machine(),
|
||||
"kernel_release": platform.release(),
|
||||
**cpu,
|
||||
"memory_bytes": memory_bytes,
|
||||
"benchmark_cgroup_swap_max": swap_max,
|
||||
**storage_information(mount["source"]),
|
||||
"storage_kind": "local-block-device",
|
||||
"storage_network_backed": False,
|
||||
"storage_copy_on_write": "reflink=0" not in xfs,
|
||||
"filesystem_type": mount["filesystem_type"].upper(),
|
||||
"filesystem_block_bytes": filesystem.f_frsize,
|
||||
"filesystem_mount_options": mount["filesystem_mount_options"],
|
||||
"fuse_userspace_version": version_output([arguments.fusermount, "--version"]),
|
||||
"runsc_version": version_output([arguments.runsc, "--version"]),
|
||||
"git_version": version_output([arguments.git, "--version"]),
|
||||
"uv_version": version_output([arguments.uv, "--version"]),
|
||||
"rust_version": version_output([arguments.rustc, "--version"]),
|
||||
}
|
||||
Path(arguments.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(arguments.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def kernel_at_least(release, minimum):
|
||||
numbers = []
|
||||
for component in release.split("."):
|
||||
digits = "".join(character for character in component if character.isdigit())
|
||||
if not digits:
|
||||
break
|
||||
numbers.append(int(digits))
|
||||
if len(numbers) == 2:
|
||||
break
|
||||
return tuple(numbers) >= minimum
|
||||
|
||||
|
||||
def validate_qualification(contract, qualification, failures):
|
||||
expected_fields = contract["qualification_environment"]["qualification_result_fields"]
|
||||
missing = [field for field in expected_fields if field not in qualification]
|
||||
if missing:
|
||||
failures.append(f"qualification result omits fields: {missing}")
|
||||
return
|
||||
if qualification.get("architecture") != "x86_64":
|
||||
failures.append("qualification architecture is not x86_64")
|
||||
if not kernel_at_least(qualification["kernel_release"], (6, 9)):
|
||||
failures.append("qualification kernel is older than Linux 6.9")
|
||||
topology = qualification["allocated_cpu_topology"]
|
||||
physical = {(cpu["package"], cpu["core"]) for cpu in topology}
|
||||
if len(physical) < 8 or len(physical) != len(topology):
|
||||
failures.append("CPU allocation is not one hardware thread from at least eight physical cores")
|
||||
if any(cpu["governor"] != "performance" for cpu in topology):
|
||||
failures.append("an allocated CPU is not using the performance governor")
|
||||
if qualification["memory_bytes"] < 17_179_869_184:
|
||||
failures.append("qualification memory is below 16 GiB")
|
||||
if qualification.get("benchmark_cgroup_swap_max") != "0":
|
||||
failures.append("swap is not disabled for the benchmark cgroup")
|
||||
if qualification.get("storage_kind") != "local-block-device" or qualification.get("storage_network_backed") is not False:
|
||||
failures.append("qualification storage is not a local non-network block device")
|
||||
if qualification.get("storage_copy_on_write") is not False:
|
||||
failures.append("qualification storage has copy-on-write enabled")
|
||||
if qualification["filesystem_type"] != "XFS" or qualification["filesystem_block_bytes"] != 4096:
|
||||
failures.append("qualification filesystem is not 4096-byte XFS")
|
||||
mount_options = set(qualification["filesystem_mount_options"])
|
||||
if not {"relatime", "inode64"}.issubset(mount_options):
|
||||
failures.append("qualification filesystem omits relatime or inode64")
|
||||
if mount_options & {"quota", "usrquota", "grpquota", "prjquota", "uquota", "gquota", "pquota"}:
|
||||
failures.append("qualification filesystem enables quota")
|
||||
|
||||
|
||||
def lower_median(values):
|
||||
ordered = sorted(values)
|
||||
require(ordered, "cannot calculate an empty median")
|
||||
return ordered[len(ordered) // 2]
|
||||
|
||||
|
||||
def relative_mad(values):
|
||||
median = lower_median(values)
|
||||
require(median > 0, "non-positive median")
|
||||
return lower_median([abs(value - median) for value in values]) / median
|
||||
|
||||
|
||||
def decode_command_records(sample):
|
||||
return [
|
||||
{
|
||||
"status": record["status"],
|
||||
"stdout": base64.b64decode(record["stdout_base64"], validate=True),
|
||||
"stderr": base64.b64decode(record["stderr_base64"], validate=True),
|
||||
}
|
||||
for record in sample["commands"]
|
||||
]
|
||||
|
||||
|
||||
def validate_result_identity(sample, failures):
|
||||
try:
|
||||
records = decode_command_records(sample)
|
||||
stream = bytearray(RESULT_CONTEXT)
|
||||
identifier = sample["workload"].encode()
|
||||
stream.extend(struct.pack(">I", len(identifier)))
|
||||
stream.extend(identifier)
|
||||
stream.extend(bytes.fromhex(sample["initial_fixture_identity"]))
|
||||
stream.extend(struct.pack(">I", len(records)))
|
||||
for record in records:
|
||||
stream.extend(struct.pack(">iQ", record["status"], len(record["stdout"])))
|
||||
stream.extend(record["stdout"])
|
||||
stream.extend(struct.pack(">Q", len(record["stderr"])))
|
||||
stream.extend(record["stderr"])
|
||||
stream.extend(bytes.fromhex(sample["post_fixture_identity"]))
|
||||
require(hashlib.sha256(stream).hexdigest() == sample["result_identity"], "result identity differs")
|
||||
require(all(record["status"] == 0 for record in records), "a command status is nonzero")
|
||||
return records
|
||||
except (AssertionError, KeyError, ValueError, TypeError) as error:
|
||||
failures.append(f"invalid sample {sample.get('fixture_id', '<unknown>')}: {error}")
|
||||
return []
|
||||
|
||||
|
||||
def validate_workload_oracle(specification, sample, records, failures):
|
||||
prefix = sample.get("fixture_id", "sample")
|
||||
if not records:
|
||||
return
|
||||
if specification["category"] == "metadata":
|
||||
try:
|
||||
output = json.loads(records[0]["stdout"])
|
||||
require(output["regular_files"] == 10_000, "metadata file count differs")
|
||||
require(output["content_bytes"] == 640_000, "metadata byte count differs")
|
||||
require(output["fixture_identity"] == sample["initial_fixture_identity"], "metadata fixture identity differs")
|
||||
except (AssertionError, KeyError, ValueError, TypeError) as error:
|
||||
failures.append(f"{prefix}: metadata oracle failed: {error}")
|
||||
elif specification["category"] == "git":
|
||||
if len(records) != 4 or records[0]["stdout"] != records[3]["stdout"]:
|
||||
failures.append(f"{prefix}: Git status streams differ")
|
||||
initial = sample.get("initial_workload_state", {})
|
||||
post = sample.get("post_workload_state", {})
|
||||
if initial.get("index_sha256") != post.get("index_sha256"):
|
||||
failures.append(f"{prefix}: Git index did not return to its initial identity")
|
||||
if sample["placement"] == "routed" and initial.get("ignored_host_sentinels_identity") != post.get("ignored_host_sentinels_identity"):
|
||||
failures.append(f"{prefix}: ignored host sentinels changed")
|
||||
elif specification["category"] == "build":
|
||||
if sample.get("initial_workload_state") != sample.get("post_workload_state"):
|
||||
failures.append(f"{prefix}: Cargo changed the lockfile or source tree")
|
||||
elif specification["category"] == "package-manager":
|
||||
post = sample.get("post_workload_state", {})
|
||||
if post.get("installed_distribution_count") != 32 or post.get("installed_values_identity") is None:
|
||||
failures.append(f"{prefix}: offline installation oracle differs")
|
||||
|
||||
|
||||
def verify_results(arguments):
|
||||
contract = load_contract(arguments.contract)
|
||||
samples = [json.loads(path.read_text()) for path in sorted(Path(arguments.samples).glob("*.json"))]
|
||||
sampling = contract["sampling"]
|
||||
failures = []
|
||||
qualification = json.loads(Path(arguments.qualification).read_text())
|
||||
validate_qualification(contract, qualification, failures)
|
||||
seen = set()
|
||||
fixture_ids = set()
|
||||
for sample in samples:
|
||||
if sample.get("schema") != "lazaret-ignored-workspace-benchmark-sample-v1":
|
||||
failures.append(f"unsupported sample schema: {sample.get('schema')}")
|
||||
key = tuple(sample.get(field) for field in (
|
||||
"workload", "preparation_state", "independent_run", "sample_kind",
|
||||
"sample_index", "pair_index", "placement",
|
||||
))
|
||||
if key in seen:
|
||||
failures.append(f"duplicate sample coordinate: {key}")
|
||||
seen.add(key)
|
||||
fixture_id = sample.get("fixture_id")
|
||||
if fixture_id in fixture_ids:
|
||||
failures.append(f"reused fixture identity: {fixture_id}")
|
||||
fixture_ids.add(fixture_id)
|
||||
records = validate_result_identity(sample, failures)
|
||||
try:
|
||||
validate_workload_oracle(workload(contract, sample["workload"]), sample, records, failures)
|
||||
except (AssertionError, KeyError) as error:
|
||||
failures.append(f"invalid sample workload: {error}")
|
||||
cells = []
|
||||
for specification in contract["workloads"]:
|
||||
for state in specification["preparation_states"]:
|
||||
run_medians = {"direct": [], "routed": [], "ratio": []}
|
||||
for run in range(sampling["independent_runs"]):
|
||||
matching = [sample for sample in samples if sample.get("workload") == specification["id"] and sample.get("preparation_state") == state and sample.get("independent_run") == run]
|
||||
for kind, expected_count in (("warmup", sampling["warmup_samples_per_cell"]), ("measured", sampling["measured_samples_per_cell"])):
|
||||
for placement in ("direct", "routed"):
|
||||
observed_count = sum(sample.get("sample_kind") == kind and sample.get("placement") == placement for sample in matching)
|
||||
if observed_count != expected_count:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: expected {expected_count} {kind} {placement} samples")
|
||||
for pair_index in {sample.get("pair_index") for sample in matching}:
|
||||
pair = [sample for sample in matching if sample.get("pair_index") == pair_index]
|
||||
if len(pair) == 2 and {sample.get("placement") for sample in pair} == {"direct", "routed"}:
|
||||
ordered = sorted(pair, key=lambda sample: sample.get("sequence_started_ns", 0))
|
||||
expected_first = "direct" if pair_index % 2 == 0 else "routed"
|
||||
if ordered[0].get("placement") != expected_first:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}/pair-{pair_index}: placement order differs")
|
||||
if pair[0].get("initial_fixture_identity") != pair[1].get("initial_fixture_identity"):
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}/pair-{pair_index}: initial fixtures differ")
|
||||
direct_sample = next(sample for sample in pair if sample["placement"] == "direct")
|
||||
routed_sample = next(sample for sample in pair if sample["placement"] == "routed")
|
||||
if specification["category"] in {"metadata", "git"}:
|
||||
direct_output = [record["stdout"] for record in decode_command_records(direct_sample)]
|
||||
routed_output = [record["stdout"] for record in decode_command_records(routed_sample)]
|
||||
if direct_output != routed_output:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}/pair-{pair_index}: routed output differs from direct oracle")
|
||||
for sample in matching:
|
||||
if sample.get("elapsed_ns", 0) > 4 * specification["absolute_median_ms"] * 1_000_000:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: sample exceeded timeout")
|
||||
background = sample.get("background_load", {})
|
||||
if background.get("external_cpu_seconds", float("inf")) > 0.05 * background.get("reference_elapsed_seconds", 0):
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: background CPU threshold exceeded")
|
||||
if any(background.get("memory_pressure_total_delta", {"missing": 1}).values()):
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: memory pressure caused reclaim")
|
||||
measured = [sample for sample in matching if sample.get("sample_kind") == "measured"]
|
||||
direct = sorted((sample for sample in measured if sample["placement"] == "direct"), key=lambda sample: sample["pair_index"])
|
||||
routed = sorted((sample for sample in measured if sample["placement"] == "routed"), key=lambda sample: sample["pair_index"])
|
||||
expected = sampling["measured_samples_per_cell"]
|
||||
if len(direct) != expected or len(routed) != expected:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: expected {expected} measured samples per placement")
|
||||
continue
|
||||
if [sample["pair_index"] for sample in direct] != [sample["pair_index"] for sample in routed]:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: paired sample identities differ")
|
||||
continue
|
||||
direct_times = [sample["elapsed_ns"] / 1_000_000 for sample in direct]
|
||||
routed_times = [sample["elapsed_ns"] / 1_000_000 for sample in routed]
|
||||
ratios = [right / left for left, right in zip(direct_times, routed_times, strict=True)]
|
||||
medians = {"direct": lower_median(direct_times), "routed": lower_median(routed_times), "ratio": lower_median(ratios)}
|
||||
for key in run_medians:
|
||||
run_medians[key].append(medians[key])
|
||||
calibration = specification["absolute_median_ms"] / specification["relative_median_ratio"]
|
||||
if medians["direct"] > calibration:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: direct calibration {medians['direct']:.3f} ms > {calibration:.3f} ms")
|
||||
if medians["routed"] > specification["absolute_median_ms"]:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: routed median exceeds absolute budget")
|
||||
if medians["ratio"] > specification["relative_median_ratio"]:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: median ratio exceeds relative budget")
|
||||
for label, values in (("direct", direct_times), ("routed", routed_times), ("ratio", ratios)):
|
||||
if relative_mad(values) > sampling["within_run_relative_mad_limit"]:
|
||||
failures.append(f"{specification['id']}/{state}/run-{run}: {label} relative MAD exceeds limit")
|
||||
cells.append({"workload": specification["id"], "preparation_state": state, "independent_run": run, "medians": medians})
|
||||
if all(len(values) == sampling["independent_runs"] for values in run_medians.values()):
|
||||
for label, values in run_medians.items():
|
||||
spread = (max(values) - min(values)) / lower_median(values)
|
||||
if spread > sampling["between_run_median_spread_limit"]:
|
||||
failures.append(f"{specification['id']}/{state}: {label} between-run spread exceeds limit")
|
||||
expected_samples = sum(
|
||||
len(specification["preparation_states"])
|
||||
* sampling["independent_runs"]
|
||||
* (sampling["warmup_samples_per_cell"] + sampling["measured_samples_per_cell"])
|
||||
* 2
|
||||
for specification in contract["workloads"]
|
||||
)
|
||||
if len(samples) != expected_samples:
|
||||
failures.append(f"expected {expected_samples} retained samples, observed {len(samples)}")
|
||||
result = {
|
||||
"schema": "lazaret-ignored-workspace-benchmark-result-v1",
|
||||
"disposition": "PASS" if not failures else "FAIL",
|
||||
"qualification": qualification,
|
||||
"cells": cells,
|
||||
"failures": failures,
|
||||
}
|
||||
Path(arguments.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(arguments.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
|
||||
require(not failures, "benchmark qualification failed")
|
||||
|
||||
|
||||
def add_tool_arguments(parser):
|
||||
parser.add_argument("--git", default="git")
|
||||
parser.add_argument("--uv", default="uv")
|
||||
parser.add_argument("--cargo", default="cargo")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
prepare = subparsers.add_parser("prepare")
|
||||
prepare.add_argument("--contract", type=Path, default=CONTRACT_PATH)
|
||||
prepare.add_argument("--workload", required=True)
|
||||
prepare.add_argument("--visible-root", type=Path, required=True)
|
||||
prepare.add_argument("--host-root", type=Path)
|
||||
prepare.add_argument("--isolated-root", type=Path)
|
||||
prepare.add_argument("--output", type=Path, required=True)
|
||||
add_tool_arguments(prepare)
|
||||
schedule = subparsers.add_parser("plan")
|
||||
schedule.add_argument("--contract", type=Path, default=CONTRACT_PATH)
|
||||
schedule.add_argument("--output", type=Path, required=True)
|
||||
sample = subparsers.add_parser("sample")
|
||||
sample.add_argument("--contract", type=Path, default=CONTRACT_PATH)
|
||||
sample.add_argument("--workload", required=True)
|
||||
sample.add_argument("--root", type=Path, required=True)
|
||||
sample.add_argument("--placement", choices=("direct", "routed"), required=True)
|
||||
sample.add_argument("--preparation-state", choices=("unprimed", "primed"), required=True)
|
||||
sample.add_argument("--independent-run", type=int, required=True)
|
||||
sample.add_argument("--sample-kind", choices=("warmup", "measured"), required=True)
|
||||
sample.add_argument("--sample-index", type=int, required=True)
|
||||
sample.add_argument("--pair-index", type=int, required=True)
|
||||
sample.add_argument("--fixture-id", required=True)
|
||||
sample.add_argument("--host-root", type=Path)
|
||||
sample.add_argument("--metadata-benchmark", default="lazaret-workspace-benchmark")
|
||||
sample.add_argument("--output", type=Path, required=True)
|
||||
add_tool_arguments(sample)
|
||||
verify = subparsers.add_parser("verify")
|
||||
verify.add_argument("--contract", type=Path, default=CONTRACT_PATH)
|
||||
verify.add_argument("--samples", type=Path, required=True)
|
||||
verify.add_argument("--qualification", type=Path, required=True)
|
||||
verify.add_argument("--output", type=Path, required=True)
|
||||
qualification = subparsers.add_parser("qualify")
|
||||
qualification.add_argument("--root", type=Path, required=True)
|
||||
qualification.add_argument("--output", type=Path, required=True)
|
||||
qualification.add_argument("--fusermount", default="fusermount3")
|
||||
qualification.add_argument("--runsc", default="runsc")
|
||||
qualification.add_argument("--git", default="git")
|
||||
qualification.add_argument("--uv", default="uv")
|
||||
qualification.add_argument("--rustc", default="rustc")
|
||||
qualification.add_argument("--xfs-info", default="xfs_info")
|
||||
arguments = parser.parse_args()
|
||||
if arguments.command == "prepare":
|
||||
materialize(arguments)
|
||||
elif arguments.command == "plan":
|
||||
plan(arguments)
|
||||
elif arguments.command == "sample":
|
||||
run_sample(arguments)
|
||||
elif arguments.command == "qualify":
|
||||
qualify(arguments)
|
||||
else:
|
||||
verify_results(arguments)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
743
tools/ignored_workspace_verifier.py
Normal file
743
tools/ignored_workspace_verifier.py
Normal file
|
|
@ -0,0 +1,743 @@
|
|||
import argparse
|
||||
import contextlib
|
||||
import ctypes
|
||||
import errno
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import mmap
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURES = ROOT / "tests/fixtures/ignored-workspace-v1"
|
||||
NAMESPACE_MARKER = "LAZARET_IGNORED_WORKSPACE_PRIVATE_NAMESPACE"
|
||||
|
||||
|
||||
def command(argv, cwd=None, env=None, check=True):
|
||||
return subprocess.run(
|
||||
[os.fspath(value) for value in argv],
|
||||
cwd=cwd,
|
||||
env=env,
|
||||
check=check,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
def expect_errno(expected, action):
|
||||
try:
|
||||
action()
|
||||
except OSError as error:
|
||||
require(error.errno == expected, f"expected errno {expected}, observed {error.errno}")
|
||||
else:
|
||||
raise AssertionError(f"expected errno {expected}, operation succeeded")
|
||||
|
||||
|
||||
class MountedFixture:
|
||||
def __init__(self, work_root, router, global_excludes=None):
|
||||
self.root = Path(tempfile.mkdtemp(prefix="ignored-workspace-", dir=work_root))
|
||||
self.host = self.root / "host"
|
||||
self.isolated = self.root / "isolated"
|
||||
self.mount = self.root / "mount"
|
||||
for path in (self.host, self.isolated, self.mount):
|
||||
path.mkdir()
|
||||
command(["git", "init", "--quiet"], self.host)
|
||||
command(["git", "config", "user.email", "[email protected]"], self.host)
|
||||
command(["git", "config", "user.name", "Fixture"], self.host)
|
||||
self.router = Path(router)
|
||||
self.global_excludes = global_excludes
|
||||
self.process = None
|
||||
self.closed = False
|
||||
|
||||
def start(self):
|
||||
argv = [self.router, self.host, self.isolated, self.mount]
|
||||
if self.global_excludes is not None:
|
||||
argv.append(self.global_excludes)
|
||||
self.process = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
if self.is_mounted():
|
||||
return self
|
||||
if self.process.poll() is not None:
|
||||
stderr = self.process.stderr.read().decode(errors="replace")
|
||||
raise RuntimeError(f"router exited before mount: {stderr}")
|
||||
time.sleep(0.01)
|
||||
raise TimeoutError("router did not mount within ten seconds")
|
||||
|
||||
def stop_router(self):
|
||||
if self.process is not None and self.process.poll() is None:
|
||||
try:
|
||||
self.process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.process.terminate()
|
||||
self.process.wait(timeout=10)
|
||||
|
||||
def is_mounted(self):
|
||||
target = os.fsencode(self.mount)
|
||||
for line in Path("/proc/self/mountinfo").read_bytes().splitlines():
|
||||
fields = line.split(b" - ", 1)[0].split()
|
||||
if len(fields) >= 5 and mountinfo_path(fields[4]) == target:
|
||||
return True
|
||||
return False
|
||||
|
||||
def force_unmount(self):
|
||||
if not self.is_mounted():
|
||||
return
|
||||
result = command(["fusermount3", "-u", "-z", self.mount], check=False)
|
||||
require(result.returncode == 0 or not self.is_mounted(), result.stderr.decode(errors="replace"))
|
||||
|
||||
def shutdown(self):
|
||||
self.force_unmount()
|
||||
self.stop_router()
|
||||
require(not self.is_mounted(), "owned FUSE mount remains")
|
||||
|
||||
def close(self):
|
||||
if self.closed:
|
||||
return
|
||||
try:
|
||||
self.shutdown()
|
||||
shutil.rmtree(self.root)
|
||||
finally:
|
||||
self.closed = not self.root.exists() and not self.is_mounted()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
self.shutdown()
|
||||
|
||||
|
||||
def mountinfo_path(value):
|
||||
return value.replace(b"\\040", b" ").replace(b"\\011", b"\t").replace(b"\\012", b"\n").replace(b"\\134", b"\\")
|
||||
|
||||
|
||||
def write(path, data):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
|
||||
def commit(fixture, relative="tracked", data=b"tracked"):
|
||||
write(fixture.host / relative, data)
|
||||
command(["git", "add", "--", relative], fixture.host)
|
||||
command(["git", "commit", "--quiet", "-m", relative], fixture.host)
|
||||
|
||||
|
||||
def ignored_pair(fixture, relative="cache/item", host=b"host", isolated=b"isolated"):
|
||||
write(fixture.host / ".gitignore", (relative.split("/", 1)[0] + "/\n").encode())
|
||||
write(fixture.host / relative, host)
|
||||
write(fixture.isolated / relative, isolated)
|
||||
|
||||
|
||||
def assert_pair_unchanged(fixture, relative, host, isolated):
|
||||
require((fixture.host / relative).read_bytes() == host, "host backing changed")
|
||||
require((fixture.isolated / relative).read_bytes() == isolated, "isolated backing changed")
|
||||
|
||||
|
||||
def classification_case(case_id, fixture):
|
||||
if case_id == "non-utf8-ignore-match":
|
||||
write(fixture.host / ".gitignore", b"*.obj\n")
|
||||
host_path = os.fsencode(fixture.host) + b"/\xff\xfe.obj"
|
||||
with open(host_path, "wb") as output:
|
||||
output.write(b"host-sentinel")
|
||||
path = os.fsencode(fixture.mount) + b"/\xff\xfe.obj"
|
||||
with fixture.start():
|
||||
require(not os.path.exists(path), "non-UTF8 host sentinel was exposed")
|
||||
descriptor = os.open(path, os.O_CREAT | os.O_WRONLY, 0o600)
|
||||
os.write(descriptor, b"isolated")
|
||||
os.close(descriptor)
|
||||
require(os.path.exists(os.fsencode(fixture.isolated) + b"/\xff\xfe.obj"), "non-UTF8 path was not isolated")
|
||||
require(Path(os.fsdecode(host_path)).read_bytes() == b"host-sentinel", "non-UTF8 host sentinel changed")
|
||||
return
|
||||
if case_id == "root-negation-precedence":
|
||||
write(fixture.host / ".gitignore", b"*.log\n!important.log\n")
|
||||
with fixture.start():
|
||||
write(fixture.mount / "ordinary.log", b"isolated")
|
||||
write(fixture.mount / "important.log", b"host")
|
||||
require((fixture.isolated / "ordinary.log").read_bytes() == b"isolated", "root ignore did not route")
|
||||
require((fixture.host / "important.log").read_bytes() == b"host", "negation did not restore host route")
|
||||
return
|
||||
if case_id == "nested-ignore-precedence":
|
||||
write(fixture.host / ".gitignore", b"*.tmp\n!nested/drop.tmp\n")
|
||||
write(fixture.host / "nested/.gitignore", b"drop.tmp\n!keep.tmp\n")
|
||||
with fixture.start():
|
||||
write(fixture.mount / "nested/drop.tmp", b"isolated")
|
||||
write(fixture.mount / "nested/keep.tmp", b"host")
|
||||
require((fixture.isolated / "nested/drop.tmp").is_file(), "nested ignored path not isolated")
|
||||
require((fixture.host / "nested/keep.tmp").is_file(), "nested negation not host-backed")
|
||||
return
|
||||
if case_id == "info-exclude":
|
||||
write(fixture.host / ".git/info/exclude", b"local.secret\n")
|
||||
write(fixture.host / "local.secret", b"host-sentinel")
|
||||
with fixture.start():
|
||||
require(not (fixture.mount / "local.secret").exists(), "info/exclude host sentinel was exposed")
|
||||
return
|
||||
if case_id == "global-excludes":
|
||||
global_file = fixture.root / "global-ignore"
|
||||
write(global_file, b"*.swap\n")
|
||||
fixture.global_excludes = global_file
|
||||
write(fixture.host / "editor.swap", b"host-sentinel")
|
||||
with fixture.start():
|
||||
require(not (fixture.mount / "editor.swap").exists(), "global exclude host sentinel was exposed")
|
||||
return
|
||||
if case_id == "ignore-case-folding":
|
||||
command(["git", "config", "core.ignoreCase", "true"], fixture.host)
|
||||
write(fixture.host / ".gitignore", b"build.log\n")
|
||||
write(fixture.host / "BUILD.LOG", b"host-sentinel")
|
||||
with fixture.start():
|
||||
require(not (fixture.mount / "BUILD.LOG").exists(), "case-folded host sentinel was exposed")
|
||||
return
|
||||
if case_id == "nested-repository-boundary":
|
||||
write(fixture.host / ".gitignore", b"*.cache\n")
|
||||
nested = fixture.host / "vendor/nested"
|
||||
nested.mkdir(parents=True)
|
||||
command(["git", "init", "--quiet"], nested)
|
||||
write(nested / ".gitignore", b"target/\n")
|
||||
write(nested / "target/sentinel", b"host-sentinel")
|
||||
with fixture.start():
|
||||
require(not (fixture.mount / "vendor/nested/target/sentinel").exists(), "nested host sentinel was exposed")
|
||||
write(fixture.mount / "vendor/nested/target/item", b"isolated")
|
||||
write(fixture.mount / "vendor/nested/ordinary.cache", b"host")
|
||||
require((fixture.isolated / "vendor/nested/target/item").is_file(), "nested policy not selected")
|
||||
require((fixture.host / "vendor/nested/ordinary.cache").is_file(), "parent policy crossed repository boundary")
|
||||
return
|
||||
if case_id == "live-ignore-publication":
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
write(fixture.host / "cache/item", b"host")
|
||||
with fixture.start():
|
||||
require((fixture.mount / "cache/item").read_bytes() == b"host", "initial host route missing")
|
||||
write(fixture.host / ".gitignore", b"cache/\n")
|
||||
require(not (fixture.mount / "cache/item").exists(), "published ignore did not hide host object")
|
||||
require((fixture.host / "cache/item").read_bytes() == b"host", "publication copied or removed host object")
|
||||
return
|
||||
raise AssertionError(f"classification case is not implemented: {case_id}")
|
||||
|
||||
|
||||
def atomic_case(case_id, fixture):
|
||||
if case_id == "cross-route-directory-exchange":
|
||||
write(fixture.host / ".gitignore", b"ignored/\n")
|
||||
(fixture.host / "source").mkdir()
|
||||
write(fixture.host / "source/marker", b"host")
|
||||
(fixture.isolated / "ignored/destination").mkdir(parents=True)
|
||||
write(fixture.isolated / "ignored/destination/marker", b"isolated")
|
||||
else:
|
||||
write(fixture.host / ".gitignore", b"ignored/\n")
|
||||
write(fixture.host / "source", b"source")
|
||||
if case_id in {"cross-route-file-rename", "cross-route-hard-link"}:
|
||||
(fixture.isolated / "ignored").mkdir()
|
||||
with fixture.start():
|
||||
if case_id == "cross-route-hard-link":
|
||||
expect_errno(errno.EXDEV, lambda: os.link(fixture.mount / "source", fixture.mount / "ignored/destination"))
|
||||
elif case_id == "cross-route-directory-exchange":
|
||||
rename_exchange(fixture.mount / "source", fixture.mount / "ignored/destination")
|
||||
else:
|
||||
expect_errno(errno.EXDEV, lambda: os.rename(fixture.mount / "source", fixture.mount / "ignored/destination"))
|
||||
if case_id == "cross-route-directory-exchange":
|
||||
require((fixture.host / "source/marker").read_bytes() == b"host", "source directory mutated after EXDEV")
|
||||
require((fixture.isolated / "ignored/destination/marker").read_bytes() == b"isolated", "destination directory mutated after EXDEV")
|
||||
else:
|
||||
require((fixture.host / "source").read_bytes() == b"source", "source mutated after EXDEV")
|
||||
require(not (fixture.host / "ignored/destination").exists() and not (fixture.isolated / "ignored/destination").exists(), "destination appeared after EXDEV")
|
||||
|
||||
|
||||
def rename_exchange(source, destination):
|
||||
observed = exchange_paths(source, destination)
|
||||
if observed == 0:
|
||||
raise AssertionError("cross-route RENAME_EXCHANGE succeeded")
|
||||
require(observed == errno.EXDEV, f"expected EXDEV from RENAME_EXCHANGE, observed {observed}")
|
||||
|
||||
|
||||
def exchange_paths(source, destination):
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
result = libc.renameat2(-100, os.fsencode(source), -100, os.fsencode(destination), 2)
|
||||
if result == 0:
|
||||
return 0
|
||||
return ctypes.get_errno()
|
||||
|
||||
|
||||
def copy_file_range(source, destination, length):
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
syscall = libc.copy_file_range
|
||||
syscall.argtypes = [
|
||||
ctypes.c_int,
|
||||
ctypes.POINTER(ctypes.c_longlong),
|
||||
ctypes.c_int,
|
||||
ctypes.POINTER(ctypes.c_longlong),
|
||||
ctypes.c_size_t,
|
||||
ctypes.c_uint,
|
||||
]
|
||||
syscall.restype = ctypes.c_ssize_t
|
||||
copied = 0
|
||||
while copied < length:
|
||||
result = syscall(source, None, destination, None, length - copied, 0)
|
||||
if result < 0:
|
||||
observed = ctypes.get_errno()
|
||||
raise OSError(observed, os.strerror(observed))
|
||||
if result == 0:
|
||||
break
|
||||
copied += result
|
||||
return copied
|
||||
|
||||
|
||||
def policy_transition_case(case_id, fixture):
|
||||
ignored_pair(fixture)
|
||||
if case_id == "non-ignored-becomes-ignored":
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
with fixture.start():
|
||||
if case_id == "ignored-becomes-non-ignored":
|
||||
require((fixture.mount / "cache/item").read_bytes() == b"isolated", "initial isolated route missing")
|
||||
old_descriptor = os.open(fixture.mount / "cache/item", os.O_RDWR)
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
require((fixture.mount / "cache/item").read_bytes() == b"host", "host route not selected")
|
||||
os.pwrite(old_descriptor, b"old-isolated", 0)
|
||||
os.close(old_descriptor)
|
||||
require((fixture.isolated / "cache/item").read_bytes().startswith(b"old-isolated"), "old handle rebound from isolated backing")
|
||||
else:
|
||||
require((fixture.mount / "cache/item").read_bytes() == b"host", "initial host route missing")
|
||||
old_descriptor = os.open(fixture.mount / "cache/item", os.O_RDWR)
|
||||
write(fixture.host / ".gitignore", b"cache/\n")
|
||||
require((fixture.mount / "cache/item").read_bytes() == b"isolated", "isolated route not selected")
|
||||
os.pwrite(old_descriptor, b"old-host", 0)
|
||||
os.close(old_descriptor)
|
||||
require((fixture.host / "cache/item").read_bytes().startswith(b"old-host"), "old handle rebound from host backing")
|
||||
if case_id == "ignored-becomes-non-ignored":
|
||||
require((fixture.host / "cache/item").read_bytes() == b"host", "transition copied to host")
|
||||
else:
|
||||
require((fixture.isolated / "cache/item").read_bytes() == b"isolated", "transition copied to isolated backing")
|
||||
|
||||
|
||||
def index_case(case_id, fixture):
|
||||
ignored_pair(fixture, "deps/lib")
|
||||
with fixture.start():
|
||||
if case_id == "guest-force-add-isolated-only":
|
||||
(fixture.host / "deps/lib").unlink()
|
||||
initial_index = (fixture.host / ".git/index").read_bytes() if (fixture.host / ".git/index").exists() else None
|
||||
result = command(["git", "add", "-f", "--", "deps/lib"], fixture.mount, check=False)
|
||||
require(result.returncode != 0, "isolated-only force-add succeeded")
|
||||
final_index = (fixture.host / ".git/index").read_bytes() if (fixture.host / ".git/index").exists() else None
|
||||
require(final_index == initial_index, "refused force-add published an index")
|
||||
require((fixture.isolated / "deps/lib").read_bytes() == b"isolated", "isolated object was lost")
|
||||
require(not (fixture.host / "deps/lib").exists(), "refused force-add created a host object")
|
||||
elif case_id == "host-established-tracked-cutover":
|
||||
(fixture.host / "deps/lib").unlink()
|
||||
old_descriptor = os.open(fixture.mount / "deps/lib", os.O_RDWR)
|
||||
write(fixture.host / "deps/lib", b"independent-host")
|
||||
command(["git", "add", "-f", "--", "deps/lib"], fixture.host)
|
||||
new_descriptor = os.open(fixture.mount / "deps/lib", os.O_RDONLY)
|
||||
require(os.read(new_descriptor, 64) == b"independent-host", "new handle did not select host object")
|
||||
os.pwrite(old_descriptor, b"old-handle", 0)
|
||||
os.close(old_descriptor)
|
||||
os.close(new_descriptor)
|
||||
require((fixture.host / "deps/lib").read_bytes() == b"independent-host", "old handle rebound to host")
|
||||
require((fixture.isolated / "deps/lib").read_bytes().startswith(b"old-handle"), "old handle did not remain isolated")
|
||||
else:
|
||||
(fixture.host / "deps/lib").unlink()
|
||||
command(["git", "update-index", "--add", "--cacheinfo", "100644", "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391", "deps/lib"], fixture.host)
|
||||
expect_errno(errno.EIO, lambda: (fixture.mount / "deps/lib").read_bytes())
|
||||
require((fixture.isolated / "deps/lib").read_bytes() == b"isolated", "inconsistent isolated object was lost")
|
||||
|
||||
|
||||
def open_handle_case(case_id, fixture):
|
||||
ignored_pair(fixture, "cache/pinned", host=b"host-data", isolated=b"isolated-data")
|
||||
with fixture.start():
|
||||
if case_id == "mmap-policy-transition":
|
||||
descriptor = os.open(fixture.mount / "cache/pinned", os.O_RDWR)
|
||||
mapping = mmap.mmap(descriptor, 0, access=mmap.ACCESS_WRITE)
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
mapping[0:8] = b"mapped!!"
|
||||
mapping.flush()
|
||||
mapping.close()
|
||||
os.close(descriptor)
|
||||
require((fixture.isolated / "cache/pinned").read_bytes().startswith(b"mapped!!"), "mapping rebound")
|
||||
require((fixture.host / "cache/pinned").read_bytes() == b"host-data", "mapping copied back")
|
||||
elif case_id == "directory-stream-policy-snapshot":
|
||||
write(fixture.isolated / "cache/isolated-child", b"isolated")
|
||||
write(fixture.host / "cache/host-child", b"host")
|
||||
iterator = os.scandir(fixture.mount / "cache")
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
names = {entry.name for entry in iterator}
|
||||
iterator.close()
|
||||
require("isolated-child" in names and "host-child" not in names, "old stream mixed snapshots")
|
||||
new_names = {entry.name for entry in os.scandir(fixture.mount / "cache")}
|
||||
require("host-child" in new_names and "isolated-child" not in new_names, "new stream did not use new snapshot")
|
||||
else:
|
||||
descriptor = os.open(fixture.mount / "cache/pinned", os.O_RDWR)
|
||||
fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB, 1)
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
new_descriptor = os.open(fixture.mount / "cache/pinned", os.O_RDWR)
|
||||
fcntl.lockf(new_descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB, 1)
|
||||
require(lock_attempt(fixture.isolated / "cache/pinned") == errno.EAGAIN, "old isolated lock was transferred or lost")
|
||||
require(lock_attempt(fixture.host / "cache/pinned") == errno.EAGAIN, "new host lock was not established")
|
||||
os.close(descriptor)
|
||||
require(lock_attempt(fixture.isolated / "cache/pinned") == 0, "isolated lock remained after old handle closed")
|
||||
require(lock_attempt(fixture.host / "cache/pinned") == errno.EAGAIN, "isolated unlock affected host lock")
|
||||
os.close(new_descriptor)
|
||||
require(lock_attempt(fixture.host / "cache/pinned") == 0, "host lock remained after new handle closed")
|
||||
|
||||
|
||||
def lock_attempt(path):
|
||||
source = """
|
||||
import fcntl, os, sys
|
||||
descriptor = os.open(sys.argv[1], os.O_RDWR)
|
||||
try:
|
||||
fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB, 1)
|
||||
except OSError as error:
|
||||
sys.exit(error.errno)
|
||||
fcntl.lockf(descriptor, fcntl.LOCK_UN, 1)
|
||||
"""
|
||||
return subprocess.run([sys.executable, "-c", source, path], check=False).returncode
|
||||
|
||||
|
||||
def operation_case(case_id, fixture):
|
||||
ignored_pair(fixture, "cache/base")
|
||||
if case_id == "timestamps":
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
with fixture.start():
|
||||
if case_id == "extended-attributes-and-acls":
|
||||
path = fixture.mount / "cache/base"
|
||||
os.setxattr(path, b"user.lazaret", b"value")
|
||||
require(os.getxattr(path, b"user.lazaret") == b"value", "xattr round trip failed")
|
||||
require(os.getxattr(fixture.isolated / "cache/base", b"user.lazaret") == b"value", "xattr reached wrong backing")
|
||||
os.removexattr(path, b"user.lazaret")
|
||||
expect_errno(errno.ENODATA, lambda: os.getxattr(fixture.isolated / "cache/base", b"user.lazaret"))
|
||||
if shutil.which("setfacl"):
|
||||
command(["setfacl", "-m", f"u:{os.getuid()}:rw", path])
|
||||
mounted_acl = command(["getfacl", "--absolute-names", path]).stdout
|
||||
backing_acl = command(["getfacl", "--absolute-names", fixture.isolated / "cache/base"]).stdout
|
||||
require(b"user:" in mounted_acl and b"user:" in backing_acl, "POSIX ACL was not retained on selected backing")
|
||||
elif case_id == "sparse-copy-range":
|
||||
source = fixture.mount / "cache/source"
|
||||
destination = fixture.mount / "cache/destination"
|
||||
with source.open("wb") as output:
|
||||
output.write(b"x" * 8192)
|
||||
output.seek(1024 * 1024)
|
||||
output.write(b"tail")
|
||||
descriptor = os.open(source, os.O_RDONLY)
|
||||
require(os.lseek(descriptor, 0, os.SEEK_DATA) == 0, "SEEK_DATA did not find first extent")
|
||||
require(os.lseek(descriptor, 0, os.SEEK_HOLE) >= 8192, "SEEK_HOLE did not find sparse gap")
|
||||
os.close(descriptor)
|
||||
command(["fallocate", "--punch-hole", "--keep-size", "--offset", "0", "--length", "4096", source])
|
||||
require(source.read_bytes()[:4096] == bytes(4096), "hole punch retained data")
|
||||
with source.open("rb") as input_file, destination.open("wb") as output:
|
||||
require(
|
||||
copy_file_range(input_file.fileno(), output.fileno(), 1024 * 1024 + 4)
|
||||
== 1024 * 1024 + 4,
|
||||
"same-route copy range ended early",
|
||||
)
|
||||
require(input_file.tell() == 1024 * 1024 + 4, "same-route source offset differs")
|
||||
require(output.tell() == 1024 * 1024 + 4, "same-route destination offset differs")
|
||||
require(destination.read_bytes() == source.read_bytes(), "same-route copy differs")
|
||||
require(destination.stat().st_size == source.stat().st_size, "same-route copy size differs")
|
||||
write(fixture.host / "host-copy", b"unchanged")
|
||||
write(fixture.host / "unrelated-host", b"host-sentinel")
|
||||
write(fixture.isolated / "cache/unrelated-isolated", b"isolated-sentinel")
|
||||
source_before = (fixture.isolated / "cache/source").read_bytes()
|
||||
same_route_destination_before = (fixture.isolated / "cache/destination").read_bytes()
|
||||
with (fixture.mount / "host-copy").open("r+b") as output, source.open("rb") as input_file:
|
||||
write(fixture.host / ".gitignore", b"different/\n")
|
||||
os.lseek(input_file.fileno(), 4096, os.SEEK_SET)
|
||||
require(copy_file_range(input_file.fileno(), output.fileno(), 1) == 1, "cross-route copy count differs")
|
||||
require((fixture.host / "host-copy").read_bytes() == b"xnchanged", "cross-route destination range differs")
|
||||
require((fixture.isolated / "cache/source").read_bytes() == source_before, "cross-route copy mutated source")
|
||||
require(
|
||||
(fixture.isolated / "cache/destination").read_bytes() == same_route_destination_before,
|
||||
"cross-route copy mutated same-route destination",
|
||||
)
|
||||
require((fixture.host / "cache/source").exists() is False, "cross-route source handle was reclassified")
|
||||
require((fixture.host / "unrelated-host").read_bytes() == b"host-sentinel", "cross-route copy mutated unrelated host object")
|
||||
require(
|
||||
(fixture.isolated / "cache/unrelated-isolated").read_bytes() == b"isolated-sentinel",
|
||||
"cross-route copy mutated unrelated isolated object",
|
||||
)
|
||||
elif case_id == "symlink-confinement":
|
||||
os.symlink("../../outside", fixture.mount / "cache/link")
|
||||
require(os.readlink(fixture.mount / "cache/link") == "../../outside", "symlink target changed")
|
||||
write(fixture.host / "virtual-target", b"virtual-resolution")
|
||||
os.symlink("../virtual-target", fixture.mount / "cache/virtual-link")
|
||||
require(
|
||||
(fixture.mount / "cache/virtual-link").read_bytes() == b"virtual-resolution",
|
||||
"virtual symlink resolution changed",
|
||||
)
|
||||
command(
|
||||
[
|
||||
"cargo",
|
||||
"nextest",
|
||||
"run",
|
||||
"--offline",
|
||||
"--locked",
|
||||
"-p",
|
||||
"lazaret-ignored-workspace",
|
||||
"--test-threads",
|
||||
"2",
|
||||
"-E",
|
||||
"test(descriptor_traversal_race_never_follows_backing_symlinks)",
|
||||
],
|
||||
ROOT,
|
||||
)
|
||||
elif case_id == "timestamps":
|
||||
path = fixture.mount / "cache/base"
|
||||
stamp = 1_700_000_000_123_456_789
|
||||
os.utime(path, ns=(stamp, stamp), follow_symlinks=False)
|
||||
metadata = os.stat(path, follow_symlinks=False)
|
||||
require(metadata.st_mtime_ns == stamp, "nanosecond timestamp changed")
|
||||
require(os.stat(fixture.host / "cache/base", follow_symlinks=False).st_mtime_ns == stamp, "timestamp was not bidirectional")
|
||||
else:
|
||||
fifo = fixture.mount / "cache/fifo"
|
||||
os.mkfifo(fifo)
|
||||
with socket.socket(socket.AF_UNIX) as endpoint:
|
||||
original_directory = os.open(".", os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.chdir(fixture.mount)
|
||||
endpoint.bind("cache/socket")
|
||||
finally:
|
||||
os.fchdir(original_directory)
|
||||
os.close(original_directory)
|
||||
require(stat.S_ISFIFO(os.lstat(fixture.isolated / "cache/fifo").st_mode), "FIFO did not reach isolated backing")
|
||||
require(stat.S_ISSOCK(os.lstat(fixture.isolated / "cache/socket").st_mode), "socket did not reach isolated backing")
|
||||
expect_errno(errno.EPERM, lambda: os.mknod(fixture.mount / "cache/device", stat.S_IFCHR | 0o600, os.makedev(1, 3)))
|
||||
|
||||
|
||||
def git_case(case_id, fixture):
|
||||
commit(fixture)
|
||||
initial_branch = command(["git", "symbolic-ref", "--short", "HEAD"], fixture.host).stdout.strip()
|
||||
ignored_pair(fixture, "cache/sentinel")
|
||||
with fixture.start():
|
||||
if case_id == "git-ordinary-workflow":
|
||||
write(fixture.mount / "ordinary", b"ordinary")
|
||||
command(["git", "status", "--porcelain=v2"], fixture.mount)
|
||||
command(["git", "add", "ordinary"], fixture.mount)
|
||||
command(["git", "commit", "--quiet", "-m", "ordinary"], fixture.mount)
|
||||
write(fixture.mount / "tracked", b"changed")
|
||||
command(["git", "restore", "tracked"], fixture.mount)
|
||||
require((fixture.host / "ordinary").read_bytes() == b"ordinary", "ordinary commit was not bidirectional")
|
||||
require((fixture.host / "tracked").read_bytes() == b"tracked", "restore did not reach host worktree")
|
||||
elif case_id == "git-history-workflow":
|
||||
command(["git", "checkout", "-b", "topic"], fixture.mount)
|
||||
write(fixture.mount / "topic-file", b"topic")
|
||||
command(["git", "add", "topic-file"], fixture.mount)
|
||||
command(["git", "commit", "--quiet", "-m", "topic"], fixture.mount)
|
||||
command(["git", "checkout", initial_branch], fixture.mount)
|
||||
write(fixture.mount / "base-file", b"base")
|
||||
command(["git", "add", "base-file"], fixture.mount)
|
||||
command(["git", "commit", "--quiet", "-m", "base"], fixture.mount)
|
||||
command(["git", "checkout", "topic"], fixture.mount)
|
||||
command(["git", "rebase", initial_branch], fixture.mount)
|
||||
command(["git", "checkout", initial_branch], fixture.mount)
|
||||
command(["git", "merge", "topic"], fixture.mount)
|
||||
write(fixture.mount / "tracked", b"stash")
|
||||
command(["git", "stash"], fixture.mount)
|
||||
command(["git", "stash", "pop"], fixture.mount)
|
||||
command(["git", "restore", "tracked"], fixture.mount)
|
||||
require(command(["git", "rev-parse", "topic"], fixture.mount).stdout == command(["git", "rev-parse", "topic"], fixture.host).stdout, "reference update was not bidirectional")
|
||||
else:
|
||||
raise AssertionError("linked-worktree and Sentry cases use dedicated harnesses")
|
||||
assert_pair_unchanged(fixture, "cache/sentinel", b"host", b"isolated")
|
||||
|
||||
|
||||
def linked_worktree_case(case_id, fixture):
|
||||
del case_id
|
||||
shutil.rmtree(fixture.host)
|
||||
base = fixture.root / "base"
|
||||
base.mkdir()
|
||||
command(["git", "init", "--quiet"], base)
|
||||
command(["git", "config", "user.email", "[email protected]"], base)
|
||||
command(["git", "config", "user.name", "Fixture"], base)
|
||||
write(base / "tracked", b"tracked")
|
||||
command(["git", "add", "tracked"], base)
|
||||
command(["git", "commit", "--quiet", "-m", "base"], base)
|
||||
command(["git", "worktree", "add", "--quiet", fixture.host], base)
|
||||
write(fixture.host / ".gitignore", b"target/\n")
|
||||
write(fixture.host / "target/item", b"host-sentinel")
|
||||
with fixture.start():
|
||||
require(not (fixture.mount / "target/item").exists(), "linked-worktree host sentinel was exposed")
|
||||
write(fixture.mount / "target/item", b"isolated")
|
||||
result = command(["git", "status", "--porcelain=v2", "--untracked-files=all"], fixture.mount)
|
||||
require(b"target/item" not in result.stdout, "linked-worktree ignore policy was not applied")
|
||||
require((fixture.isolated / "target/item").read_bytes() == b"isolated", "linked-worktree isolated state missing")
|
||||
require((fixture.host / "target/item").read_bytes() == b"host-sentinel", "linked-worktree host sentinel changed")
|
||||
|
||||
|
||||
def sentry_case(case_id, fixture, sentry_binary):
|
||||
del case_id
|
||||
commit(fixture)
|
||||
transient = fixture.root / "transient-hooks"
|
||||
transient.mkdir()
|
||||
write(fixture.host / ".git/hooks/host-sentinel", b"host")
|
||||
host_config = (fixture.host / ".git/config").read_bytes()
|
||||
with fixture.start():
|
||||
process = subprocess.Popen(
|
||||
[sentry_binary, fixture.mount, fixture.host / ".git", transient],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
require(process.stdout.readline() == b"ready\n", "Git Sentry did not become ready")
|
||||
expect_errno(errno.EROFS, lambda: write(fixture.mount / ".git/config", b"changed"))
|
||||
require(not (fixture.mount / ".git/hooks/host-sentinel").exists(), "host hook remained visible")
|
||||
write(fixture.mount / ".git/hooks/sandbox-hook", b"transient")
|
||||
command(["git", "status", "--porcelain=v2"], fixture.mount)
|
||||
finally:
|
||||
process.stdin.close()
|
||||
return_code = process.wait(timeout=10)
|
||||
require(return_code == 0, process.stderr.read().decode(errors="replace"))
|
||||
require((fixture.mount / ".git/config").read_bytes() == host_config, "host config was not restored")
|
||||
require((fixture.mount / ".git/hooks/host-sentinel").read_bytes() == b"host", "host hooks were not restored")
|
||||
require((fixture.host / ".git/hooks/host-sentinel").read_bytes() == b"host", "host hook changed")
|
||||
require((transient / "sandbox-hook").read_bytes() == b"transient", "transient hook missing")
|
||||
|
||||
|
||||
def invalidation_or_race_case(case_id, fixture):
|
||||
ignored_pair(fixture)
|
||||
with fixture.start():
|
||||
if case_id == "concurrent-host-sandbox-mutation":
|
||||
barrier = threading.Barrier(2)
|
||||
errors = []
|
||||
def host_replace():
|
||||
try:
|
||||
barrier.wait()
|
||||
write(fixture.host / "ordinary.tmp", b"replacement")
|
||||
os.replace(fixture.host / "ordinary.tmp", fixture.host / "ordinary")
|
||||
except BaseException as error:
|
||||
errors.append(error)
|
||||
write(fixture.host / "ordinary", b"initial")
|
||||
require((fixture.mount / "ordinary").read_bytes() == b"initial", "initial host bytes differ")
|
||||
write(fixture.host / "ordinary", b"changed")
|
||||
require((fixture.mount / "ordinary").read_bytes() == b"changed", "same-size host overwrite was stale")
|
||||
thread = threading.Thread(target=host_replace)
|
||||
thread.start()
|
||||
barrier.wait()
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.replace(fixture.mount / "ordinary", fixture.mount / "ordinary-renamed")
|
||||
thread.join()
|
||||
require(not errors, f"host racer failed: {errors}")
|
||||
names = [entry.name for entry in os.scandir(fixture.mount)]
|
||||
require(len(names) == len(set(names)), "concurrent enumeration duplicated a name")
|
||||
candidates = [name for name in ("ordinary", "ordinary-renamed") if (fixture.mount / name).exists()]
|
||||
require(candidates, "native rename race lost every host-backed outcome")
|
||||
write(fixture.mount / candidates[0], b"sandbox-write")
|
||||
(fixture.mount / candidates[0]).unlink()
|
||||
require((fixture.isolated / "cache/item").read_bytes() == b"isolated", "race crossed backing")
|
||||
else:
|
||||
fixture.process.kill()
|
||||
fixture.process.wait(timeout=10)
|
||||
try:
|
||||
(fixture.mount / "cache/item").read_bytes()
|
||||
except OSError as error:
|
||||
require(error.errno in {errno.ENOTCONN, errno.EIO}, f"unexpected dead-router errno: {error.errno}")
|
||||
else:
|
||||
raise AssertionError("dead router fell through to host backing")
|
||||
fixture.force_unmount()
|
||||
assert_pair_unchanged(fixture, "cache/item", b"host", b"isolated")
|
||||
fixture.start()
|
||||
require((fixture.mount / "cache/item").read_bytes() == b"isolated", "recovered router selected wrong backing")
|
||||
|
||||
|
||||
CASE_GROUPS = {
|
||||
**{name: classification_case for name in [
|
||||
"non-utf8-ignore-match", "root-negation-precedence", "nested-ignore-precedence",
|
||||
"info-exclude", "global-excludes", "ignore-case-folding", "nested-repository-boundary",
|
||||
"live-ignore-publication",
|
||||
]},
|
||||
**{name: atomic_case for name in ["cross-route-file-rename", "cross-route-directory-exchange", "cross-route-hard-link"]},
|
||||
**{name: policy_transition_case for name in ["ignored-becomes-non-ignored", "non-ignored-becomes-ignored"]},
|
||||
**{name: index_case for name in ["guest-force-add-isolated-only", "host-established-tracked-cutover", "unmediated-inconsistent-index"]},
|
||||
**{name: open_handle_case for name in ["mmap-policy-transition", "locks-policy-transition", "directory-stream-policy-snapshot"]},
|
||||
**{name: operation_case for name in ["extended-attributes-and-acls", "sparse-copy-range", "symlink-confinement", "timestamps", "special-files"]},
|
||||
**{name: git_case for name in ["git-ordinary-workflow", "git-history-workflow"]},
|
||||
**{name: invalidation_or_race_case for name in ["concurrent-host-sandbox-mutation", "router-loss-fails-closed"]},
|
||||
}
|
||||
|
||||
|
||||
def run_corpus(arguments):
|
||||
inventory = json.loads((FIXTURES / "corpus.json").read_text())["cases"]
|
||||
selected = [case for case in inventory if arguments.case in (None, case["id"])]
|
||||
require(selected, "requested corpus case does not exist")
|
||||
missing = [case["id"] for case in inventory if case["id"] not in CASE_GROUPS and case["id"] not in {"linked-worktree-index", "git-sentry"}]
|
||||
require(not missing, f"corpus adapter is incomplete: {missing}")
|
||||
results = []
|
||||
Path(arguments.work_root).mkdir(parents=True, exist_ok=True)
|
||||
for case in selected:
|
||||
identifier = case["id"]
|
||||
started = time.monotonic_ns()
|
||||
fixture = MountedFixture(arguments.work_root, arguments.router)
|
||||
try:
|
||||
if identifier == "linked-worktree-index":
|
||||
linked_worktree_case(identifier, fixture)
|
||||
elif identifier == "git-sentry":
|
||||
sentry_case(identifier, fixture, arguments.git_sentry)
|
||||
else:
|
||||
CASE_GROUPS[identifier](identifier, fixture)
|
||||
disposition = "PASS"
|
||||
detail = None
|
||||
except BaseException as error:
|
||||
disposition = "FAIL"
|
||||
detail = repr(error)
|
||||
finally:
|
||||
try:
|
||||
fixture.close()
|
||||
except BaseException as cleanup_error:
|
||||
if disposition == "PASS":
|
||||
disposition = "FAIL"
|
||||
detail = f"cleanup failed: {cleanup_error!r}"
|
||||
else:
|
||||
detail = f"{detail}; cleanup failed: {cleanup_error!r}"
|
||||
results.append({"id": identifier, "disposition": disposition, "elapsed_ns": time.monotonic_ns() - started, "detail": detail})
|
||||
if disposition != "PASS" and arguments.fail_fast:
|
||||
break
|
||||
output = {"schema": "lazaret-ignored-workspace-corpus-result-v1", "results": results}
|
||||
Path(arguments.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(arguments.output).write_text(json.dumps(output, indent=2, sort_keys=True) + "\n")
|
||||
require(all(result["disposition"] == "PASS" for result in results), "one or more corpus cases failed")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
corpus = subparsers.add_parser("corpus")
|
||||
corpus.add_argument("--router", type=Path, required=True)
|
||||
corpus.add_argument("--git-sentry", type=Path, required=True)
|
||||
corpus.add_argument("--work-root", type=Path, required=True)
|
||||
corpus.add_argument("--output", type=Path, required=True)
|
||||
corpus.add_argument("--case")
|
||||
corpus.add_argument("--fail-fast", action="store_true")
|
||||
arguments = parser.parse_args()
|
||||
if arguments.command == "corpus":
|
||||
if arguments.case in (None, "git-sentry") and os.environ.get(NAMESPACE_MARKER) != "1":
|
||||
environment = os.environ.copy()
|
||||
environment[NAMESPACE_MARKER] = "1"
|
||||
os.execvpe(
|
||||
"unshare",
|
||||
[
|
||||
"unshare",
|
||||
"--user",
|
||||
"--map-root-user",
|
||||
"--mount",
|
||||
"--propagation",
|
||||
"private",
|
||||
"--",
|
||||
sys.executable,
|
||||
os.fspath(Path(__file__).resolve()),
|
||||
*sys.argv[1:],
|
||||
],
|
||||
environment,
|
||||
)
|
||||
run_corpus(arguments)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
81
tools/materialize-fuser
Executable file
81
tools/materialize-fuser
Executable file
|
|
@ -0,0 +1,81 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
version=0.17.0
|
||||
crate_checksum=80a5eca878900c2e39e9e52fd797954b7fc39eeefc8558257114bfea6a698fcf
|
||||
patch_file="$root/patches/fuser-$version-lock-flags.patch"
|
||||
destination="$root/.lazaret/vendor/fuser-$version"
|
||||
cargo_home="${CARGO_HOME:-$HOME/.cargo}"
|
||||
fetch_tests=false
|
||||
|
||||
if [ "${1:-}" = "--test-dependencies" ] && [ "$#" -eq 1 ]; then
|
||||
fetch_tests=true
|
||||
elif [ "$#" -ne 0 ]; then
|
||||
printf '%s\n' "usage: tools/materialize-fuser [--test-dependencies]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
fail() {
|
||||
printf '%s\n' "materialize-fuser: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
for command in cargo patch sha256sum tar; do
|
||||
command -v "$command" >/dev/null 2>&1 || fail "$command is required"
|
||||
done
|
||||
|
||||
[ -f "$patch_file" ] || fail "missing patch: $patch_file"
|
||||
patch_checksum="$(sha256sum "$patch_file" | cut -d ' ' -f 1)"
|
||||
stamp="$crate_checksum $patch_checksum"
|
||||
|
||||
fetch_test_dependencies() {
|
||||
local manifest="$destination/Cargo.toml"
|
||||
if ! cargo fetch --manifest-path "$manifest" --locked --offline >/dev/null 2>&1; then
|
||||
cargo fetch --manifest-path "$manifest" --locked
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -f "$destination/.lazaret-materialization" ] &&
|
||||
[ "$(tr -d '\n' < "$destination/.lazaret-materialization")" = "$stamp" ]; then
|
||||
if $fetch_tests; then
|
||||
fetch_test_dependencies
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
find_archive() {
|
||||
local candidate actual
|
||||
for candidate in "$cargo_home"/registry/cache/*/"fuser-$version.crate"; do
|
||||
[ -f "$candidate" ] || continue
|
||||
actual="$(sha256sum "$candidate" | cut -d ' ' -f 1)"
|
||||
if [ "$actual" = "$crate_checksum" ]; then
|
||||
printf '%s\n' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
archive="$(find_archive || true)"
|
||||
if [ -z "$archive" ]; then
|
||||
cargo --manifest-path "$root/Cargo.toml" --config 'paths=[]' fetch --locked
|
||||
archive="$(find_archive || true)"
|
||||
fi
|
||||
[ -n "$archive" ] || fail "Cargo did not fetch the checksum-matching fuser archive"
|
||||
|
||||
mkdir -p "$root/.lazaret/vendor"
|
||||
temporary="$(mktemp -d "$root/.lazaret/vendor/.fuser-$version.XXXXXX")"
|
||||
trap 'rm -rf "$temporary"' EXIT
|
||||
|
||||
tar -xzf "$archive" -C "$temporary"
|
||||
source="$temporary/fuser-$version"
|
||||
[ -d "$source" ] || fail "archive did not contain fuser-$version"
|
||||
patch --directory="$source" --strip=1 --batch --forward < "$patch_file"
|
||||
printf '%s\n' "$stamp" > "$source/.lazaret-materialization"
|
||||
|
||||
rm -rf "$destination"
|
||||
mv "$source" "$destination"
|
||||
if $fetch_tests; then
|
||||
fetch_test_dependencies
|
||||
fi
|
||||
4
tools/verify-ignored-workspace
Executable file
4
tools/verify-ignored-workspace
Executable file
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
exec uv run --python '>=3.11' --no-project --offline --no-python-downloads --no-cache \
|
||||
python3 "$(dirname "$0")/ignored_workspace_verifier.py" "$@"
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue