ce7cb702218693d08ea6d83464d2457c9250027c

services/project-delivery/k8s/workflows/controller/build.fsx

@@ -0,0 +1,199 @@
+#load "plan.fsx"
+
+open System
+open System.Formats.Tar
+open System.IO
+open System.IO.Compression
+open System.Security.Cryptography
+open System.Text.Json
+open System.Text.Json.Nodes
+open Contract
+open Plan
+
+let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
+let plan = read ()
+let source = "/source/repository/services/project-delivery/controller"
+let output = "/output"
+let layout = Path.Combine(output, "layout")
+let baseImage = "/toolchain/image-base"
+let go = "/toolchain/usr/lib/go/bin/go"
+
+require (validRevision revision && plan.SourceRevision = revision) "source revision mismatch"
+require (File.Exists go && File.Exists "/toolchain/.ready") "toolchain unavailable"
+require (Directory.Exists source) "controller source unavailable"
+require (File.Exists(Path.Combine(baseImage, "index.json"))) "base image unavailable"
+
+let executableMode =
+    UnixFileMode.UserRead
+    ||| UnixFileMode.UserWrite
+    ||| UnixFileMode.UserExecute
+    ||| UnixFileMode.GroupRead
+    ||| UnixFileMode.GroupExecute
+    ||| UnixFileMode.OtherRead
+    ||| UnixFileMode.OtherExecute
+
+let environment (architecture: string) =
+    [ "CGO_ENABLED", "0"
+      "GOCACHE", "/build-cache"
+      "GOENV", "off"
+      "GOOS", "linux"
+      "GOARCH", architecture
+      "GOMODCACHE", "/toolchain/modules"
+      "GOPATH", "/toolchain/gopath"
+      "GOPROXY", "off"
+      "GOROOT", "/toolchain/usr/lib/go"
+      "HOME", "/tmp" ]
+
+let build (architecture: string) =
+    let binary =
+        Path.Combine(output, "bin", architecture, "project-delivery-controller")
+
+    directory (Path.GetDirectoryName binary) |> ignore
+
+    requireSuccess
+        go
+        [ "build"
+          "-mod=readonly"
+          "-trimpath"
+          "-ldflags=-s -w -buildid="
+          "-o"
+          binary
+          "." ]
+        (Some source)
+        (environment architecture)
+    |> ignore
+
+    File.SetUnixFileMode(binary, executableMode)
+    let header = File.ReadAllBytes binary |> Array.truncate 20
+    let machine = if architecture = "amd64" then 62us else 183us
+
+    require
+        (header.Length = 20
+         && header[0..3] = [| 0x7fuy; 0x45uy; 0x4cuy; 0x46uy |]
+         && header[4..5] = [| 2uy; 1uy |]
+         && BitConverter.ToUInt16(header, 18) = machine)
+        $"invalid {architecture} executable"
+
+    binary
+
+let copyTree (sourceRoot: string) (targetRoot: string) =
+    if Directory.Exists targetRoot then
+        Directory.Delete(targetRoot, true)
+
+    directory targetRoot |> ignore
+
+    Directory.EnumerateDirectories(sourceRoot, "*", SearchOption.AllDirectories)
+    |> Seq.iter (fun path ->
+        directory (Path.Combine(targetRoot, Path.GetRelativePath(sourceRoot, path)))
+        |> ignore)
+
+    Directory.EnumerateFiles(sourceRoot, "*", SearchOption.AllDirectories)
+    |> Seq.iter (fun path -> File.Copy(path, Path.Combine(targetRoot, Path.GetRelativePath(sourceRoot, path))))
+
+let node (path: string) =
+    let value = JsonNode.Parse(File.ReadAllText path)
+    require (not <| isNull value) $"invalid JSON: {path}"
+    value
+
+let text (value: JsonNode) = value.GetValue<string>()
+
+let digestPath (root: string) (digest: string) =
+    require (validDigest digest) $"invalid OCI digest: {digest}"
+    Path.Combine(root, digest[7..])
+
+let blob (bytes: byte array) =
+    let digest = SHA256.HashData bytes |> Convert.ToHexString |> _.ToLowerInvariant()
+    File.WriteAllBytes(Path.Combine(layout, "blobs/sha256", digest), bytes)
+    $"sha256:{digest}", int64 bytes.Length
+
+let createLayer (architecture: string) (binary: string) =
+    let tarPath = Path.Combine(output, $"{architecture}.tar")
+    let gzipPath = tarPath + ".gz"
+
+    use tarOutput = File.Create tarPath
+    use writer = new TarWriter(tarOutput, TarEntryFormat.Pax, true)
+    let entry = PaxTarEntry(TarEntryType.RegularFile, "project-delivery-controller")
+    entry.Gid <- 0
+    entry.Uid <- 0
+    entry.Mode <- executableMode
+    entry.ModificationTime <- DateTimeOffset.UnixEpoch
+    entry.DataStream <- File.OpenRead binary
+    writer.WriteEntry entry
+    entry.DataStream.Dispose()
+    writer.Dispose()
+    tarOutput.Dispose()
+    let diff = sha256 tarPath
+
+    use input = File.OpenRead tarPath
+    use compressed = File.Create gzipPath
+    use encoder = new GZipStream(compressed, CompressionLevel.SmallestSize, true)
+    input.CopyTo encoder
+    encoder.Dispose()
+    compressed.Dispose()
+    let digest = sha256 gzipPath
+    File.Delete tarPath
+    diff, digest, FileInfo(gzipPath).Length, gzipPath
+
+let addLayer (architecture: string) (binary: string) =
+    let index = node (Path.Combine(layout, "index.json"))
+
+    let descriptor =
+        index["manifests"].AsArray()
+        |> Seq.find (fun item -> text (item["platform"]["architecture"]) = architecture)
+
+    let blobs = Path.Combine(layout, "blobs/sha256")
+    let manifestPath = digestPath blobs (text (descriptor["digest"]))
+    let manifest = node manifestPath
+    let configDescriptor = manifest["config"]
+    let config = node (digestPath blobs (text (configDescriptor["digest"])))
+    let diff, digest, size, layerPath = createLayer architecture binary
+    File.Move(layerPath, Path.Combine(blobs, digest), true)
+    let layer = JsonObject()
+    layer["mediaType"] <- "application/vnd.oci.image.layer.v1.tar+gzip"
+    layer["digest"] <- $"sha256:{digest}"
+    layer["size"] <- size
+    manifest["layers"].AsArray().Add(layer)
+    let diffId = $"sha256:{diff}"
+    let rootfs = config["rootfs"]
+    let diffIds = rootfs["diff_ids"].AsArray()
+    diffIds.Add(diffId)
+
+    let configDigest, configSize =
+        JsonSerializer.SerializeToUtf8Bytes(config, json) |> blob
+
+    configDescriptor["digest"] <- configDigest
+    configDescriptor["size"] <- configSize
+
+    let manifestDigest, manifestSize =
+        JsonSerializer.SerializeToUtf8Bytes(manifest, json) |> blob
+
+    descriptor["digest"] <- manifestDigest
+    descriptor["size"] <- manifestSize
+    File.WriteAllBytes(Path.Combine(layout, "index.json"), JsonSerializer.SerializeToUtf8Bytes(index, json))
+
+let nameIndex () =
+    let indexPath = Path.Combine(layout, "index.json")
+    let index = node indexPath
+    let nested = JsonObject()
+    nested["schemaVersion"] <- 2
+    nested["mediaType"] <- "application/vnd.oci.image.index.v1+json"
+    nested["manifests"] <- index["manifests"].DeepClone()
+    let digest, size = JsonSerializer.SerializeToUtf8Bytes(nested, json) |> blob
+    let descriptor = JsonObject()
+    descriptor["mediaType"] <- "application/vnd.oci.image.index.v1+json"
+    descriptor["digest"] <- digest
+    descriptor["size"] <- size
+    let annotations = JsonObject()
+    annotations["org.opencontainers.image.ref.name"] <- "delivery"
+    descriptor["annotations"] <- annotations
+    index["manifests"] <- JsonArray(descriptor)
+    File.WriteAllBytes(indexPath, JsonSerializer.SerializeToUtf8Bytes(index, json))
+    File.WriteAllText(Path.Combine(output, "image-digest"), digest + "\n")
+    digest
+
+copyTree baseImage layout
+
+[ "amd64"; "arm64" ]
+|> List.iter (fun architecture -> build architecture |> addLayer architecture)
+
+printfn "build=image digest=%s" (nameIndex ())

services/project-delivery/k8s/workflows/controller/cache.fsx

@@ -0,0 +1,246 @@
+#r "/opt/fsi/packages/AWSSDK.Core.dll"
+#r "/opt/fsi/packages/AWSSDK.S3.dll"
+#nowarn "0760"
+#load "plan.fsx"
+#load "cache-gc.fsx"
+
+open System
+open System.Collections.Generic
+open System.IO
+open System.Net
+open System.Net.Http
+open System.Security.Cryptography
+open System.Security.Cryptography.X509Certificates
+open System.Text
+open System.Text.Json
+open System.Text.Json.Serialization
+open System.Threading.Tasks
+open Amazon.Runtime
+open Amazon.S3
+open Amazon.S3.Model
+open Contract
+open Plan
+
+[<CLIMutable>]
+type CacheManifest =
+    { [<JsonPropertyName("schemaVersion")>]
+      SchemaVersion: int
+      [<JsonPropertyName("cacheVersion")>]
+      CacheVersion: string
+      [<JsonPropertyName("inputHashes")>]
+      InputHashes: Dictionary<string, string>
+      [<JsonPropertyName("payloadKey")>]
+      PayloadKey: string }
+
+let bucket = "project-delivery"
+
+let credential path =
+    require (File.Exists path) "cache credential unavailable"
+    let value = File.ReadAllText(path).Trim()
+    require (not <| String.IsNullOrWhiteSpace value) "cache credential unavailable"
+    value
+
+let root = X509CertificateLoader.LoadCertificateFromFile "/cache-ca/ca.crt"
+
+type CacheClientFactory(root: X509Certificate2) =
+    inherit HttpClientFactory()
+
+    override _.CreateHttpClient(_) =
+        let handler = new HttpClientHandler()
+
+        handler.ServerCertificateCustomValidationCallback <-
+            fun _ certificate chain errors -> validateCertificate root certificate chain errors
+
+        let client = new HttpClient(handler, true)
+        client.Timeout <- TimeSpan.FromMinutes 15.0
+        client
+
+let configuration = AmazonS3Config()
+configuration.ServiceURL <- "https://s3.quorate.internal"
+configuration.AuthenticationRegion <- "us-east-1"
+configuration.ForcePathStyle <- true
+configuration.HttpClientFactory <- CacheClientFactory(root)
+
+let client =
+    AmazonS3Client(
+        BasicAWSCredentials(credential "/cache-auth/accesskey", credential "/cache-auth/secretkey"),
+        configuration
+    )
+
+let result (operation: Task<'T>) = operation.GetAwaiter().GetResult()
+
+let parallel work =
+    Async.StartAsTask(Async.Parallel work, cancellationToken = cancellationToken)
+    |> result
+    |> ignore
+
+let sameHashes (left: Dictionary<string, string>) (right: Dictionary<string, string>) =
+    not (isNull (box left))
+    && not (isNull (box right))
+    && left.Count = right.Count
+    && left |> Seq.forall (fun item -> right.TryGetValue item.Key = (true, item.Value))
+
+let select (name: string) (assignment: Assignment) =
+    try
+        let request = GetObjectRequest(BucketName = bucket, Key = assignment.ExportKey)
+        use reply = result (client.GetObjectAsync(request, cancellationToken))
+        require (reply.HttpStatusCode = HttpStatusCode.OK) $"{name} cache manifest GET failed"
+        use reader = new StreamReader(reply.ResponseStream, Encoding.UTF8)
+        let manifest = reader.ReadToEnd() |> decode<CacheManifest>
+
+        if
+            manifest.SchemaVersion <> 1
+            || manifest.CacheVersion <> assignment.CacheVersion
+            || not (sameHashes manifest.InputHashes assignment.InputHashes)
+        then
+            printfn "cache=miss component=%s reason=incompatible" name
+            assignment
+        else
+            let payload = manifest.PayloadKey
+
+            require
+                (not <| isNull payload
+                 && payload.StartsWith("objects/", StringComparison.Ordinal)
+                 && payload.EndsWith(".tar.zst", StringComparison.Ordinal))
+                $"invalid {name} cache payload key"
+
+            let digest =
+                Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension payload)
+
+            require (validSha256 digest) $"invalid {name} cache digest"
+
+            let prefix =
+                assignment.ExportKey[.. assignment.ExportKey.Length - "_latest.json".Length - 1]
+
+            { assignment with
+                Origin = "cache"
+                Cache =
+                    { Key = prefix + payload
+                      Sha256 = digest } }
+    with :? AmazonS3Exception as error when
+        error.StatusCode = HttpStatusCode.Forbidden
+        || error.StatusCode = HttpStatusCode.NotFound ->
+        printfn "cache=miss component=%s reason=absent" name
+        assignment
+
+let download (item: CacheObject) (destination: string) =
+    let temporary = destination + ".incoming"
+
+    try
+        let request = GetObjectRequest(BucketName = bucket, Key = item.Key)
+        use reply = result (client.GetObjectAsync(request, cancellationToken))
+        require (reply.HttpStatusCode = HttpStatusCode.OK) "cache payload GET failed"
+        use input = reply.ResponseStream
+        use output = File.Create temporary
+        use algorithm = SHA256.Create()
+        use hashing = new CryptoStream(output, algorithm, CryptoStreamMode.Write, true)
+        input.CopyTo hashing
+        hashing.FlushFinalBlock()
+        let digest = Convert.ToHexString(algorithm.Hash).ToLowerInvariant()
+        require (digest = item.Sha256) "cache payload digest mismatch"
+        File.Move(temporary, destination, true)
+    with error ->
+        if File.Exists temporary then
+            File.Delete temporary
+
+        raise error
+
+let restore (name: string) (assignment: Assignment) (destination: string) =
+    let archive = Path.Combine("/tmp", Path.GetRandomFileName() + ".tar.zst")
+
+    try
+        download assignment.Cache archive
+        extractZstd archive destination
+        printfn "cache=restored component=%s" name
+    finally
+        if File.Exists archive then
+            File.Delete archive
+
+let objectDigest (key: string) =
+    try
+        let request = GetObjectMetadataRequest(BucketName = bucket, Key = key)
+        let reply = result (client.GetObjectMetadataAsync(request, cancellationToken))
+
+        require (reply.HttpStatusCode = HttpStatusCode.OK) "cache metadata HEAD failed"
+        Some reply.Metadata["x-amz-meta-sha256"]
+    with :? AmazonS3Exception as error when error.StatusCode = HttpStatusCode.NotFound ->
+        None
+
+let upload (key: string) (path: string) (digest: string) =
+    match objectDigest key with
+    | Some actual ->
+        require (actual = digest) "existing cache payload digest mismatch"
+        printfn "cache=present key=%s" key
+    | None ->
+        use input = File.OpenRead path
+
+        let request =
+            PutObjectRequest(BucketName = bucket, Key = key, InputStream = input, IfNoneMatch = "*")
+
+        request.AutoCloseStream <- false
+        request.AutoResetStreamPosition <- false
+        request.Metadata["sha256"] <- digest
+
+        try
+            let reply = result (client.PutObjectAsync(request, cancellationToken))
+            require (reply.HttpStatusCode = HttpStatusCode.OK) "cache payload PUT failed"
+            printfn "cache=uploaded key=%s bytes=%d" key input.Length
+        with :? AmazonS3Exception as error when error.StatusCode = HttpStatusCode.PreconditionFailed ->
+            require (objectDigest key = Some digest) "existing cache payload digest mismatch"
+            printfn "cache=present key=%s" key
+
+let putManifest (assignment: Assignment) (payloadKey: string) =
+    let manifest =
+        { SchemaVersion = 1
+          CacheVersion = assignment.CacheVersion
+          InputHashes = assignment.InputHashes
+          PayloadKey = payloadKey }
+
+    use input = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, json))
+
+    let request =
+        PutObjectRequest(BucketName = bucket, Key = assignment.ExportKey, InputStream = input)
+
+    request.ContentType <- "application/json"
+    let reply = result (client.PutObjectAsync(request, cancellationToken))
+    require (reply.HttpStatusCode = HttpStatusCode.OK) "cache manifest PUT failed"
+    CacheGc.prune client bucket assignment.ExportKey payloadKey cancellationToken
+
+let export (name: string) (assignment: Assignment) (root: string) (paths: string seq) =
+    let archive = Path.Combine("/tmp", Path.GetRandomFileName() + ".tar.zst")
+
+    try
+        let digest = createZstd root paths archive
+        let payloadKey = $"objects/{digest}.tar.zst"
+
+        let prefix =
+            assignment.ExportKey[.. assignment.ExportKey.Length - "_latest.json".Length - 1]
+
+        upload (prefix + payloadKey) archive digest
+        putManifest assignment payloadKey
+        printfn "cache=exported component=%s" name
+    finally
+        if File.Exists archive then
+            File.Delete archive
+
+match Environment.GetEnvironmentVariable "CACHE_ACTION" with
+| "pull" ->
+    let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
+    let plan = create revision select
+    write plan
+
+    [ "toolchain", plan.Toolchain, "/toolchain"
+      "go-build", plan.Build, "/build-cache" ]
+    |> List.choose (fun (name, assignment, destination) ->
+        if assignment.Origin = "cache" then
+            Some(async { restore name assignment destination })
+        else
+            None)
+    |> parallel
+| "export" ->
+    let plan = read ()
+
+    [ async { export "toolchain" plan.Toolchain "/toolchain" [ "usr/lib/go"; "image-base"; "modules"; ".ready" ] }
+      async { export "go-build" plan.Build "/build-cache" [ "." ] } ]
+    |> parallel
+| _ -> fail "invalid cache action"

services/project-delivery/k8s/workflows/controller/contract.fsx

@@ -0,0 +1,321 @@
+#r "System.Formats.Tar"
+#r "/opt/fsi/packages/ZstdSharp.dll"
+
+open System
+open System.Collections.Generic
+open System.Diagnostics
+open System.Formats.Tar
+open System.IO
+open System.IO.Compression
+open System.Net.Security
+open System.Runtime.InteropServices
+open System.Security.Cryptography
+open System.Security.Cryptography.X509Certificates
+open System.Text.Json
+open System.Text.Json.Serialization
+open System.Threading
+open System.Threading.Tasks
+open ZstdSharp
+open ZstdSharp.Unsafe
+
+[<CLIMutable>]
+type Target =
+    { [<JsonPropertyName("kind")>]
+      Kind: string
+      [<JsonPropertyName("repository")>]
+      Repository: string }
+
+[<CLIMutable>]
+type Published =
+    { [<JsonPropertyName("kind")>]
+      Kind: string
+      [<JsonPropertyName("repository")>]
+      Repository: string
+      [<JsonPropertyName("reference")>]
+      Reference: string }
+
+[<CLIMutable>]
+type Delivery =
+    { [<JsonPropertyName("sourceRevision")>]
+      SourceRevision: string
+      [<JsonPropertyName("targets")>]
+      Targets: Published array }
+
+type ProcessResult =
+    { Code: int
+      Error: string
+      Output: string }
+
+let json =
+    JsonSerializerOptions(
+        PropertyNamingPolicy = null,
+        WriteIndented = false,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
+    )
+
+let private stopping = new CancellationTokenSource()
+let mutable private signalExitCode = 143
+
+let private registerSignal signal exitCode =
+    PosixSignalRegistration.Create(
+        signal,
+        fun context ->
+            context.Cancel <- true
+            signalExitCode <- exitCode
+            stopping.Cancel()
+    )
+
+let private interrupt = registerSignal PosixSignal.SIGINT 130
+let private terminate = registerSignal PosixSignal.SIGTERM 143
+let cancellationToken = stopping.Token
+
+let private exitForSignal<'T> () =
+    Environment.Exit signalExitCode
+    Unchecked.defaultof<'T>
+
+let fail message = invalidOp message
+
+let require valid message =
+    if not valid then
+        fail message
+
+let directory (path: string) =
+    Directory.CreateDirectory path |> ignore
+    path
+
+let sha256 (path: string) =
+    use input = File.OpenRead path
+    Convert.ToHexString(SHA256.HashData input).ToLowerInvariant()
+
+let hashText (value: string) =
+    value
+    |> Text.Encoding.UTF8.GetBytes
+    |> SHA256.HashData
+    |> Convert.ToHexString
+    |> _.ToLowerInvariant()
+
+let validRevision (value: string) =
+    not <| isNull value
+    && (value.Length = 40 || value.Length = 64)
+    && value
+       |> Seq.forall (fun character -> Char.IsAsciiHexDigit character && not <| Char.IsUpper character)
+
+let validSha256 (value: string) =
+    not <| isNull value
+    && value.Length = 64
+    && value
+       |> Seq.forall (fun character -> Char.IsAsciiHexDigit character && not <| Char.IsUpper character)
+
+let validDigest (value: string) =
+    not <| isNull value
+    && value.StartsWith("sha256:", StringComparison.Ordinal)
+    && validSha256 value[7..]
+
+let decode<'T> (value: string) =
+    let result = JsonSerializer.Deserialize<'T>(value, json)
+    require (not <| isNull (box result)) "JSON document is missing"
+    result
+
+let sleep (milliseconds: int) =
+    try
+        Task.Delay(milliseconds, cancellationToken).GetAwaiter().GetResult()
+    with :? OperationCanceledException when cancellationToken.IsCancellationRequested ->
+        exitForSignal ()
+
+let run
+    (executable: string)
+    (arguments: string seq)
+    (workingDirectory: string option)
+    (environment: (string * string) seq)
+    =
+    let start = ProcessStartInfo executable
+    start.UseShellExecute <- false
+    start.RedirectStandardOutput <- true
+    start.RedirectStandardError <- true
+
+    workingDirectory |> Option.iter (fun path -> start.WorkingDirectory <- path)
+    arguments |> Seq.iter start.ArgumentList.Add
+    environment |> Seq.iter (fun (name, value) -> start.Environment[name] <- value)
+
+    use child = Process.Start start
+    let output = child.StandardOutput.ReadToEndAsync()
+    let error = child.StandardError.ReadToEndAsync()
+
+    try
+        child.WaitForExitAsync(cancellationToken).GetAwaiter().GetResult()
+    with :? OperationCanceledException when cancellationToken.IsCancellationRequested ->
+        if not child.HasExited then
+            child.Kill true
+            child.WaitForExit()
+
+        exitForSignal ()
+
+    { Code = child.ExitCode
+      Error = error.GetAwaiter().GetResult()
+      Output = output.GetAwaiter().GetResult() }
+
+let requireSuccess executable arguments workingDirectory environment =
+    let result = run executable arguments workingDirectory environment
+    require (result.Code = 0) $"{Path.GetFileName executable} failed: {result.Error.Trim()}"
+    result.Output
+
+let private descendants (root: string) =
+    seq {
+        let pending = Stack<string>(Directory.EnumerateFileSystemEntries root)
+
+        while pending.Count > 0 do
+            let path = pending.Pop()
+            yield path
+
+            let attributes = File.GetAttributes path
+
+            if
+                attributes.HasFlag FileAttributes.Directory
+                && not (attributes.HasFlag FileAttributes.ReparsePoint)
+            then
+                Directory.EnumerateFileSystemEntries path |> Seq.iter pending.Push
+    }
+
+let private tree (root: string) =
+    seq {
+        yield root
+
+        if Directory.Exists root then
+            yield! descendants root
+    }
+
+let private safePath (root: string) (name: string) =
+    require (not <| Path.IsPathFullyQualified name) "archive path is absolute"
+    let path = Path.GetFullPath(Path.Combine(root, name))
+    let prefix = root + string Path.DirectorySeparatorChar
+    require (path = root || path.StartsWith(prefix, StringComparison.Ordinal)) "archive path escapes destination"
+    path
+
+let private extract (input: Stream) (destination: string) =
+    let root = Path.GetFullPath destination
+    directory root |> ignore
+    let directoryModes = ResizeArray<string * UnixFileMode>()
+    use reader = new TarReader(input, leaveOpen = false)
+
+    for entry in
+        Seq.initInfinite (fun _ -> reader.GetNextEntry())
+        |> Seq.takeWhile (isNull >> not) do
+        let path = safePath root entry.Name
+
+        match entry.EntryType with
+        | TarEntryType.Directory ->
+            directory path |> ignore
+            directoryModes.Add(path, entry.Mode)
+        | TarEntryType.RegularFile
+        | TarEntryType.V7RegularFile
+        | TarEntryType.ContiguousFile ->
+            directory (Path.GetDirectoryName path) |> ignore
+            use output = File.Create path
+
+            if not <| isNull entry.DataStream then
+                entry.DataStream.CopyTo output
+
+            output.Dispose()
+            File.SetUnixFileMode(path, entry.Mode)
+        | TarEntryType.SymbolicLink ->
+            require (not <| Path.IsPathFullyQualified entry.LinkName) "archive link is absolute"
+
+            let target =
+                Path.GetFullPath(Path.Combine(Path.GetDirectoryName path, entry.LinkName))
+
+            let prefix = root + string Path.DirectorySeparatorChar
+
+            require
+                (target = root || target.StartsWith(prefix, StringComparison.Ordinal))
+                "archive link escapes destination"
+
+            directory (Path.GetDirectoryName path) |> ignore
+            File.CreateSymbolicLink(path, entry.LinkName) |> ignore
+        | TarEntryType.GlobalExtendedAttributes -> ()
+        | _ -> fail $"unsupported archive entry: {entry.Name}"
+
+    directoryModes
+    |> Seq.sortByDescending (fun (path, _) -> path.Length)
+    |> Seq.iter File.SetUnixFileMode
+
+let extractGzip (source: string) (destination: string) =
+    use input = File.OpenRead source
+    use gzip = new GZipStream(input, CompressionMode.Decompress)
+    extract gzip destination
+
+let extractZstd (source: string) (destination: string) =
+    use input = File.OpenRead source
+    use zstd = new DecompressionStream(input, 131072, false, false)
+    extract zstd destination
+
+let private relativeName (root: string) (path: string) =
+    Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/')
+
+let private writeEntry (writer: TarWriter) (root: string) (path: string) =
+    let name = relativeName root path
+    let attributes = File.GetAttributes path
+
+    if attributes.HasFlag FileAttributes.ReparsePoint then
+        let item: FileSystemInfo =
+            if attributes.HasFlag FileAttributes.Directory then
+                DirectoryInfo path
+            else
+                FileInfo path
+
+        require (not <| isNull item.LinkTarget) "invalid symbolic link"
+        require (not <| Path.IsPathFullyQualified item.LinkTarget) "archive link is absolute"
+        let entry = PaxTarEntry(TarEntryType.SymbolicLink, name)
+        entry.LinkName <- item.LinkTarget
+        entry.Mode <- File.GetUnixFileMode path
+        entry.ModificationTime <- DateTimeOffset.UnixEpoch
+        writer.WriteEntry entry
+    elif attributes.HasFlag FileAttributes.Directory then
+        let entry = PaxTarEntry(TarEntryType.Directory, name)
+        entry.Mode <- File.GetUnixFileMode path
+        entry.ModificationTime <- DateTimeOffset.UnixEpoch
+        writer.WriteEntry entry
+    else
+        let entry = PaxTarEntry(TarEntryType.RegularFile, name)
+        entry.Mode <- File.GetUnixFileMode path
+        entry.ModificationTime <- DateTimeOffset.UnixEpoch
+        entry.DataStream <- File.OpenRead path
+        writer.WriteEntry entry
+        entry.DataStream.Dispose()
+
+let createZstd (root: string) (paths: string seq) (destination: string) =
+    use algorithm = SHA256.Create()
+    use output = File.Create destination
+    use hashing = new CryptoStream(output, algorithm, CryptoStreamMode.Write, true)
+    use zstd = new CompressionStream(hashing, 1, 131072, true)
+    zstd.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, 2)
+    use writer = new TarWriter(zstd, TarEntryFormat.Pax, leaveOpen = false)
+
+    paths
+    |> Seq.collect (fun path -> tree (Path.Combine(root, path)))
+    |> Seq.distinct
+    |> Seq.sortBy (relativeName root)
+    |> Seq.iter (writeEntry writer root)
+
+    writer.Dispose()
+    zstd.Dispose()
+    hashing.FlushFinalBlock()
+    Convert.ToHexString(algorithm.Hash).ToLowerInvariant()
+
+let validateCertificate
+    (root: X509Certificate2)
+    (certificate: X509Certificate)
+    (providedChain: X509Chain)
+    (errors: SslPolicyErrors)
+    =
+    not (isNull certificate)
+    && not (isNull providedChain)
+    && not (errors.HasFlag SslPolicyErrors.RemoteCertificateNameMismatch)
+    && (use chain = new X509Chain()
+        chain.ChainPolicy.TrustMode <- X509ChainTrustMode.CustomRootTrust
+        chain.ChainPolicy.CustomTrustStore.Add root |> ignore
+        chain.ChainPolicy.RevocationMode <- X509RevocationMode.NoCheck
+
+        providedChain.ChainElements
+        |> Seq.iter (fun item -> chain.ChainPolicy.ExtraStore.Add item.Certificate |> ignore)
+
+        chain.Build(new X509Certificate2(certificate)))

services/project-delivery/k8s/workflows/controller/controller.apko.yaml

@@ -0,0 +1,22 @@
+contents:
+  keyring:
+    - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+  repositories:
+    - https://packages.wolfi.dev/os
+  packages:
+    - ca-certificates-bundle=20261020-r0
+    - wolfi-baselayout=20230201-r30
+accounts:
+  groups:
+    - groupname: application
+      gid: 65532
+  users:
+    - username: application
+      uid: 65532
+      gid: 65532
+  run-as: 65532
+entrypoint:
+  command: /project-delivery-controller
+environment:
+  PATH: /usr/bin:/bin
+  SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt

services/project-delivery/k8s/workflows/controller/initialize.fsx

@@ -0,0 +1,21 @@
+open System
+open System.IO
+
+[ "source/archive"
+  "source/repository"
+  "source/tmp"
+  "toolchain"
+  "toolchain/tmp"
+  "state/cache-tmp"
+  "output/go-cache"
+  "output/bin"
+  "output/layout"
+  "output/result"
+  "output/tmp"
+  "output/publish-tmp"
+  "output/cache-tmp" ]
+|> List.iter (fun path -> Directory.CreateDirectory(Path.Combine("/workspace", path)) |> ignore)
+
+match Environment.GetEnvironmentVariable "POD_NAME" with
+| value when String.IsNullOrWhiteSpace value -> invalidOp "pod name is unavailable"
+| value -> File.WriteAllText("/workspace/.claim-name", $"{value}-workflow-workspace")

services/project-delivery/k8s/workflows/controller/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - network-policy.yaml
+  - workflowtemplate.yaml
+configMapGenerator:
+  - name: controller-orch
+    namespace: project-delivery
+    files:
+      - build.fsx=build.fsx
+      - cache.fsx=cache.fsx
+      - cache-gc.fsx=../../../actions/cache-gc.fsx
+      - controller.apko.yaml=controller.apko.yaml
+      - contract.fsx=contract.fsx
+      - initialize.fsx=initialize.fsx
+      - plan.fsx=plan.fsx
+      - publish.fsx=publish.fsx
+      - source.fsx=source.fsx
+      - toolchain.fsx=toolchain.fsx
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    app.kubernetes.io/name: project-delivery
+    app.kubernetes.io/instance: controller

services/project-delivery/k8s/workflows/controller/network-policy.yaml

@@ -0,0 +1,128 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ctrl-source
+  namespace: project-delivery
+spec:
+  endpointSelector:
+    matchLabels:
+      delivery.quorate.dev/stage: ctrl-source
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - {port: "53", protocol: UDP}
+            - {port: "53", protocol: TCP}
+          rules:
+            dns:
+              - matchName: forgejo-http.forgejo.svc.cluster.local
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: forgejo
+            app.kubernetes.io/name: forgejo
+            app.kubernetes.io/instance: forgejo
+      toPorts:
+        - ports:
+            - {port: "3000", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ctrl-toolchain
+  namespace: project-delivery
+spec:
+  endpointSelector:
+    matchLabels:
+      delivery.quorate.dev/stage: ctrl-toolchain
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - {port: "53", protocol: UDP}
+            - {port: "53", protocol: TCP}
+          rules:
+            dns:
+              - matchName: packages.wolfi.dev
+              - matchName: proxy.golang.org
+              - matchName: storage.googleapis.com
+              - matchName: sum.golang.org
+    - toFQDNs:
+        - {matchName: packages.wolfi.dev}
+        - {matchName: proxy.golang.org}
+        - {matchName: storage.googleapis.com}
+        - {matchName: sum.golang.org}
+      toPorts:
+        - ports:
+            - {port: "443", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ctrl-cache
+  namespace: project-delivery
+spec:
+  endpointSelector:
+    matchExpressions:
+      - key: delivery.quorate.dev/stage
+        operator: In
+        values: [ctrl-cache-pull, ctrl-cache-export]
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - {port: "53", protocol: UDP}
+            - {port: "53", protocol: TCP}
+          rules:
+            dns:
+              - matchName: s3.quorate.internal
+    - toCIDR: [10.66.127.81/32]
+      toPorts:
+        - ports:
+            - {port: "443", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: versitygw
+            app.kubernetes.io/name: versitygw
+            app.kubernetes.io/instance: versitygw
+      toPorts:
+        - ports:
+            - {port: "7070", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ctrl-publication
+  namespace: project-delivery
+spec:
+  endpointSelector:
+    matchLabels:
+      delivery.quorate.dev/stage: publish-internal
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - {port: "53", protocol: UDP}
+            - {port: "53", protocol: TCP}
+    - toCIDR: [10.108.252.250/32, 10.66.127.81/32]
+      toPorts:
+        - ports:
+            - {port: "443", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: quay
+            app: quay
+      toPorts:
+        - ports:
+            - {port: "8443", protocol: TCP}

services/project-delivery/k8s/workflows/controller/plan.fsx

@@ -0,0 +1,157 @@
+#load "contract.fsx"
+
+open System
+open System.Collections.Generic
+open System.Diagnostics
+open System.IO
+open System.Text.Json
+open System.Text.Json.Serialization
+open Contract
+
+[<CLIMutable>]
+type CacheObject =
+    { [<JsonPropertyName("key")>]
+      Key: string
+      [<JsonPropertyName("sha256")>]
+      Sha256: string }
+
+[<CLIMutable>]
+type Assignment =
+    { [<JsonPropertyName("origin")>]
+      Origin: string
+      [<JsonPropertyName("cache")>]
+      Cache: CacheObject
+      [<JsonPropertyName("exportKey")>]
+      ExportKey: string
+      [<JsonPropertyName("cacheVersion")>]
+      CacheVersion: string
+      [<JsonPropertyName("inputHashes")>]
+      InputHashes: Dictionary<string, string> }
+
+[<CLIMutable>]
+type InputPlan =
+    { [<JsonPropertyName("version")>]
+      Version: int
+      [<JsonPropertyName("project")>]
+      Project: string
+      [<JsonPropertyName("sourceRevision")>]
+      SourceRevision: string
+      [<JsonPropertyName("toolchain")>]
+      Toolchain: Assignment
+      [<JsonPropertyName("build")>]
+      Build: Assignment }
+
+let private planPath = "/state/cache-plan.json"
+let private sourceRoot = "/source/repository"
+
+let private controllerRoot =
+    Path.Combine(sourceRoot, "services/project-delivery/controller")
+
+let private identity fields =
+    fields |> String.concat "\n" |> (fun value -> hashText (value + "\n"))
+
+let private inputHashes () =
+    let hashes = Dictionary<string, string>()
+
+    [ "services/project-delivery/controller/go.mod"
+      "services/project-delivery/controller/go.sum" ]
+    |> List.iter (fun path ->
+        let source = Path.Combine(sourceRoot, path)
+        require (File.Exists source) $"missing cache input: {path}"
+        hashes[path] <- sha256 source)
+
+    hashes
+
+let private cacheVersion name =
+    let files =
+        match name with
+        | "toolchain" ->
+            [ "cache.fsx"
+              "contract.fsx"
+              "controller.apko.yaml"
+              "plan.fsx"
+              "toolchain.fsx" ]
+        | "go-build" -> [ "build.fsx"; "cache.fsx"; "contract.fsx"; "plan.fsx"; "toolchain.fsx" ]
+        | _ -> fail "invalid cache name"
+
+    [ "controller-delivery-cache-v1"
+      name
+      "apko=1.2.39-r0"
+      "go-1.27=1.27.3-r0"
+      "ca-certificates-bundle=20261020-r0" ]
+    @ (files
+       |> List.map (fun path ->
+           let digest = sha256 (Path.Combine("/orchestration", path))
+           $"{path}={digest}"))
+    |> identity
+
+let private assignment (name: string) (hashes: Dictionary<string, string>) =
+    { Origin = "primary"
+      Cache = Unchecked.defaultof<CacheObject>
+      ExportKey = $"projects/controller/controller-delivery/cache/{name}/_latest.json"
+      CacheVersion = cacheVersion name
+      InputHashes = Dictionary<string, string>(hashes) }
+
+let create (revision: string) (select: string -> Assignment -> Assignment) =
+    require (validRevision revision) "invalid source revision"
+    require (Directory.Exists controllerRoot) "controller source is unavailable"
+    let hashes = inputHashes ()
+
+    { Version = 1
+      Project = "controller"
+      SourceRevision = revision
+      Toolchain = select "toolchain" (assignment "toolchain" hashes)
+      Build = select "go-build" (assignment "go-build" hashes) }
+
+let write (plan: InputPlan) =
+    directory (Path.GetDirectoryName planPath) |> ignore
+    let temporary = planPath + ".incoming"
+    File.WriteAllText(temporary, JsonSerializer.Serialize(plan, json))
+    File.Move(temporary, planPath, true)
+
+let private validateAssignment (name: string) (assignment: Assignment) =
+    require (not <| isNull (box assignment)) $"missing {name} assignment"
+
+    require
+        (assignment.ExportKey = $"projects/controller/controller-delivery/cache/{name}/_latest.json")
+        $"invalid {name} cache key"
+
+    require (validSha256 assignment.CacheVersion) $"invalid {name} cache version"
+    require (not <| isNull (box assignment.InputHashes)) $"missing {name} input hashes"
+
+    assignment.InputHashes
+    |> Seq.iter (fun item ->
+        require (not (String.IsNullOrWhiteSpace item.Key) && validSha256 item.Value) $"invalid {name} input hash")
+
+    match assignment.Origin with
+    | "primary" -> require (isNull (box assignment.Cache)) $"unexpected {name} cache object"
+    | "cache" ->
+        require
+            (not <| isNull (box assignment.Cache)
+             && assignment.Cache.Key.StartsWith(
+                 $"projects/controller/controller-delivery/cache/{name}/objects/",
+                 StringComparison.Ordinal
+             )
+             && assignment.Cache.Key.EndsWith(".tar.zst", StringComparison.Ordinal)
+             && validSha256 assignment.Cache.Sha256)
+            $"invalid {name} cache object"
+    | _ -> fail $"invalid {name} cache origin"
+
+let read () =
+    let elapsed = Stopwatch.StartNew()
+
+    while not (File.Exists planPath) && elapsed.Elapsed < TimeSpan.FromMinutes 15.0 do
+        sleep 100
+
+    require (File.Exists planPath) "cache plan unavailable"
+    let plan = File.ReadAllText(planPath) |> decode<InputPlan>
+
+    require
+        (plan.Version = 1
+         && plan.Project = "controller"
+         && validRevision plan.SourceRevision)
+        "invalid cache plan identity"
+
+    validateAssignment "toolchain" plan.Toolchain
+    validateAssignment "go-build" plan.Build
+    plan

services/project-delivery/k8s/workflows/controller/publish.fsx

@@ -0,0 +1,52 @@
+#r "/opt/fsi/packages/Oci.dll"
+#load "contract.fsx"
+
+open System
+open System.IO
+open System.Text.Json
+open Contract
+open Quorate.Oci
+
+let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
+let targets = Environment.GetEnvironmentVariable "TARGETS" |> decode<Target array>
+let repository = "quorate/project-delivery-controller"
+let layout = "/handoff/layout"
+let digest = File.ReadAllText("/handoff/image-digest").Trim()
+
+require (validRevision revision) "invalid source revision"
+require (validDigest digest) "invalid image digest"
+
+require
+    (targets.Length = 1
+     && targets[0].Kind = "QuayOCI"
+     && targets[0].Repository = repository)
+    "invalid delivery target"
+
+let publish () =
+    let auth = RegistryAuth.Load("/auth/.dockerconfigjson", "registry.quorate.internal")
+    use root = Registry.LoadRoot "/registry-ca/ca.crt"
+    use registry = new Registry(Uri("https://registry.quorate.internal"), auth, root)
+
+    registry
+        .PushAsync(repository, layout, "delivery", $"delivery-{revision}", cancellationToken)
+        .GetAwaiter()
+        .GetResult()
+
+let published = publish ()
+
+require (published = digest) "published digest mismatch"
+directory "/result" |> ignore
+
+File.WriteAllText(
+    "/result/delivery-result",
+    JsonSerializer.Serialize(
+        { SourceRevision = revision
+          Targets =
+            [| { Kind = "QuayOCI"
+                 Repository = repository
+                 Reference = $"registry.quorate.internal/{repository}@{digest}" } |] },
+        json
+    )
+)
+
+printfn "publish=%s digest=%s" repository digest

services/project-delivery/k8s/workflows/controller/source.fsx

@@ -0,0 +1,54 @@
+#load "contract.fsx"
+
+open System
+open System.IO
+open System.IO.Compression
+open System.Net.Http
+open System.Net.Http.Headers
+open Contract
+
+let repository = Environment.GetEnvironmentVariable "SOURCE_REPOSITORY"
+let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
+let archive = "/source/tmp/source.tar.gz"
+let incoming = "/source/archive"
+let destination = "/source/repository"
+
+require (repository = "https://forge.quorate.internal/quorate/cluster") "invalid source repository"
+require (validRevision revision) "invalid source revision"
+require (File.Exists "/source-auth/token") "source authentication unavailable"
+require (Directory.EnumerateFileSystemEntries(destination) |> Seq.isEmpty) "source destination is not empty"
+
+let token = File.ReadAllText("/source-auth/token").Trim()
+require (not <| String.IsNullOrWhiteSpace token) "source authentication unavailable"
+
+let download () =
+    use request =
+        new HttpRequestMessage(
+            HttpMethod.Get,
+            $"http://forgejo-http.forgejo.svc.cluster.local.:3000/api/v1/repos/quorate/cluster/archive/{revision}.tar.gz"
+        )
+
+    request.Headers.Authorization <- AuthenticationHeaderValue("token", token)
+    use client = new HttpClient(Timeout = TimeSpan.FromMinutes 10.0)
+
+    use reply =
+        client.Send(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
+
+    require reply.IsSuccessStatusCode $"source download failed: {int reply.StatusCode}"
+    use input = reply.Content.ReadAsStream(cancellationToken)
+    use output = File.Create archive
+    input.CopyTo output
+
+download ()
+extractGzip archive incoming
+
+let root =
+    Directory.EnumerateFileSystemEntries incoming
+    |> Seq.tryExactlyOne
+    |> Option.defaultWith (fun () -> fail "source archive has an invalid root")
+
+require (Directory.Exists root) "source archive root is not a directory"
+Directory.Delete(destination)
+Directory.Move(root, destination)
+File.WriteAllText("/source/.ready", revision + "\n")
+printfn "source=primary revision=%s" revision

services/project-delivery/k8s/workflows/controller/toolchain.fsx

@@ -0,0 +1,78 @@
+#load "plan.fsx"
+
+open System
+open System.Diagnostics
+open System.IO
+open System.Runtime.InteropServices
+open Contract
+open Plan
+
+let plan = read ()
+let root = "/toolchain"
+let ready = Path.Combine(root, ".ready")
+let baseImage = Path.Combine(root, "image-base")
+
+let environment =
+    [ "GOCACHE", Path.Combine(root, "download-cache")
+      "GOENV", "off"
+      "GOMODCACHE", Path.Combine(root, "modules")
+      "GOPATH", Path.Combine(root, "gopath")
+      "GOPROXY", "https://proxy.golang.org"
+      "GOROOT", Path.Combine(root, "usr/lib/go")
+      "GOSUMDB", "sum.golang.org"
+      "HOME", Path.Combine(root, "home") ]
+
+match plan.Toolchain.Origin with
+| "cache" ->
+    let elapsed = Stopwatch.StartNew()
+
+    while not (File.Exists ready) && elapsed.Elapsed < TimeSpan.FromMinutes 15.0 do
+        sleep 100
+
+    require (File.Exists ready && File.Exists(Path.Combine(baseImage, "index.json"))) "cached toolchain unavailable"
+    printfn "toolchain=cache"
+| "primary" ->
+    require (RuntimeInformation.ProcessArchitecture = Architecture.X64) "unsupported toolchain architecture"
+
+    requireSuccess
+        "/sbin/apk"
+        [ "add"
+          "--root"
+          root
+          "--arch"
+          "x86_64"
+          "--initdb"
+          "--no-scripts"
+          "--keys-dir"
+          "/etc/apk/keys"
+          "--repository"
+          "https://packages.wolfi.dev/os"
+          "ca-certificates-bundle=20261020-r0"
+          "go-1.27=1.27.3-r0"
+          "wolfi-baselayout=20230201-r30" ]
+        None
+        []
+    |> ignore
+
+    let go = Path.Combine(root, "usr/lib/go/bin/go")
+    let source = "/source/repository/services/project-delivery/controller"
+    directory (Path.Combine(root, "modules")) |> ignore
+    requireSuccess go [ "mod"; "download" ] (Some source) environment |> ignore
+    directory baseImage |> ignore
+
+    requireSuccess
+        "/usr/bin/apko"
+        [ "build"
+          "--arch"
+          "x86_64,arm64"
+          "--sbom=false"
+          "/orchestration/controller.apko.yaml"
+          "controller:base"
+          baseImage ]
+        None
+        environment
+    |> ignore
+
+    File.WriteAllText(ready, plan.SourceRevision + "\n")
+    printfn "toolchain=primary"
+| _ -> fail "invalid toolchain origin"

services/project-delivery/k8s/workflows/controller/workflowtemplate.yaml

@@ -0,0 +1,352 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: controller-delivery
+  namespace: project-delivery
+  labels:
+    app.kubernetes.io/name: project-delivery
+    app.kubernetes.io/instance: controller
+spec:
+  entrypoint: workflow
+  serviceAccountName: project-delivery-executor
+  automountServiceAccountToken: false
+  executor:
+    serviceAccountName: project-delivery-executor
+  parallelism: 3
+  podGC:
+    strategy: OnWorkflowCompletion
+  podMetadata:
+    labels:
+      app.kubernetes.io/name: project-delivery
+      delivery.quorate.dev/project: controller
+  podSpecPatch: |
+    dnsConfig:
+      options:
+        - name: ndots
+          value: "1"
+    containers:
+      - name: wait
+        volumeMounts:
+          - name: exec-sa-token
+            mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+            readOnly: true
+    volumes:
+      - name: exec-sa-token
+        secret: null
+        projected:
+          defaultMode: 0440
+          sources:
+            - serviceAccountToken:
+                expirationSeconds: 1800
+                path: token
+            - configMap:
+                name: kube-root-ca.crt
+                items:
+                  - key: ca.crt
+                    path: ca.crt
+            - downwardAPI:
+                items:
+                  - path: namespace
+                    fieldRef:
+                      apiVersion: v1
+                      fieldPath: metadata.namespace
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65532
+    runAsGroup: 65532
+    fsGroup: 65532
+    fsGroupChangePolicy: OnRootMismatch
+    seccompProfile:
+      type: RuntimeDefault
+  arguments:
+    parameters:
+      - name: source-repository
+      - name: source-ref
+      - name: source-revision
+      - name: targets
+  templates:
+    - name: workflow
+      steps:
+        - - name: initialize
+            template: initialize
+        - - name: delivery
+            template: delivery
+            arguments:
+              parameters:
+                - name: working-pvc
+                  value: "{{steps.initialize.outputs.parameters.working-pvc}}"
+      outputs:
+        parameters:
+          - name: delivery-result
+            valueFrom:
+              parameter: "{{steps.delivery.outputs.parameters.delivery-result}}"
+
+    - name: initialize
+      metadata:
+        labels:
+          delivery.quorate.dev/stage: ctrl-workspace-init
+      container:
+        image: &runner registry.quorate.internal/quorate/fsharp-runner@sha256:d7b904f4f0cae9fdba7c7345cdf08c91162245834a8a13658d93396272aeec83
+        command: &fsi [/usr/share/dotnet/dotnet, /opt/fsi/fsi.dll, --exec]
+        args: [/orchestration/initialize.fsx]
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.name
+        resources:
+          requests: {cpu: 100m, memory: 64Mi}
+          limits: {cpu: 500m, memory: 256Mi}
+        securityContext: &restricted
+          allowPrivilegeEscalation: false
+          capabilities: {drop: [ALL]}
+          readOnlyRootFilesystem: true
+        volumeMounts:
+          - {name: orchestration, mountPath: /orchestration, readOnly: true}
+          - {name: workflow-workspace, mountPath: /workspace}
+          - {name: temporary, mountPath: /tmp}
+      volumes:
+        - {name: orchestration, configMap: {name: controller-orch, defaultMode: 0444}}
+        - {name: temporary, emptyDir: {}}
+        - name: workflow-workspace
+          ephemeral:
+            volumeClaimTemplate:
+              spec:
+                accessModes: [ReadWriteOnce]
+                storageClassName: openebs-hostpath-ssd
+                resources:
+                  requests:
+                    storage: 5Gi
+      outputs:
+        parameters:
+          - name: working-pvc
+            valueFrom: {path: /workspace/.claim-name}
+
+    - name: delivery
+      inputs:
+        parameters:
+          - name: working-pvc
+      dag:
+        tasks:
+          - name: source-pull
+            template: source-pull
+            arguments:
+              parameters:
+                - {name: working-pvc, value: "{{inputs.parameters.working-pvc}}"}
+          - name: cache-pull
+            dependencies: [source-pull]
+            template: cache-pull
+            arguments:
+              parameters:
+                - {name: working-pvc, value: "{{inputs.parameters.working-pvc}}"}
+          - name: toolchain-pull
+            dependencies: [source-pull]
+            template: toolchain-pull
+            arguments:
+              parameters:
+                - {name: working-pvc, value: "{{inputs.parameters.working-pvc}}"}
+          - name: build
+            dependencies: [cache-pull, toolchain-pull]
+            template: build
+            arguments:
+              parameters:
+                - {name: working-pvc, value: "{{inputs.parameters.working-pvc}}"}
+          - name: image-publish
+            dependencies: [build]
+            template: image-publish
+            arguments:
+              parameters:
+                - {name: working-pvc, value: "{{inputs.parameters.working-pvc}}"}
+          - name: cache-export
+            dependencies: [build]
+            template: cache-export
+            continueOn:
+              failed: true
+              error: true
+            arguments:
+              parameters:
+                - {name: working-pvc, value: "{{inputs.parameters.working-pvc}}"}
+      outputs:
+        parameters:
+          - name: delivery-result
+            valueFrom:
+              parameter: "{{tasks.image-publish.outputs.parameters.delivery-result}}"
+
+    - name: source-pull
+      inputs: &working-pvc-input
+        parameters:
+          - name: working-pvc
+      metadata:
+        labels:
+          delivery.quorate.dev/stage: ctrl-source
+      container:
+        image: *runner
+        command: *fsi
+        args: [/orchestration/source.fsx]
+        env:
+          - {name: SOURCE_REPOSITORY, value: "{{workflow.parameters.source-repository}}"}
+          - {name: SOURCE_REVISION, value: "{{workflow.parameters.source-revision}}"}
+        resources:
+          requests: {cpu: 100m, memory: 128Mi}
+          limits: {cpu: "1", memory: 512Mi}
+        securityContext: *restricted
+        volumeMounts:
+          - {name: orchestration, mountPath: /orchestration, readOnly: true}
+          - {name: source-auth, mountPath: /source-auth, readOnly: true}
+          - {name: workflow-workspace, mountPath: /source, subPath: source}
+          - {name: workflow-workspace, mountPath: /tmp, subPath: source/tmp}
+      volumes:
+        - {name: orchestration, configMap: {name: controller-orch, defaultMode: 0444}}
+        - {name: source-auth, secret: {secretName: quorate-pull, defaultMode: 0440}}
+        - &workflow-workspace
+          name: workflow-workspace
+          persistentVolumeClaim:
+            claimName: "{{inputs.parameters.working-pvc}}"
+
+    - name: cache-pull
+      inputs: *working-pvc-input
+      metadata:
+        labels:
+          delivery.quorate.dev/stage: ctrl-cache-pull
+      container:
+        image: *runner
+        command: *fsi
+        args: [/orchestration/cache.fsx]
+        env:
+          - {name: CACHE_ACTION, value: pull}
+          - {name: SOURCE_REVISION, value: "{{workflow.parameters.source-revision}}"}
+        resources:
+          requests: {cpu: 500m, memory: 256Mi}
+          limits: {cpu: "2", memory: 1Gi}
+        securityContext: *restricted
+        volumeMounts:
+          - {name: orchestration, mountPath: /orchestration, readOnly: true}
+          - {name: cache-auth, mountPath: /cache-auth, readOnly: true}
+          - {name: cache-ca, mountPath: /cache-ca, readOnly: true}
+          - {name: workflow-workspace, mountPath: /source, subPath: source, readOnly: true}
+          - {name: workflow-workspace, mountPath: /toolchain, subPath: toolchain}
+          - {name: workflow-workspace, mountPath: /state, subPath: state}
+          - {name: workflow-workspace, mountPath: /build-cache, subPath: output/go-cache}
+          - {name: workflow-workspace, mountPath: /tmp, subPath: state/cache-tmp}
+      volumes:
+        - {name: orchestration, configMap: {name: controller-orch, defaultMode: 0444}}
+        - {name: cache-auth, secret: {secretName: project-delivery-cache-reader, defaultMode: 0440}}
+        - {name: cache-ca, secret: {secretName: quay-internal-ca, defaultMode: 0440}}
+        - *workflow-workspace
+
+    - name: toolchain-pull
+      inputs: *working-pvc-input
+      podSpecPatch: |
+        runtimeClassName: gvisor
+      metadata:
+        labels:
+          delivery.quorate.dev/stage: ctrl-toolchain
+      container:
+        image: *runner
+        command: *fsi
+        args: [/orchestration/toolchain.fsx]
+        resources:
+          requests: {cpu: "1", memory: 512Mi}
+          limits: {cpu: "4", memory: 2Gi}
+        securityContext: *restricted
+        volumeMounts:
+          - {name: orchestration, mountPath: /orchestration, readOnly: true}
+          - {name: workflow-workspace, mountPath: /source, subPath: source, readOnly: true}
+          - {name: workflow-workspace, mountPath: /toolchain, subPath: toolchain}
+          - {name: workflow-workspace, mountPath: /state, subPath: state, readOnly: true}
+          - {name: workflow-workspace, mountPath: /tmp, subPath: toolchain/tmp}
+      volumes:
+        - {name: orchestration, configMap: {name: controller-orch, defaultMode: 0444}}
+        - *workflow-workspace
+
+    - name: build
+      inputs: *working-pvc-input
+      podSpecPatch: |
+        runtimeClassName: gvisor
+      metadata:
+        labels:
+          delivery.quorate.dev/stage: ctrl-build
+      container:
+        image: *runner
+        command: *fsi
+        args: [/orchestration/build.fsx]
+        env:
+          - {name: SOURCE_REVISION, value: "{{workflow.parameters.source-revision}}"}
+        resources:
+          requests: {cpu: "2", memory: 1Gi}
+          limits: {cpu: "6", memory: 4Gi}
+        securityContext: *restricted
+        volumeMounts:
+          - {name: orchestration, mountPath: /orchestration, readOnly: true}
+          - {name: workflow-workspace, mountPath: /source, subPath: source, readOnly: true}
+          - {name: workflow-workspace, mountPath: /toolchain, subPath: toolchain, readOnly: true}
+          - {name: workflow-workspace, mountPath: /state, subPath: state, readOnly: true}
+          - {name: workflow-workspace, mountPath: /build-cache, subPath: output/go-cache}
+          - {name: workflow-workspace, mountPath: /output, subPath: output}
+          - {name: workflow-workspace, mountPath: /tmp, subPath: output/tmp}
+      volumes:
+        - {name: orchestration, configMap: {name: controller-orch, defaultMode: 0444}}
+        - *workflow-workspace
+
+    - name: image-publish
+      inputs: *working-pvc-input
+      metadata:
+        labels:
+          delivery.quorate.dev/stage: publish-internal
+      container:
+        image: *runner
+        command: *fsi
+        args: [/orchestration/publish.fsx]
+        env:
+          - {name: SOURCE_REVISION, value: "{{workflow.parameters.source-revision}}"}
+          - {name: TARGETS, value: "{{workflow.parameters.targets}}"}
+        resources:
+          requests: {cpu: 250m, memory: 256Mi}
+          limits: {cpu: "2", memory: 1Gi}
+        securityContext: *restricted
+        volumeMounts:
+          - {name: orchestration, mountPath: /orchestration, readOnly: true}
+          - {name: registry-auth, mountPath: /auth, readOnly: true}
+          - {name: registry-ca, mountPath: /registry-ca, readOnly: true}
+          - {name: workflow-workspace, mountPath: /handoff, subPath: output, readOnly: true}
+          - {name: workflow-workspace, mountPath: /result, subPath: output/result}
+          - {name: workflow-workspace, mountPath: /tmp, subPath: output/publish-tmp}
+      volumes:
+        - {name: orchestration, configMap: {name: controller-orch, defaultMode: 0444}}
+        - {name: registry-auth, secret: {secretName: quay-writer, defaultMode: 0440}}
+        - {name: registry-ca, secret: {secretName: quay-internal-ca, defaultMode: 0440}}
+        - *workflow-workspace
+      outputs:
+        parameters:
+          - name: delivery-result
+            valueFrom: {path: /result/delivery-result}
+
+    - name: cache-export
+      inputs: *working-pvc-input
+      metadata:
+        labels:
+          delivery.quorate.dev/stage: ctrl-cache-export
+      container:
+        image: *runner
+        command: *fsi
+        args: [/orchestration/cache.fsx]
+        env:
+          - {name: CACHE_ACTION, value: export}
+        resources:
+          requests: {cpu: "1", memory: 512Mi}
+          limits: {cpu: "3", memory: 2Gi}
+        securityContext: *restricted
+        volumeMounts:
+          - {name: orchestration, mountPath: /orchestration, readOnly: true}
+          - {name: cache-auth, mountPath: /cache-auth, readOnly: true}
+          - {name: cache-ca, mountPath: /cache-ca, readOnly: true}
+          - {name: workflow-workspace, mountPath: /toolchain, subPath: toolchain, readOnly: true}
+          - {name: workflow-workspace, mountPath: /state, subPath: state, readOnly: true}
+          - {name: workflow-workspace, mountPath: /build-cache, subPath: output/go-cache, readOnly: true}
+          - {name: workflow-workspace, mountPath: /tmp, subPath: output/cache-tmp}
+      volumes:
+        - {name: orchestration, configMap: {name: controller-orch, defaultMode: 0444}}
+        - {name: cache-auth, secret: {secretName: project-delivery-cache-writer, defaultMode: 0440}}
+        - {name: cache-ca, secret: {secretName: quay-internal-ca, defaultMode: 0440}}
+        - *workflow-workspace