ce7cb702218693d08ea6d83464d2457c9250027c

services/mirrored-image/k8s/bootstrap/access.yaml

@@ -0,0 +1,148 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: mirrored-image
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: mirror-bootstrap
+      app.kubernetes.io/instance: mirrored-image
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - {port: "53", protocol: UDP}
+            - {port: "53", protocol: TCP}
+    - toEntities: [kube-apiserver]
+      toPorts:
+        - ports:
+            - {port: "443", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: vault
+            app.kubernetes.io/name: vault
+            component: server
+      toPorts:
+        - ports:
+            - {port: "8200", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: zitadel
+            app.kubernetes.io/name: zitadel
+      toPorts:
+        - ports:
+            - {port: "8080", protocol: TCP}
+    - toCIDR: [10.108.252.250/32]
+      toPorts:
+        - ports:
+            - {port: "443", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: quay
+            app: quay
+      toPorts:
+        - ports:
+            - {port: "8443", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: vault
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: vault
+      component: server
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "8200", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: zitadel
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: zitadel
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "8080", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: quay
+spec:
+  endpointSelector:
+    matchLabels:
+      app: quay
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "8443", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: mirror-boot-replies
+spec:
+  nodeSelector: {}
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "32768", endPort: 60999, protocol: TCP}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mirror-login-key
+  namespace: zitadel
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [zitadel-login-service-key]
+    verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mirror-login-key
+  namespace: zitadel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mirror-login-key
+subjects:
+  - kind: ServiceAccount
+    name: mirror-bootstrap
+    namespace: mirrored-image

services/mirrored-image/k8s/bootstrap/account.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mirror-bootstrap
+  namespace: mirrored-image
+imagePullSecrets:
+  - name: quay-cluster-pull

services/mirrored-image/k8s/bootstrap/bootstrap-lib.fsx

@@ -0,0 +1,93 @@
+module Bootstrap
+
+open System
+
+type App =
+    { Name: string
+      User: string
+      Org: string
+      Scopes: string list
+      Client: string }
+
+type Session = { User: string; State: obj }
+
+type Store =
+    { Read: string -> Map<string, string>
+      Write: string -> Map<string, string> -> unit }
+
+type Quay =
+    { LoginAuth: unit -> string
+      Login: string -> string -> string -> Session
+      Access: string -> App -> unit
+      Application: string -> App -> string
+      Assign: string -> Session -> App -> unit
+      Issue: Session -> App -> string }
+
+type Config = { AdminPath: string }
+
+let apps =
+    [ { Name = "candidates-reader"
+        User = "candidates-bot"
+        Org = "candidates"
+        Scopes = [ "repo:read" ]
+        Client = "" }
+      { Name = "candidates-writer"
+        User = "candidates-bot"
+        Org = "candidates"
+        Scopes = [ "repo:create"; "repo:write" ]
+        Client = "" }
+      { Name = "mirror-reader"
+        User = "mirror-bot"
+        Org = "mirror"
+        Scopes = [ "repo:read" ]
+        Client = "" }
+      { Name = "mirror-writer"
+        User = "mirror-bot"
+        Org = "mirror"
+        Scopes = [ "repo:create"; "repo:write" ]
+        Client = "" } ]
+
+let personaPath user = $"secret/data/quay/personas/{user}"
+let tokenPath app = $"secret/data/quay/oauth/{app.Name}"
+
+let private field name (values: Map<string, string>) =
+    match Map.tryFind name values with
+    | Some value when not (String.IsNullOrWhiteSpace value) && value <> "PLACEHOLDER" -> value
+    | _ -> invalidOp "credential is absent"
+
+let run config store quay =
+    let pending =
+        apps
+        |> List.filter (fun app ->
+            let values = store.Read(tokenPath app)
+
+            match Map.tryFind "token" values with
+            | Some value when not (String.IsNullOrWhiteSpace value) && value <> "PLACEHOLDER" -> false
+            | _ -> true)
+
+    if not pending.IsEmpty then
+        let admin = store.Read config.AdminPath |> field "token"
+
+        let loginAuth = quay.LoginAuth()
+
+        let sessions =
+            pending
+            |> List.map _.User
+            |> List.distinct
+            |> List.map (fun user ->
+                let password = store.Read(personaPath user) |> field "password"
+                let session = quay.Login user password loginAuth
+
+                let app = pending |> List.find (fun app -> app.User = user)
+
+                quay.Access admin app
+                user, session)
+            |> Map.ofList
+
+        for definition in pending do
+            let client = quay.Application admin definition
+            let app = { definition with Client = client }
+            let session = sessions[app.User]
+            quay.Assign admin session app
+            let token = quay.Issue session app
+            store.Write (tokenPath app) (Map.ofList (List.singleton ("token", token)))

services/mirrored-image/k8s/bootstrap/bootstrap-tests.fsx

@@ -0,0 +1,127 @@
+#load "bootstrap-lib.fsx"
+
+open System
+open System.Collections.Generic
+open Bootstrap
+
+let require condition message =
+    if not condition then
+        invalidOp message
+
+let config = { AdminPath = "secret/data/bootstrap/admin" }
+
+let allPending () =
+    let reads = Dictionary<string, Map<string, string>>()
+
+    for app in apps do
+        reads[tokenPath app] <- Map.ofList (List.singleton ("token", "PLACEHOLDER"))
+
+    reads[config.AdminPath] <- Map.ofList (List.singleton ("token", "admin-value"))
+    reads[personaPath "candidates-bot"] <- Map.ofList (List.singleton ("password", "candidates-value"))
+    reads[personaPath "mirror-bot"] <- Map.ofList (List.singleton ("password", "mirror-value"))
+
+    let writes = ResizeArray<string * Map<string, string>>()
+    let logins = ResizeArray<string>()
+    let access = ResizeArray<string>()
+    let assigned = ResizeArray<App>()
+
+    let store =
+        { Read = fun path -> reads[path]
+          Write = fun path value -> writes.Add(path, value) }
+
+    let quay =
+        { LoginAuth = fun () -> "login-value"
+          Login =
+            fun user _ _ ->
+                logins.Add user
+                { User = user; State = null }
+          Access = fun _ app -> access.Add app.User
+          Application = fun _ app -> $"client-{app.Name}"
+          Assign = fun _ _ app -> assigned.Add app
+          Issue = fun _ app -> $"token-{app.Name}" }
+
+    run config store quay
+    require (Set.ofSeq logins = Set.ofList [ "candidates-bot"; "mirror-bot" ]) "personas were not logged in once"
+    require (Set.ofSeq access = Set.ofList [ "candidates-bot"; "mirror-bot" ]) "persona access was not reconciled once"
+    require (assigned.Count = 4) "OAuth assignments were incomplete"
+    require (writes.Count = 4) "OAuth tokens were not written"
+
+    for app in assigned do
+        require (app.Client = $"client-{app.Name}") "application client ID was not retained"
+
+    let writers = assigned |> Seq.filter (fun app -> app.Name.EndsWith "-writer")
+
+    for app in writers do
+        require (app.Scopes = [ "repo:create"; "repo:write" ]) "writer scopes were broader than prescribed"
+
+let complete () =
+    let mutable reads = 0
+    let mutable calls = 0
+
+    let store =
+        { Read =
+            fun _ ->
+                reads <- reads + 1
+                Map.ofList (List.singleton ("token", "existing"))
+          Write = fun _ _ -> calls <- calls + 1 }
+
+    let quay =
+        { LoginAuth =
+            fun () ->
+                calls <- calls + 1
+                ""
+          Login =
+            fun _ _ _ ->
+                calls <- calls + 1
+                { User = ""; State = null }
+          Access = fun _ _ -> calls <- calls + 1
+          Application =
+            fun _ _ ->
+                calls <- calls + 1
+                ""
+          Assign = fun _ _ _ -> calls <- calls + 1
+          Issue =
+            fun _ _ ->
+                calls <- calls + 1
+                "" }
+
+    run config store quay
+    require (reads = 4) "completed outputs were not inspected exactly once"
+    require (calls = 0) "completed bootstrap used privileged operations"
+
+let partial () =
+    let mutable logins = 0
+    let writes = ResizeArray<string>()
+
+    let store =
+        { Read =
+            fun path ->
+                if path = tokenPath apps[0] then
+                    Map.ofList (List.singleton ("token", "PLACEHOLDER"))
+                elif path = config.AdminPath then
+                    Map.ofList (List.singleton ("token", "value"))
+                elif path = personaPath "candidates-bot" then
+                    Map.ofList (List.singleton ("password", "value"))
+                else
+                    Map.ofList (List.singleton ("token", "existing"))
+          Write = fun path _ -> writes.Add path }
+
+    let quay =
+        { LoginAuth = fun () -> "login-token"
+          Login =
+            fun user _ _ ->
+                logins <- logins + 1
+                { User = user; State = null }
+          Access = fun _ _ -> ()
+          Application = fun _ _ -> "client"
+          Assign = fun _ _ _ -> ()
+          Issue = fun _ _ -> "token" }
+
+    run config store quay
+    require (logins = 1) "partial bootstrap logged in an unrelated persona"
+    require (Seq.toList writes = [ tokenPath apps[0] ]) "partial bootstrap changed unrelated outputs"
+
+allPending ()
+complete ()
+partial ()
+printfn "bootstrap tests passed"

services/mirrored-image/k8s/bootstrap/bootstrap.fsx

@@ -0,0 +1,39 @@
+#load "bootstrap-lib.fsx"
+#load "gateway.fsx"
+
+open System
+open Bootstrap
+open Gateway
+
+let env name =
+    let value = Environment.GetEnvironmentVariable name
+
+    if String.IsNullOrWhiteSpace value then
+        invalidOp $"{name} is unavailable"
+
+    value
+
+let store =
+    makeStore
+        { URL = env "VAULT_ADDR"
+          Role = env "VAULT_ROLE"
+          JwtPath = env "JWT_PATH"
+          CaPath = env "CA_PATH" }
+
+let loginKey =
+    readKubeKey
+        { URL = env "KUBE_URL"
+          TokenPath = env "JWT_PATH"
+          CaPath = env "KUBE_CA"
+          Namespace = env "LOGIN_NAMESPACE"
+          Secret = env "LOGIN_SECRET" }
+
+let quay =
+    makeQuay
+        { URL = env "QUAY_URL"
+          ZitadelAPI = env "ZITADEL_API"
+          ZitadelIssuer = env "ZITADEL_ISSUER"
+          CaPath = env "CA_PATH"
+          LoginKey = loginKey }
+
+run { AdminPath = env "ADMIN_PATH" } store quay

services/mirrored-image/k8s/bootstrap/gateway.fsx

@@ -0,0 +1,577 @@
+#load "bootstrap-lib.fsx"
+
+open System
+open System.Collections.Generic
+open System.IO
+open System.Net
+open System.Net.Http
+open System.Net.Http.Headers
+open System.Net.Http.Json
+open System.Net.Security
+open System.Security.Cryptography
+open System.Security.Cryptography.X509Certificates
+open System.Text
+open System.Text.Json
+open System.Text.Json.Nodes
+open Bootstrap
+
+type VaultConfig =
+    { URL: string
+      Role: string
+      JwtPath: string
+      CaPath: string }
+
+type QuayConfig =
+    { URL: string
+      ZitadelAPI: string
+      ZitadelIssuer: string
+      CaPath: string
+      LoginKey: string }
+
+type KubeConfig =
+    { URL: string
+      TokenPath: string
+      CaPath: string
+      Namespace: string
+      Secret: string }
+
+let private require condition message =
+    if not condition then
+        invalidOp message
+
+let private stringNode (value: string) = JsonValue.Create(value) :> JsonNode
+
+let private jsonObject (values: (string * JsonNode) list) =
+    let result = JsonObject()
+
+    for name, value in values do
+        result[name] <- value
+
+    result
+
+let private property (name: string) (root: JsonElement) =
+    let mutable value = Unchecked.defaultof<JsonElement>
+    require (root.TryGetProperty(name, &value)) $"response omitted {name}"
+    value
+
+let private stringProp name root =
+    let value = property name root
+    require (value.ValueKind = JsonValueKind.String) $"response {name} is not a string"
+
+    value.GetString()
+    |> Option.ofObj
+    |> Option.defaultWith (fun () -> invalidOp $"response {name} is empty")
+
+let private readJson operation status (response: HttpResponseMessage) =
+    use response = response
+    require (response.StatusCode = status) $"{operation} failed with HTTP {int response.StatusCode}"
+    use document = JsonDocument.Parse(response.Content.ReadAsStream())
+    document.RootElement.Clone()
+
+let private expect operation status (response: HttpResponseMessage) =
+    use response = response
+    require (response.StatusCode = status) $"{operation} failed with HTTP {int response.StatusCode}"
+
+let private handler caPath allowRedirect =
+    let handler = new HttpClientHandler(AllowAutoRedirect = allowRedirect)
+
+    if not (String.IsNullOrWhiteSpace caPath) then
+        let root = X509Certificate2.CreateFromPem(File.ReadAllText caPath)
+
+        handler.ServerCertificateCustomValidationCallback <-
+            fun _ certificate peerChain errors ->
+                if errors = SslPolicyErrors.None then
+                    true
+                elif isNull certificate || errors <> SslPolicyErrors.RemoteCertificateChainErrors then
+                    false
+                else
+                    use chain = new X509Chain()
+                    chain.ChainPolicy.TrustMode <- X509ChainTrustMode.CustomRootTrust
+                    chain.ChainPolicy.CustomTrustStore.Add root |> ignore
+                    chain.ChainPolicy.RevocationMode <- X509RevocationMode.NoCheck
+
+                    if not (isNull peerChain) then
+                        for element in peerChain.ChainElements do
+                            chain.ChainPolicy.ExtraStore.Add element.Certificate |> ignore
+
+                    chain.Build certificate
+
+    handler
+
+let private client caPath allowRedirect =
+    let client = new HttpClient(handler caPath allowRedirect, true)
+    client.Timeout <- TimeSpan.FromSeconds 30.
+    client
+
+let private sessionClient caPath =
+    let handler = handler caPath false
+    handler.CookieContainer <- CookieContainer()
+    let client = new HttpClient(handler, true)
+    client.Timeout <- TimeSpan.FromSeconds 30.
+    client
+
+let private sendRequest
+    (client: HttpClient)
+    (method: HttpMethod)
+    (endpoint: string)
+    (bearer: string option)
+    (basic: (string * string) option)
+    (csrf: string option)
+    (body: JsonNode option)
+    (host: string option)
+    (headers: (string * string) list)
+    =
+    use message = new HttpRequestMessage(method, endpoint)
+
+    host |> Option.iter (fun value -> message.Headers.Host <- value)
+    headers |> List.iter (fun (name, value) -> message.Headers.Add(name, value))
+
+    match bearer, basic with
+    | Some token, _ -> message.Headers.Authorization <- AuthenticationHeaderValue("Bearer", token)
+    | _, Some(user, token) ->
+        let encoded = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{user}:{token}"))
+        message.Headers.Authorization <- AuthenticationHeaderValue("Basic", encoded)
+    | _ -> ()
+
+    csrf |> Option.iter (fun token -> message.Headers.Add("X-CSRF-Token", token))
+
+    body
+    |> Option.iter (fun value ->
+        let content =
+            new StringContent(value.ToJsonString(), Encoding.UTF8, "application/json")
+
+        content.Headers.ContentType.CharSet <- null
+        message.Content <- content)
+
+    client.Send message
+
+let private request client method endpoint bearer basic csrf body =
+    sendRequest client method endpoint bearer basic csrf body None []
+
+let private endpoint (root: string) (path: string) =
+    $"{root.TrimEnd('/')}/{path.TrimStart('/')}"
+
+let private base64Url (content: byte array) =
+    Convert.ToBase64String content
+    |> _.TrimEnd('=')
+    |> _.Replace('+', '-')
+    |> _.Replace('/', '_')
+
+let private systemToken zitadel (key: string) =
+    let issued = DateTimeOffset.UtcNow.ToUnixTimeSeconds()
+
+    let header = JsonSerializer.SerializeToUtf8Bytes {| alg = "RS256" |} |> base64Url
+
+    let claims =
+        JsonSerializer.SerializeToUtf8Bytes
+            {| iss = "login-client"
+               sub = "login-client"
+               aud = zitadel
+               iat = issued
+               exp = issued + 300L |}
+        |> base64Url
+
+    let unsigned = $"{header}.{claims}"
+    use rsa = RSA.Create()
+    rsa.ImportFromPem key
+
+    let signature =
+        rsa.SignData(Encoding.UTF8.GetBytes unsigned, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1)
+        |> base64Url
+
+    $"{unsigned}.{signature}"
+
+let readKubeKeyWith (client: HttpClient) (config: KubeConfig) =
+    let token = File.ReadAllText(config.TokenPath).Trim()
+    require (not (String.IsNullOrWhiteSpace token)) "service account token is empty"
+
+    let path =
+        $"api/v1/namespaces/{Uri.EscapeDataString config.Namespace}/secrets/{Uri.EscapeDataString config.Secret}"
+
+    let response =
+        request client HttpMethod.Get (endpoint config.URL path) (Some token) None None None
+
+    let data =
+        readJson "read ZITADEL login key" HttpStatusCode.OK response |> property "data"
+
+    let encoded = stringProp "tls.key" data
+    Encoding.UTF8.GetString(Convert.FromBase64String encoded)
+
+let readKubeKey (config: KubeConfig) =
+    readKubeKeyWith (client config.CaPath true) config
+
+let private vaultRequest
+    (client: HttpClient)
+    (method: HttpMethod)
+    (endpoint: string)
+    (token: string)
+    (body: JsonNode option)
+    =
+    use message = new HttpRequestMessage(method, endpoint)
+    message.Headers.Add("X-Vault-Token", token)
+    body |> Option.iter (fun value -> message.Content <- JsonContent.Create value)
+    client.Send message
+
+let makeStoreWith (client: HttpClient) (config: VaultConfig) =
+    let jwt = File.ReadAllText(config.JwtPath).Trim()
+    require (not (String.IsNullOrWhiteSpace jwt)) "service account token is empty"
+
+    let payload = jsonObject [ "role", stringNode config.Role; "jwt", stringNode jwt ]
+
+    let response =
+        request client HttpMethod.Post (endpoint config.URL "v1/auth/kubernetes/login") None None None (Some payload)
+
+    let login = readJson "Vault login" HttpStatusCode.OK response
+    let token = login |> property "auth" |> stringProp "client_token"
+
+    let read (path: string) =
+        let response =
+            vaultRequest client HttpMethod.Get (endpoint config.URL $"v1/{path}") token None
+
+        let root = readJson "Vault read" HttpStatusCode.OK response
+        let data = root |> property "data" |> property "data"
+
+        data.EnumerateObject()
+        |> Seq.map (fun item -> item.Name, item.Value.GetString())
+        |> Seq.choose (fun (name, value) -> value |> Option.ofObj |> Option.map (fun text -> name, text))
+        |> Map.ofSeq
+
+    let write (path: string) (values: Map<string, string>) =
+        let data = JsonObject()
+
+        for KeyValue(name, value) in values do
+            data[name] <- stringNode value
+
+        let payload = jsonObject (List.singleton ("data", data))
+
+        let response =
+            vaultRequest client HttpMethod.Post (endpoint config.URL $"v1/{path}") token (Some payload)
+
+        expect "Vault write" HttpStatusCode.OK response
+
+    { Read = read; Write = write }
+
+let makeStore (config: VaultConfig) =
+    makeStoreWith (client config.CaPath true) config
+
+let private query values =
+    let pairs = values |> List.map (fun (name, value) -> KeyValuePair(name, value))
+    use content = new FormUrlEncodedContent(pairs)
+    content.ReadAsStringAsync().GetAwaiter().GetResult()
+
+let private assignments quayURL (session: Session) =
+    let client = session.State :?> HttpClient
+
+    let response =
+        request client HttpMethod.Get (endpoint quayURL "/api/v1/user/assignedauthorization") None None None None
+
+    readJson "list assignments" HttpStatusCode.OK response
+    |> property "authorizations"
+    |> _.EnumerateArray()
+    |> Seq.map _.Clone()
+    |> Seq.toList
+
+let private assignment quayURL app items =
+    let matches =
+        items
+        |> List.filter (fun item -> item |> property "application" |> stringProp "clientId" = app.Client)
+
+    match matches with
+    | [] -> None
+    | [ item ] ->
+        let scopes =
+            item
+            |> property "scopes"
+            |> _.EnumerateArray()
+            |> Seq.map (stringProp "scope")
+            |> Seq.sort
+            |> Seq.toList
+
+        require (scopes = List.sort app.Scopes) "assigned authorization has different scopes"
+
+        require
+            (stringProp "redirectUri" item = endpoint quayURL "/oauth/localapp")
+            "assigned authorization has a different redirect URI"
+
+        require (stringProp "responseType" item = "token") "assigned authorization has a different response type"
+        Some item
+    | _ -> invalidOp "multiple assigned authorizations use the same client ID"
+
+let makeQuayWith (adminClient: HttpClient) makeSession (config: QuayConfig) =
+    let quayURL path = endpoint config.URL path
+    let zitadelURL path = endpoint config.ZitadelIssuer path
+    let issuer = Uri config.ZitadelIssuer
+    let apiRoot = Uri config.ZitadelAPI
+
+    let zitadelRequest client method url bearer body =
+        let source = Uri url
+        let builder = UriBuilder apiRoot
+        builder.Path <- source.AbsolutePath
+        builder.Query <- source.Query.TrimStart('?')
+
+        sendRequest
+            client
+            method
+            (builder.Uri.ToString())
+            bearer
+            None
+            None
+            body
+            (Some issuer.Host)
+            [ ("X-Zitadel-Public-Host", issuer.Host) ]
+
+    let login user password loginAuth =
+        let client = makeSession ()
+
+        let csrfRoot =
+            request client HttpMethod.Get (quayURL "/csrf_token") None None None None
+
+        let csrf =
+            readJson "get Quay CSRF token" HttpStatusCode.OK csrfRoot
+            |> stringProp "csrf_token"
+
+        let payload = jsonObject (List.singleton ("kind", stringNode "login"))
+
+        let externalResponse =
+            request
+                client
+                HttpMethod.Post
+                (quayURL "/api/v1/externallogin/zitadel")
+                None
+                None
+                (Some csrf)
+                (Some payload)
+
+        let external = readJson "start Quay login" HttpStatusCode.OK externalResponse
+        let authURL = stringProp "auth_url" external
+        use started = zitadelRequest client HttpMethod.Get authURL None None
+
+        require
+            (int started.StatusCode >= 300 && int started.StatusCode < 400)
+            $"start ZITADEL login failed with HTTP {int started.StatusCode}"
+
+        require (not (isNull started.Headers.Location)) "ZITADEL login returned no redirect"
+        let loginURL = Uri(Uri(zitadelURL "/"), started.Headers.Location)
+        let requestID = loginURL.Query.TrimStart('?').Split('&')
+
+        let requestID =
+            requestID
+            |> Array.tryPick (fun item ->
+                match item.Split('=', 2) with
+                | [| name; value |] when name = "authRequest" -> Some(Uri.UnescapeDataString value)
+                | _ -> None)
+            |> Option.defaultWith (fun () -> invalidOp "ZITADEL redirect omitted authRequest")
+
+        let userCheck =
+            jsonObject (
+                List.singleton (
+                    "checks",
+                    jsonObject (List.singleton ("user", jsonObject (List.singleton ("loginName", stringNode user))))
+                )
+            )
+
+        let createdResponse =
+            zitadelRequest client HttpMethod.Post (zitadelURL "/v2/sessions") (Some loginAuth) (Some userCheck)
+
+        let created =
+            readJson "create ZITADEL session" HttpStatusCode.Created createdResponse
+
+        let sessionID = stringProp "sessionId" created
+
+        let passwordCheck =
+            jsonObject (
+                List.singleton (
+                    "checks",
+                    jsonObject (
+                        List.singleton ("password", jsonObject (List.singleton ("password", stringNode password)))
+                    )
+                )
+            )
+
+        let checkedResponse =
+            zitadelRequest
+                client
+                HttpMethod.Patch
+                (zitadelURL $"/v2/sessions/{Uri.EscapeDataString sessionID}")
+                (Some loginAuth)
+                (Some passwordCheck)
+
+        let passwordResult =
+            readJson "check ZITADEL password" HttpStatusCode.OK checkedResponse
+
+        let sessionToken = stringProp "sessionToken" passwordResult
+
+        let finish =
+            jsonObject (
+                List.singleton (
+                    "session",
+                    jsonObject [ "sessionId", stringNode sessionID; "sessionToken", stringNode sessionToken ]
+                )
+            )
+
+        let finishResponse =
+            zitadelRequest
+                client
+                HttpMethod.Post
+                (zitadelURL $"/v2/oidc/auth_requests/{Uri.EscapeDataString requestID}")
+                (Some loginAuth)
+                (Some finish)
+
+        let finished = readJson "finish ZITADEL request" HttpStatusCode.OK finishResponse
+        let callback = stringProp "callbackUrl" finished
+        use callbackResponse = request client HttpMethod.Get callback None None None None
+
+        require
+            (int callbackResponse.StatusCode >= 200 && int callbackResponse.StatusCode < 400)
+            $"finish Quay login failed with HTTP {int callbackResponse.StatusCode}"
+
+        { User = user; State = client }
+
+    let admin method path token body operation status =
+        request adminClient method (quayURL path) (Some token) None None body
+        |> expect operation status
+
+    let adminJson method path token body operation =
+        request adminClient method (quayURL path) (Some token) None None body
+        |> readJson operation HttpStatusCode.OK
+
+    let access adminToken app =
+        let teamPath =
+            $"/api/v1/organization/{Uri.EscapeDataString app.Org}/team/mirrored-image"
+
+        let team = jsonObject (List.singleton ("role", stringNode "creator"))
+        admin HttpMethod.Put teamPath adminToken (Some team) "ensure Quay team" HttpStatusCode.OK
+
+        let memberPath = $"{teamPath}/members/{Uri.EscapeDataString app.User}"
+        admin HttpMethod.Put memberPath adminToken None "ensure Quay member" HttpStatusCode.OK
+        let protoPath = $"/api/v1/organization/{Uri.EscapeDataString app.Org}/prototypes"
+
+        let listed =
+            adminJson HttpMethod.Get protoPath adminToken None "list Quay prototypes"
+
+        let exists =
+            listed
+            |> property "prototypes"
+            |> _.EnumerateArray()
+            |> Seq.exists (fun item ->
+                let delegateValue = property "delegate" item
+
+                stringProp "role" item = "write"
+                && stringProp "name" delegateValue = "mirrored-image"
+                && stringProp "kind" delegateValue = "team")
+
+        if not exists then
+            let payload =
+                jsonObject
+                    [ "role", stringNode "write"
+                      "delegate", jsonObject [ "name", stringNode "mirrored-image"; "kind", stringNode "team" ] ]
+
+            admin HttpMethod.Post protoPath adminToken (Some payload) "create Quay prototype" HttpStatusCode.OK
+
+    let application adminToken app =
+        let path = $"/api/v1/organization/{Uri.EscapeDataString app.Org}/applications"
+        let listed = adminJson HttpMethod.Get path adminToken None "list Quay applications"
+        let redirect = quayURL "/oauth/localapp"
+
+        let existing =
+            listed
+            |> property "applications"
+            |> _.EnumerateArray()
+            |> Seq.tryFind (fun item -> stringProp "name" item = app.Name)
+
+        match existing with
+        | Some item ->
+            require (stringProp "redirect_uri" item = redirect) "existing application has a different redirect URI"
+            stringProp "client_id" item
+        | None ->
+            let payload =
+                jsonObject
+                    [ "name", stringNode app.Name
+                      "redirect_uri", stringNode redirect
+                      "application_uri", stringNode redirect ]
+
+            adminJson HttpMethod.Post path adminToken (Some payload) "create Quay application"
+            |> stringProp "client_id"
+
+    let assign adminToken session app =
+        if assignment config.URL app (assignments config.URL session) |> Option.isNone then
+            let values =
+                [ "username", app.User
+                  "response_type", "token"
+                  "client_id", app.Client
+                  "scope", String.concat " " app.Scopes
+                  "redirect_uri", quayURL "/oauth/localapp"
+                  "format", "json" ]
+
+            let assignPath = quayURL "/oauth/authorize/assignuser"
+            let url = $"{assignPath}?{query values}"
+
+            request adminClient HttpMethod.Post url None (Some("$oauthtoken", adminToken)) None None
+            |> expect "assign Quay authorization" HttpStatusCode.OK
+
+            assignment config.URL app (assignments config.URL session)
+            |> Option.defaultWith (fun () -> invalidOp "assigned authorization is absent")
+            |> ignore
+
+    let issue session app =
+        let client = session.State :?> HttpClient
+
+        let assigned =
+            assignment config.URL app (assignments config.URL session)
+            |> Option.defaultWith (fun () -> invalidOp "assigned authorization is absent")
+
+        let scopes =
+            assigned
+            |> property "scopes"
+            |> _.EnumerateArray()
+            |> Seq.map (stringProp "scope")
+            |> String.concat " "
+
+        let application = property "application" assigned
+
+        let values =
+            [ "response_type", stringProp "responseType" assigned
+              "client_id", stringProp "clientId" application
+              "scope", scopes
+              "redirect_uri", stringProp "redirectUri" assigned
+              "assignment_uuid", stringProp "uuid" assigned ]
+
+        let approvePath = quayURL "/oauth/authorize"
+        let approveQuery = query (values @ List.singleton ("format", "json"))
+        let approveURL = $"{approvePath}?{approveQuery}"
+        let approvalResponse = request client HttpMethod.Get approveURL None None None None
+
+        let csrf =
+            readJson "load Quay authorization" HttpStatusCode.OK approvalResponse
+            |> stringProp "csrf_token_val"
+
+        use message = new HttpRequestMessage(HttpMethod.Post, quayURL "/oauth/authorizeapp")
+
+        message.Content <-
+            new FormUrlEncodedContent(values |> List.map (fun (name, value) -> KeyValuePair(name, value)))
+
+        message.Headers.Add("X-CSRF-Token", csrf)
+        use response = client.Send message
+
+        require
+            (response.StatusCode = HttpStatusCode.Found)
+            $"issue Quay token failed with HTTP {int response.StatusCode}"
+
+        require (not (isNull response.Headers.Location)) "Quay token response omitted redirect"
+
+        response.Headers.Location.Fragment.TrimStart('#').Split('&')
+        |> Array.tryPick (fun item ->
+            match item.Split('=', 2) with
+            | [| name; value |] when name = "access_token" -> Some(Uri.UnescapeDataString value)
+            | _ -> None)
+        |> Option.defaultWith (fun () -> invalidOp "Quay token response omitted access token")
+
+    { LoginAuth = fun () -> systemToken config.ZitadelIssuer config.LoginKey
+      Login = login
+      Access = access
+      Application = application
+      Assign = assign
+      Issue = issue }
+
+let makeQuay (config: QuayConfig) =
+    makeQuayWith (client config.CaPath false) (fun () -> sessionClient config.CaPath) config

services/mirrored-image/k8s/bootstrap/job.yaml

@@ -0,0 +1,59 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mirror-bootstrap
+  namespace: mirrored-image
+  annotations:
+    kustomize.toolkit.fluxcd.io/force: Enabled
+  labels:
+    app.kubernetes.io/name: mirror-bootstrap
+    app.kubernetes.io/instance: mirrored-image
+spec:
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mirror-bootstrap
+        app.kubernetes.io/instance: mirrored-image
+    spec:
+      serviceAccountName: mirror-bootstrap
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: bootstrap
+          image: registry.quorate.internal/quorate/fsharp-runner@sha256:e4cbf2cfcbd903ff4600590b84a08c0e09d95fa1e2edda4daf8d32a6aeda35f7
+          command: [/usr/share/dotnet/dotnet, /opt/fsi/fsi.dll, --exec]
+          args: [/bootstrap/bootstrap.fsx]
+          env:
+            - {name: VAULT_ADDR, value: https://vault.vault.svc.cluster.local:8200}
+            - {name: VAULT_ROLE, value: mirror-bootstrap}
+            - {name: JWT_PATH, value: /var/run/secrets/kubernetes.io/serviceaccount/token}
+            - {name: CA_PATH, value: /registry-ca/ca.crt}
+            - {name: QUAY_URL, value: https://registry.quorate.internal}
+            - {name: ZITADEL_API, value: http://zitadel.zitadel.svc.cluster.local:8080}
+            - {name: ZITADEL_ISSUER, value: https://auth.quorate.dev}
+            - {name: KUBE_URL, value: https://kubernetes.default.svc}
+            - {name: KUBE_CA, value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt}
+            - {name: LOGIN_NAMESPACE, value: zitadel}
+            - {name: LOGIN_SECRET, value: zitadel-login-service-key}
+            - {name: ADMIN_PATH, value: secret/data/quay/bootstrap}
+          resources:
+            requests: {cpu: 50m, memory: 64Mi}
+            limits: {cpu: 500m, memory: 256Mi}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - {name: scripts, mountPath: /bootstrap, readOnly: true}
+            - {name: registry-ca, mountPath: /registry-ca, readOnly: true}
+            - {name: temporary, mountPath: /tmp}
+      volumes:
+        - {name: scripts, configMap: {name: mirror-bootstrap, defaultMode: 0444}}
+        - {name: registry-ca, secret: {secretName: mirror-ca, defaultMode: 0440}}
+        - {name: temporary, emptyDir: {}}

services/mirrored-image/k8s/bootstrap/kustomization.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - account.yaml
+  - access.yaml
+  - job.yaml
+configMapGenerator:
+  - name: mirror-bootstrap
+    namespace: mirrored-image
+    files:
+      - bootstrap.fsx=bootstrap.fsx
+      - bootstrap-lib.fsx=bootstrap-lib.fsx
+      - gateway.fsx=gateway.fsx
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    app.kubernetes.io/name: mirror-bootstrap
+    app.kubernetes.io/instance: mirrored-image

services/mirrored-image/k8s/bootstrap/quay-tests.fsx

@@ -0,0 +1,188 @@
+#load "gateway.fsx"
+
+open System
+open System.Net
+open System.Net.Http
+open System.Net.Http.Headers
+open System.Security.Cryptography
+open System.Text
+open System.Text.Json
+open System.Threading
+open System.Threading.Tasks
+open Bootstrap
+open Gateway
+
+let require condition message =
+    if not condition then
+        invalidOp message
+
+let base64Bytes (value: string) =
+    let value = value.Replace('-', '+').Replace('_', '/')
+    let padded = value.PadRight(value.Length + ((4 - value.Length % 4) % 4), '=')
+    Convert.FromBase64String padded
+
+type QuayHandler(rsa: RSA, wrongScopes: bool) =
+    inherit HttpMessageHandler()
+
+    let mutable assigned = false
+    let seen = ResizeArray<string>()
+    member _.Seen = seen
+
+    member private _.Respond(request: HttpRequestMessage) =
+        let path = request.RequestUri.AbsolutePath
+        seen.Add($"{request.Method} {path}")
+
+        if request.RequestUri.Host = "zitadel-api" then
+            require (request.Headers.Host = "zitadel") "canonical ZITADEL host changed"
+
+            require
+                (request.Headers.GetValues("X-Zitadel-Public-Host") |> Seq.exactlyOne = "zitadel")
+                "public ZITADEL host changed"
+
+        if path.StartsWith "/v2/" then
+            require (request.Headers.Authorization.Scheme = "Bearer") "ZITADEL system JWT was omitted"
+            let segments = request.Headers.Authorization.Parameter.Split('.')
+            require (segments.Length = 3) "JWT segment count changed"
+            use header = JsonDocument.Parse(base64Bytes segments[0])
+            use claims = JsonDocument.Parse(base64Bytes segments[1])
+            require (header.RootElement.GetProperty("alg").GetString() = "RS256") "JWT algorithm changed"
+            require (claims.RootElement.GetProperty("iss").GetString() = "login-client") "JWT issuer changed"
+            require (claims.RootElement.GetProperty("sub").GetString() = "login-client") "JWT subject changed"
+            require (claims.RootElement.GetProperty("aud").GetString() = "http://zitadel") "JWT audience changed"
+            let issued = claims.RootElement.GetProperty("iat").GetInt64()
+            require (claims.RootElement.GetProperty("exp").GetInt64() = issued + 300L) "JWT lifetime changed"
+
+            require
+                (rsa.VerifyData(
+                    Encoding.UTF8.GetBytes($"{segments[0]}.{segments[1]}"),
+                    base64Bytes segments[2],
+                    HashAlgorithmName.SHA256,
+                    RSASignaturePadding.Pkcs1
+                ))
+                "JWT signature is invalid"
+
+        let status, body, location =
+            match request.Method.Method, path with
+            | "GET", "/csrf_token" -> HttpStatusCode.OK, """{"csrf_token":"csrf"}""", None
+            | "POST", "/api/v1/externallogin/zitadel" ->
+                require (request.Headers.GetValues("X-CSRF-Token") |> Seq.exactlyOne = "csrf") "Quay CSRF token changed"
+                require (isNull request.Content.Headers.ContentType.CharSet) "Quay JSON media type included a charset"
+                require (request.Content.Headers.ContentLength.HasValue) "Quay JSON request omitted its content length"
+                HttpStatusCode.OK, """{"auth_url":"http://zitadel/oidc/start"}""", None
+            | "GET", "/oidc/start" -> HttpStatusCode.Found, "", Some "/ui/v2/login/login?authRequest=request-id"
+            | "POST", "/v2/sessions" -> HttpStatusCode.Created, """{"sessionId":"session-id"}""", None
+            | "PATCH", "/v2/sessions/session-id" -> HttpStatusCode.OK, """{"sessionToken":"session-token"}""", None
+            | "POST", "/v2/oidc/auth_requests/request-id" ->
+                HttpStatusCode.OK, """{"callbackUrl":"http://quay/callback"}""", None
+            | "GET", "/callback" -> HttpStatusCode.Found, "", Some "http://quay/"
+            | "PUT", "/api/v1/organization/candidates/team/mirrored-image" -> HttpStatusCode.OK, "", None
+            | "PUT", "/api/v1/organization/candidates/team/mirrored-image/members/candidates-bot" ->
+                HttpStatusCode.OK, "", None
+            | "GET", "/api/v1/organization/candidates/prototypes" -> HttpStatusCode.OK, """{"prototypes":[]}""", None
+            | "POST", "/api/v1/organization/candidates/prototypes" -> HttpStatusCode.OK, "", None
+            | "GET", "/api/v1/organization/candidates/applications" ->
+                HttpStatusCode.OK, """{"applications":[]}""", None
+            | "POST", "/api/v1/organization/candidates/applications" ->
+                HttpStatusCode.OK, """{"client_id":"client-id"}""", None
+            | "GET", "/api/v1/user/assignedauthorization" when not assigned ->
+                HttpStatusCode.OK, """{"authorizations":[]}""", None
+            | "POST", "/oauth/authorize/assignuser" ->
+                require (request.Headers.Authorization.Scheme = "Basic") "admin token did not use Quay Basic auth"
+                let query = Uri.UnescapeDataString request.RequestUri.Query
+                require (query.Contains "scope=repo:create+repo:write") "writer scopes changed"
+                require (not (query.Contains "repo:read")) "writer token redundantly requested read"
+                assigned <- true
+                HttpStatusCode.OK, "", None
+            | "GET", "/api/v1/user/assignedauthorization" ->
+                let scopes =
+                    if wrongScopes then
+                        """[{"scope":"repo:read"}]"""
+                    else
+                        """[{"scope":"repo:create"},{"scope":"repo:write"}]"""
+
+                HttpStatusCode.OK,
+                sprintf
+                    """{"authorizations":[{"application":{"clientId":"client-id"},"uuid":"assignment-id","redirectUri":"http://quay/oauth/localapp","responseType":"token","scopes":%s}]}"""
+                    scopes,
+                None
+            | "GET", "/oauth/authorize" -> HttpStatusCode.OK, """{"csrf_token_val":"approval-csrf"}""", None
+            | "POST", "/oauth/authorizeapp" ->
+                require
+                    (request.Headers.GetValues("X-CSRF-Token") |> Seq.exactlyOne = "approval-csrf")
+                    "approval CSRF token changed"
+
+                HttpStatusCode.Found, "", Some "http://quay/oauth/localapp#access_token=issued-token"
+            | _ -> invalidOp $"unexpected request {request.Method} {request.RequestUri}"
+
+        let response = new HttpResponseMessage(status)
+        response.Content <- new StringContent(body, Encoding.UTF8, "application/json")
+
+        location
+        |> Option.iter (fun value -> response.Headers.Location <- Uri(value, UriKind.RelativeOrAbsolute))
+
+        response
+
+    override this.Send(request: HttpRequestMessage, _: CancellationToken) = this.Respond request
+
+    override this.SendAsync(request: HttpRequestMessage, _: CancellationToken) = Task.FromResult(this.Respond request)
+
+let rsa = RSA.Create(2048)
+let handler = new QuayHandler(rsa, false)
+let adminClient = new HttpClient(handler, false)
+
+let sessionClient () = new HttpClient(handler, false)
+
+let quay =
+    makeQuayWith
+        adminClient
+        sessionClient
+        { URL = "http://quay"
+          ZitadelAPI = "http://zitadel-api"
+          ZitadelIssuer = "http://zitadel"
+          CaPath = ""
+          LoginKey = rsa.ExportRSAPrivateKeyPem() }
+
+let session = quay.Login "candidates-bot" "password" (quay.LoginAuth())
+let definition = apps |> List.find (fun app -> app.Name = "candidates-writer")
+quay.Access "admin-token" definition
+let clientID = quay.Application "admin-token" definition
+let app = { definition with Client = clientID }
+quay.Assign "admin-token" session app
+let token = quay.Issue session app
+require (token = "issued-token") "issued Quay token changed"
+require (handler.Seen.Count = 19) "bootstrap request sequence changed"
+adminClient.Dispose()
+
+let wrongHandler = new QuayHandler(rsa, true)
+let wrongAdmin = new HttpClient(wrongHandler, false)
+
+let wrongQuay =
+    makeQuayWith
+        wrongAdmin
+        (fun () -> new HttpClient(wrongHandler, false))
+        { URL = "http://quay"
+          ZitadelAPI = "http://zitadel-api"
+          ZitadelIssuer = "http://zitadel"
+          CaPath = ""
+          LoginKey = rsa.ExportRSAPrivateKeyPem() }
+
+let wrongSession =
+    wrongQuay.Login "candidates-bot" "password" (wrongQuay.LoginAuth())
+
+wrongQuay.Access "admin-token" definition
+
+let wrongApp =
+    { definition with
+        Client = wrongQuay.Application "admin-token" definition }
+
+let scopeRejected =
+    try
+        wrongQuay.Assign "admin-token" wrongSession wrongApp
+        false
+    with :? InvalidOperationException as error ->
+        error.Message = "assigned authorization has different scopes"
+
+require scopeRejected "wrong assigned scopes were accepted"
+wrongAdmin.Dispose()
+rsa.Dispose()
+printfn "Quay bootstrap tests passed"

services/mirrored-image/k8s/bootstrap/vault-tests.fsx

@@ -0,0 +1,124 @@
+#load "gateway.fsx"
+
+open System
+open System.IO
+open System.Net
+open System.Net.Http
+open System.Text
+open System.Text.Json
+open System.Threading
+open System.Threading.Tasks
+open Gateway
+
+let require condition message =
+    if not condition then
+        invalidOp message
+
+type VaultHandler() =
+    inherit HttpMessageHandler()
+
+    let seen = ResizeArray<string * string>()
+    member _.Seen = seen
+
+    member private _.Respond(request: HttpRequestMessage) =
+        let path = request.RequestUri.AbsolutePath
+
+        let body =
+            if path = "/v1/auth/kubernetes/login" then
+                use document = JsonDocument.Parse(request.Content.ReadAsStream())
+                require (document.RootElement.GetProperty("role").GetString() = "bootstrap") "Vault role changed"
+
+                require
+                    (document.RootElement.GetProperty("jwt").GetString() = "service-account-jwt")
+                    "Vault JWT changed"
+
+                """{"auth":{"client_token":"vault-token"}}"""
+            elif request.Method = HttpMethod.Get then
+                let token = request.Headers.GetValues("X-Vault-Token") |> Seq.exactlyOne
+                seen.Add(path, token)
+                """{"data":{"data":{"token":"stored"}}}"""
+            else
+                let token = request.Headers.GetValues("X-Vault-Token") |> Seq.exactlyOne
+                seen.Add(path, token)
+                use document = JsonDocument.Parse(request.Content.ReadAsStream())
+
+                require
+                    (document.RootElement.GetProperty("data").GetProperty("token").GetString() = "issued")
+                    "Vault write did not use a KVv2 envelope"
+
+                "{}"
+
+        let response = new HttpResponseMessage(HttpStatusCode.OK)
+        response.Content <- new StringContent(body, Encoding.UTF8, "application/json")
+
+        response
+
+    override this.Send(request: HttpRequestMessage, _: CancellationToken) = this.Respond request
+
+    override this.SendAsync(request: HttpRequestMessage, _: CancellationToken) =
+        let response = this.Respond request
+        Task.FromResult response
+
+type KubeHandler() =
+    inherit HttpMessageHandler()
+
+    member private _.Respond(request: HttpRequestMessage) =
+        require
+            (request.RequestUri.AbsolutePath = "/api/v1/namespaces/zitadel/secrets/zitadel-login-service-key")
+            "Kubernetes Secret path changed"
+
+        require (request.Headers.Authorization.Scheme = "Bearer") "Kubernetes bearer token was omitted"
+        require (request.Headers.Authorization.Parameter = "service-account-jwt") "Kubernetes bearer token changed"
+        let encoded = Convert.ToBase64String(Encoding.UTF8.GetBytes "private-key")
+        let response = new HttpResponseMessage(HttpStatusCode.OK)
+
+        response.Content <-
+            new StringContent($"{{\"data\":{{\"tls.key\":\"{encoded}\"}}}}", Encoding.UTF8, "application/json")
+
+        response
+
+    override this.Send(request: HttpRequestMessage, _: CancellationToken) = this.Respond request
+
+    override this.SendAsync(request: HttpRequestMessage, _: CancellationToken) = Task.FromResult(this.Respond request)
+
+let jwtPath = Path.GetTempFileName()
+
+try
+    File.WriteAllText(jwtPath, "service-account-jwt")
+    let handler = new VaultHandler()
+
+    let config =
+        { URL = "http://vault"
+          Role = "bootstrap"
+          JwtPath = jwtPath
+          CaPath = "" }
+
+    use vaultClient = new HttpClient(handler, true)
+    let store = makeStoreWith vaultClient config
+
+    require
+        (store.Read "secret/data/test" = Map.ofList (List.singleton ("token", "stored")))
+        "Vault secret was not decoded"
+
+    store.Write "secret/data/test" (Map.ofList (List.singleton ("token", "issued")))
+
+    require
+        (handler.Seen |> Seq.forall (fun (_, token) -> token = "vault-token"))
+        "Vault operation omitted its client token"
+
+    use kubeClient = new HttpClient(new KubeHandler(), true)
+
+    let key =
+        readKubeKeyWith
+            kubeClient
+            { URL = "http://kubernetes"
+              TokenPath = jwtPath
+              CaPath = ""
+              Namespace = "zitadel"
+              Secret = "zitadel-login-service-key" }
+
+    require (key = "private-key") "Kubernetes Secret key was not decoded"
+
+    printfn "bootstrap HTTP tests passed"
+finally
+    File.Delete jwtPath

services/mirrored-image/k8s/canary.yaml

@@ -0,0 +1,11 @@
+apiVersion: mirror.quorate.dev/v1alpha1
+kind: MirroredImage
+metadata:
+  name: nginx
+  namespace: mirrored-image
+spec:
+  upstream: cgr.dev/chainguard/nginx
+  policy:
+    rejectAtOrAbove: Critical
+  digests:
+    - digest: sha256:c3e810caa72b130c880f750986c2bab5ea7ca7ad03365ea562718be77d7d7c2e

services/mirrored-image/k8s/controller.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mirrored-image
+  namespace: mirrored-image
+  labels:
+    app.kubernetes.io/name: mirrored-image
+    app.kubernetes.io/instance: mirrored-image
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mirrored-image
+      app.kubernetes.io/instance: mirrored-image
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: mirrored-image
+        app.kubernetes.io/instance: mirrored-image
+    spec:
+      serviceAccountName: mirrored-image
+      automountServiceAccountToken: true
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: controller
+          image: mirrored-image-controller
+          args: [run]
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef: {fieldPath: metadata.namespace}
+            - {name: REGISTRY, value: registry.quorate.internal}
+            - {name: QUAY_API, value: https://registry.quorate.internal}
+          resources:
+            requests: {cpu: 20m, memory: 32Mi}
+            limits: {cpu: 200m, memory: 128Mi}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: [ALL]}
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - {name: candidates-reader, mountPath: /candidates-reader, readOnly: true}
+            - {name: mirror-reader, mountPath: /mirror-reader, readOnly: true}
+            - {name: registry-ca, mountPath: /registry-ca, readOnly: true}
+      volumes:
+        - {name: candidates-reader, secret: {secretName: candidates-reader, defaultMode: 0440}}
+        - {name: mirror-reader, secret: {secretName: mirror-reader, defaultMode: 0440}}
+        - {name: registry-ca, secret: {secretName: mirror-ca, defaultMode: 0440}}

services/mirrored-image/k8s/crd.yaml

@@ -0,0 +1,142 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: mirroredimages.mirror.quorate.dev
+spec:
+  group: mirror.quorate.dev
+  names:
+    kind: MirroredImage
+    listKind: MirroredImageList
+    plural: mirroredimages
+    singular: mirroredimage
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          required: [spec]
+          properties:
+            apiVersion:
+              type: string
+            kind:
+              type: string
+            metadata:
+              type: object
+            spec:
+              type: object
+              required: [upstream, policy, digests]
+              properties:
+                upstream:
+                  type: string
+                  minLength: 1
+                policy:
+                  type: object
+                  required: [rejectAtOrAbove]
+                  properties:
+                    rejectAtOrAbove:
+                      type: string
+                      enum: [Unknown, Negligible, Low, Medium, High, Critical]
+                digests:
+                  type: array
+                  x-kubernetes-list-type: map
+                  x-kubernetes-list-map-keys: [digest]
+                  items:
+                    type: object
+                    required: [digest]
+                    properties:
+                      digest:
+                        type: string
+                        pattern: ^sha256:[0-9a-f]{64}$
+                      tag:
+                        type: string
+                        maxLength: 128
+                        pattern: ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$
+            status:
+              type: object
+              properties:
+                observedGeneration:
+                  type: integer
+                  format: int64
+                message:
+                  type: string
+                lastHandledRetry:
+                  type: string
+                  maxLength: 63
+                  pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                digests:
+                  type: array
+                  x-kubernetes-list-type: map
+                  x-kubernetes-list-map-keys: [digest]
+                  items:
+                    type: object
+                    required: [digest]
+                    properties:
+                      digest:
+                        type: string
+                        pattern: ^sha256:[0-9a-f]{64}$
+                      tag:
+                        type: string
+                        maxLength: 128
+                        pattern: ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$
+                      prepare:
+                        type: object
+                        required: [workflowName, phase, generation]
+                        properties:
+                          workflowName:
+                            type: string
+                            maxLength: 63
+                            pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                          retryOf:
+                            type: string
+                            maxLength: 63
+                            pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                          phase:
+                            type: string
+                            enum: [Pending, Running, Succeeded, Failed, Error]
+                          generation:
+                            type: integer
+                            format: int64
+                      scan:
+                        type: string
+                        enum: [Pending, Accepted, Rejected]
+                      promotion:
+                        type: object
+                        required: [workflowName, phase, generation]
+                        properties:
+                          workflowName:
+                            type: string
+                            maxLength: 63
+                            pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                          retryOf:
+                            type: string
+                            maxLength: 63
+                            pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                          phase:
+                            type: string
+                            enum: [Pending, Running, Succeeded, Failed, Error]
+                          generation:
+                            type: integer
+                            format: int64
+                      cleanup:
+                        type: object
+                        required: [workflowName, phase, generation]
+                        properties:
+                          workflowName:
+                            type: string
+                            maxLength: 63
+                            pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                          retryOf:
+                            type: string
+                            maxLength: 63
+                            pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
+                          phase:
+                            type: string
+                            enum: [Pending, Running, Succeeded, Failed, Error]
+                          generation:
+                            type: integer
+                            format: int64

services/mirrored-image/k8s/delivery/controller/kustomization.yaml

@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namePrefix: mirrored-image-
+resources:
+  - ../../../../project-delivery/k8s/workflows/go-oci/base
+  - project.yaml
+configMapGenerator:
+  - name: project
+    namespace: project-delivery
+    files:
+      - projects.fsx=projects.fsx
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    app.kubernetes.io/name: project-delivery
+    app.kubernetes.io/instance: mirrored-image
+patches:
+  - path: workflow-patch.yaml
+    target:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name: build
+replacements:
+  - source:
+      group: argoproj.io
+      version: v1alpha1
+      kind: WorkflowTemplate
+      name: mirrored-image-build
+      fieldPath: metadata.name
+    targets:
+      - select:
+          group: delivery.quorate.dev
+          version: v1alpha1
+          kind: ProjectDelivery
+          name: mirrored-image-delivery
+        fieldPaths:
+          - spec.workflowTemplateRef.name

services/mirrored-image/k8s/delivery/controller/project.yaml

@@ -0,0 +1,18 @@
+apiVersion: delivery.quorate.dev/v1alpha1
+kind: ProjectDelivery
+metadata:
+  name: delivery
+  namespace: project-delivery
+spec:
+  source:
+    repository: https://forge.quorate.internal/quorate/cluster
+    continuous:
+      ref: refs/heads/master
+      paths:
+        - services/mirrored-image/controller
+        - services/mirrored-image/k8s/delivery
+  targets:
+    - kind: QuayOCI
+      repository: quorate/mirrored-image-controller
+  workflowTemplateRef:
+    name: build

services/mirrored-image/k8s/delivery/controller/projects.fsx

@@ -0,0 +1,73 @@
+#load "contract.fsx"
+
+open System.IO
+open Contract
+
+type Binary =
+    { Module: string
+      Package: string
+      Destination: string }
+
+type Project =
+    { Name: string
+      Workflow: string
+      Repository: string
+      Modules: string list
+      Binaries: Binary list
+      FileCaps: Map<string, FileCap list>
+      Entrypoint: string list
+      WorkingDir: string }
+
+type ToolchainProfile =
+    { Name: string
+      Packages: string list
+      TestApplets: string list
+      CachePaths: string list }
+
+let projects =
+    [ { Name = "mirrored-image"
+        Workflow = "mirrored-image-build"
+        Repository = "quorate/mirrored-image-controller"
+        Modules = [ "services/mirrored-image/controller" ]
+        Binaries =
+          [ { Module = "services/mirrored-image/controller"
+              Package = "."
+              Destination = "mirrored-image" } ]
+        FileCaps = Map.empty
+        Entrypoint = [ "/mirrored-image" ]
+        WorkingDir = "/" } ]
+
+let selectProject (targets: Target array) =
+    require (targets.Length = 1 && targets[0].Kind = "QuayOCI") "invalid delivery target"
+
+    projects
+    |> List.tryFind (fun project -> project.Repository = targets[0].Repository)
+    |> Option.defaultWith (fun () -> fail "unsupported delivery target")
+
+let moduleRoot sourceRoot (path: string) = Path.Combine(sourceRoot, path)
+
+let validateProject sourceRoot project =
+    project.Modules
+    |> List.iter (fun path ->
+        let root = moduleRoot sourceRoot path
+        require (Directory.Exists root) $"missing module: {path}"
+        require (File.Exists(Path.Combine(root, "go.mod"))) $"missing go.mod: {path}")
+
+let inputPaths sourceRoot project =
+    project.Modules
+    |> List.collect (fun path ->
+        [ Path.Combine(path, "go.mod"); Path.Combine(path, "go.sum") ]
+        |> List.filter (fun item -> File.Exists(Path.Combine(sourceRoot, item))))
+    |> List.distinct
+
+let toolchainProfile =
+    { Name = "pure-go"
+      Packages = []
+      TestApplets = []
+      CachePaths = [] }
+
+let toolchainProfileName = toolchainProfile.Name
+let runtimeAssets (_: Project) : (string * string * string) list = []
+
+let toolchainCachePaths (_: Project) =
+    [ "usr/lib/go"; "image-base"; "modules"; "gopath" ]

services/mirrored-image/k8s/delivery/controller/workflow-patch.yaml

@@ -0,0 +1,31 @@
+- op: replace
+  path: /spec/templates/1/volumes/0
+  value: &orchestration
+    name: orchestration
+    projected:
+      defaultMode: 0444
+      sources:
+        - configMap:
+            name: go-oci-orch
+        - configMap:
+            name: mirrored-image-project
+            items:
+              - {key: projects.fsx, path: projects.fsx}
+- op: replace
+  path: /spec/templates/3/volumes/0
+  value: *orchestration
+- op: replace
+  path: /spec/templates/4/volumes/0
+  value: *orchestration
+- op: replace
+  path: /spec/templates/5/volumes/0
+  value: *orchestration
+- op: replace
+  path: /spec/templates/6/volumes/0
+  value: *orchestration
+- op: replace
+  path: /spec/templates/7/volumes/0
+  value: *orchestration
+- op: replace
+  path: /spec/templates/8/volumes/0
+  value: *orchestration

services/mirrored-image/k8s/delivery/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - controller

services/mirrored-image/k8s/kustomizeconfig.yaml

@@ -0,0 +1,3 @@
+images:
+  - path: spec/templates/container/image
+    kind: WorkflowTemplate

services/mirrored-image/k8s/namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mirrored-image
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted