ce7cb702218693d08ea6d83464d2457c9250027c

services/mirrored-image/actions/cleanup.fsx

@@ -0,0 +1,13 @@
+#load "contract.fsx"
+#load "registry.fsx"
+
+open Contract
+open Registry
+
+let registry = env "REGISTRY"
+let repository = env "REPOSITORY"
+let digest = env "DIGEST"
+let tlsVerify = env "TLS_VERIFY" |> System.Boolean.Parse
+let certPath = "/registry-ca/ca.crt"
+
+deleteManifest registry repository digest "$oauthtoken" (secret (env "AUTH_TOKEN_PATH")) certPath tlsVerify

services/mirrored-image/actions/contract.fsx

@@ -0,0 +1,22 @@
+module Contract
+
+open System
+open System.IO
+
+let fail message =
+    raise (InvalidOperationException message)
+
+let require condition message =
+    if not condition then
+        fail message
+
+let env name =
+    let value = Environment.GetEnvironmentVariable name
+    require (not (String.IsNullOrWhiteSpace value)) $"{name} is unavailable"
+    value
+
+let secret path =
+    require (File.Exists path) $"secret is unavailable: {Path.GetFileName path}"
+    let value = File.ReadAllText(path).Trim()
+    require (not (String.IsNullOrWhiteSpace value)) $"secret is empty: {Path.GetFileName path}"
+    value

services/mirrored-image/actions/prepare.fsx

@@ -0,0 +1,18 @@
+#load "contract.fsx"
+#load "registry.fsx"
+
+open Contract
+open Registry
+
+let registry = env "REGISTRY"
+let api = env "QUAY_API"
+let source = env "SOURCE_REPOSITORY"
+let target = env "TARGET_REPOSITORY"
+let digest = env "DIGEST"
+let tag = System.Environment.GetEnvironmentVariable "TAG"
+let tlsVerify = env "TLS_VERIFY" |> System.Boolean.Parse
+let certPath = "/registry-ca/ca.crt"
+
+let writer = secret "/candidate-writer/token"
+ensureRepo api target writer certPath tlsVerify
+copy source registry api target digest tag None (Some("$oauthtoken", writer)) certPath tlsVerify

services/mirrored-image/actions/promote.fsx

@@ -0,0 +1,30 @@
+#load "contract.fsx"
+#load "registry.fsx"
+
+open Contract
+open Registry
+
+let registry = env "REGISTRY"
+let api = env "QUAY_API"
+let source = env "SOURCE_REPOSITORY"
+let target = env "TARGET_REPOSITORY"
+let digest = env "DIGEST"
+let tag = System.Environment.GetEnvironmentVariable "TAG"
+let tlsVerify = env "TLS_VERIFY" |> System.Boolean.Parse
+let certPath = "/registry-ca/ca.crt"
+
+let reader = secret "/candidate-reader/token"
+let writer = secret "/mirror-writer/token"
+ensureRepo api target writer certPath tlsVerify
+
+copy
+    $"{registry}/{source}"
+    registry
+    api
+    target
+    digest
+    tag
+    (Some("$oauthtoken", reader))
+    (Some("$oauthtoken", writer))
+    certPath
+    tlsVerify

services/mirrored-image/actions/registry-tests.fsx

@@ -0,0 +1,290 @@
+#load "registry.fsx"
+
+open System
+open System.Collections.Concurrent
+open System.Collections.Generic
+open System.IO
+open System.Net
+open System.Net.Sockets
+open System.Security.Cryptography
+open System.Text
+open System.Text.Json
+open System.Threading.Tasks
+open Registry
+
+let require value message =
+    if not value then
+        invalidOp message
+
+let hash (content: byte array) =
+    "sha256:"
+    + (SHA256.HashData content |> Convert.ToHexString |> _.ToLowerInvariant())
+
+let media = "application/vnd.oci.image.manifest.v1+json"
+let indexMedia = "application/vnd.oci.image.index.v1+json"
+let configDigest = "sha256:" + String.replicate 64 "a"
+
+let manifest architecture =
+    Encoding.UTF8.GetBytes(
+        $"""{{"schemaVersion":2,"mediaType":"{media}","config":{{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"{configDigest}","size":2}},"layers":[],"annotations":{{"architecture":"{architecture}"}}}}"""
+    )
+
+let check shape alias fault =
+    let children = [| manifest "amd64"; manifest "arm64" |]
+
+    let root =
+        if shape = "single" then
+            children[0]
+        else
+            JsonSerializer.SerializeToUtf8Bytes
+                {| schemaVersion = 2
+                   mediaType = indexMedia
+                   manifests =
+                    children
+                    |> Array.map (fun child ->
+                        {| mediaType = media
+                           digest = hash child
+                           size = child.Length |}) |}
+
+    let rootDigest = hash root
+    let retained = retentionTag rootDigest
+    let otherDigest = "sha256:" + String.replicate 64 "b"
+
+    let tag =
+        match alias with
+        | "own" -> retained
+        | "collision" -> retentionTag otherDigest
+        | value -> value
+
+    let contents =
+        (root :: Array.toList children)
+        |> List.map (fun value -> hash value, value)
+        |> Map.ofList
+
+    use portProbe = new TcpListener(IPAddress.Loopback, 0)
+    portProbe.Start()
+    let port = (portProbe.LocalEndpoint :?> IPEndPoint).Port
+    portProbe.Stop()
+    let host = $"127.0.0.1:{port}"
+    use listener = new HttpListener()
+    listener.Prefixes.Add($"http://{host}/")
+    listener.Start()
+    let writes = ConcurrentQueue<string>()
+    let requests = ConcurrentQueue<string>()
+    let reads = ConcurrentQueue<string>()
+    let tags = ConcurrentDictionary<string, string * bool>()
+    let faults = ConcurrentDictionary<string, bool>()
+    let errors = ConcurrentQueue<exn>()
+    faults[fault] <- true
+
+    if fault = "conflict" then
+        tags[retained] <- (otherDigest, true)
+
+    let serve =
+        Task.Run(fun () ->
+            try
+                while listener.IsListening do
+                    let context = listener.GetContext()
+
+                    try
+                        let request = context.Request
+                        let response = context.Response
+                        let reference = request.Url.Segments |> Array.last |> Uri.UnescapeDataString
+                        requests.Enqueue request.RawUrl
+
+                        require
+                            (String.IsNullOrEmpty request.Headers["Quay-Manifest-Retention"])
+                            "legacy retention header"
+
+                        if request.Url.AbsolutePath.StartsWith("/api/", StringComparison.Ordinal) then
+                            require
+                                (request.Headers["Authorization"] = "Bearer synthetic-writer")
+                                "wrong API authority"
+
+                            match request.HttpMethod with
+                            | "GET" ->
+                                if faults.ContainsKey "metadata" then
+                                    response.StatusCode <- 503
+                                else
+                                    let name = request.QueryString["specificTag"]
+                                    require (request.QueryString["onlyActiveTags"] = "true") "inactive tag lookup"
+                                    let values = ResizeArray<Dictionary<string, obj>>()
+
+                                    match tags.TryGetValue name with
+                                    | true, (digest, permanent) ->
+                                        let value = Dictionary<string, obj>()
+                                        value["name"] <- name
+                                        value["manifest_digest"] <- digest
+
+                                        if not permanent then
+                                            value["end_ts"] <- 2147483647
+
+                                        values.Add value
+                                    | _ -> ()
+
+                                    response.ContentType <- "application/json"
+
+                                    response.OutputStream.Write(
+                                        JsonSerializer.SerializeToUtf8Bytes
+                                            {| tags = values
+                                               has_additional = false |}
+                                    )
+                            | "PUT" ->
+                                use payload = JsonDocument.Parse request.InputStream
+
+                                require
+                                    (payload.RootElement.GetProperty("expiration").ValueKind = JsonValueKind.Null)
+                                    "expiry not cleared"
+
+                                if faults.ContainsKey "expiry" || (faults.ContainsKey "alias" && reference = tag) then
+                                    response.StatusCode <- 503
+                                else
+                                    response.StatusCode <- 201
+
+                                    if not (faults.ContainsKey "unconfirmed") then
+                                        let actual, _ = tags[reference]
+                                        tags[reference] <- (actual, true)
+                            | method -> invalidOp $"unexpected API method {method}"
+                        else
+                            match request.HttpMethod with
+                            | "GET" ->
+                                reads.Enqueue reference
+                                let content = contents[reference]
+
+                                response.ContentType <-
+                                    if shape = "index" && reference = rootDigest then
+                                        indexMedia
+                                    else
+                                        media
+
+                                response.Headers.Add("Docker-Content-Digest", reference)
+                                response.OutputStream.Write content
+                            | "HEAD" -> response.StatusCode <- 200
+                            | "PUT" ->
+                                use buffer = new MemoryStream()
+                                request.InputStream.CopyTo buffer
+                                let actual = hash (buffer.ToArray())
+
+                                require
+                                    (reference = retained
+                                     || reference = tag
+                                     || (reference = actual && actual <> rootDigest))
+                                    "unexpected tag alias"
+
+                                writes.Enqueue reference
+
+                                if reference <> actual then
+                                    tags[reference] <- (actual, false)
+
+                                response.StatusCode <- 201
+
+                                response.Headers.Add(
+                                    "Docker-Content-Digest",
+                                    if faults.ContainsKey "digest" then otherDigest else actual
+                                )
+                            | "DELETE" ->
+                                require (reference = rootDigest) "cleanup changed digest"
+
+                                let owned =
+                                    tags
+                                    |> Seq.filter (fun item -> fst item.Value = rootDigest)
+                                    |> Seq.map _.Key
+                                    |> Seq.toArray
+
+                                for name in owned do
+                                    tags.TryRemove name |> ignore
+
+                                response.StatusCode <- if owned.Length > 0 then 202 else 404
+                            | method -> invalidOp $"unexpected method {method}"
+
+                        response.Close()
+                    with error ->
+                        errors.Enqueue error
+                        context.Response.Abort()
+            with
+            | :? HttpListenerException -> ()
+            | :? ObjectDisposedException -> ())
+
+    let copyImage () =
+        copy
+            $"{host}/upstream"
+            host
+            $"http://{host}"
+            "mirror/test"
+            rootDigest
+            tag
+            None
+            (Some("$oauthtoken", "synthetic-writer"))
+            ""
+            false
+
+    try
+        if alias = "collision" then
+            let mutable rejected = false
+
+            try
+                copyImage ()
+            with :? InvalidOperationException ->
+                rejected <- true
+
+            require rejected "RetentionTags.AliasAllowed"
+            require requests.IsEmpty "alias collision performed registry I/O"
+        else
+            if fault <> "none" then
+                let mutable rejected = false
+
+                try
+                    copyImage ()
+                with :? InvalidOperationException ->
+                    rejected <- true
+
+                require rejected $"fault {fault} was accepted"
+
+                if fault = "metadata" || fault = "conflict" then
+                    require writes.IsEmpty "unsafe write after failed observation"
+                elif fault = "alias" then
+                    require (tags[retained] = (rootDigest, true)) "alias failure damaged retention"
+                elif fault <> "digest" || shape = "single" then
+                    require (tags[retained] = (rootDigest, false)) "partial publication was not preserved"
+
+                faults.Clear()
+
+                if fault = "conflict" then
+                    tags.TryRemove retained |> ignore
+
+            copyImage ()
+            let confirmedWrites = writes.Count
+            let confirmedReads = reads.Count
+
+            for _ in 1..3 do
+                copyImage ()
+
+            require (writes.Count = confirmedWrites) "TagExpiry.Write: permanent tag was rewritten"
+            require (reads.Count = confirmedReads) "retry fetched already retained content"
+
+            let expected =
+                if tag = "" || tag = retained then
+                    Set.singleton retained
+                else
+                    Set.ofList [ retained; tag ]
+
+            require (Set.ofSeq tags.Keys = expected) "RetryPreservesTagIdentity"
+            require (tags.Values |> Seq.forall (fun value -> value = (rootDigest, true))) "ConfirmedRetention"
+            deleteManifest host "mirror/test" rootDigest "synthetic" "synthetic" "" false
+            deleteManifest host "mirror/test" rootDigest "synthetic" "synthetic" "" false
+            require tags.IsEmpty "cleanup retained a public tag"
+
+        require errors.IsEmpty "fixture failed"
+    finally
+        listener.Stop()
+        serve.GetAwaiter().GetResult()
+
+for shape in [ "single"; "index" ] do
+    for alias in [ ""; "stable"; "own" ] do
+        for fault in [ "none"; "metadata"; "conflict"; "expiry"; "unconfirmed"; "digest" ] do
+            check shape alias fault
+
+    check shape "stable" "alias"
+    check shape "collision" "none"
+
+printfn "MirroredImage registry action tests passed: 40 cases"

services/mirrored-image/actions/registry.fsx

@@ -0,0 +1,602 @@
+#load "contract.fsx"
+
+open System
+open System.Collections.Generic
+open System.IO
+open System.Net.Http
+open System.Net.Http.Headers
+open System.Security.Cryptography
+open System.Security.Cryptography.X509Certificates
+open System.Text
+open System.Text.Json
+open System.Text.RegularExpressions
+open Contract
+
+type private Descriptor =
+    { Digest: string
+      Size: int64
+      MediaType: string }
+
+type private Manifest =
+    { Digest: string
+      MediaType: string
+      Content: byte array }
+
+type private Pending =
+    { Digest: string
+      Size: int64 option
+      MediaType: string option
+      Finish: bool }
+
+let private httpHandler certPath tlsVerify =
+    let handler = new SocketsHttpHandler()
+
+    if tlsVerify then
+        let root = X509Certificate2.CreateFromPem(File.ReadAllText certPath)
+
+        handler.SslOptions.CertificateChainPolicy <-
+            X509ChainPolicy(DisableCertificateDownloads = true, RevocationMode = X509RevocationMode.NoCheck)
+
+        handler.SslOptions.RemoteCertificateValidationCallback <-
+            fun _ certificate peerChain errors ->
+                if errors = Net.Security.SslPolicyErrors.None then
+                    true
+                elif
+                    isNull certificate
+                    || errors <> Net.Security.SslPolicyErrors.RemoteCertificateChainErrors
+                then
+                    false
+                else
+                    use chain = new X509Chain()
+                    chain.ChainPolicy.TrustMode <- X509ChainTrustMode.CustomRootTrust
+                    chain.ChainPolicy.CustomTrustStore.Add root |> ignore
+                    chain.ChainPolicy.RevocationMode <- X509RevocationMode.NoCheck
+                    chain.ChainPolicy.DisableCertificateDownloads <- true
+
+                    if not (isNull peerChain) then
+                        for element in peerChain.ChainElements do
+                            chain.ChainPolicy.ExtraStore.Add element.Certificate |> ignore
+
+                    match certificate with
+                    | :? X509Certificate2 as leaf -> chain.Build leaf
+                    | _ -> false
+
+    handler
+
+let private challengeValues (header: AuthenticationHeaderValue) =
+    Regex.Matches(header.Parameter, "([A-Za-z]+)=\"([^\"]*)\"")
+    |> Seq.cast<Match>
+    |> Seq.map (fun item -> item.Groups[1].Value.ToLowerInvariant(), item.Groups[2].Value)
+    |> Map.ofSeq
+
+let private bearerToken (client: HttpClient) (response: HttpResponseMessage) (auth: (string * string) option) =
+    let challenge =
+        response.Headers.WwwAuthenticate
+        |> Seq.tryFind (fun item -> String.Equals(item.Scheme, "Bearer", StringComparison.OrdinalIgnoreCase))
+        |> Option.defaultWith (fun () -> fail "registry did not return a Bearer challenge")
+
+    let values = challengeValues challenge
+
+    let value name =
+        values
+        |> Map.tryFind name
+        |> Option.defaultWith (fun () -> fail $"registry challenge omitted {name}")
+
+    let realm = value "realm"
+    let separator = if realm.Contains '?' then "&" else "?"
+    let service = Uri.EscapeDataString(value "service")
+    let scope = Uri.EscapeDataString(value "scope")
+
+    let account =
+        auth
+        |> Option.map (fun (username, _) -> $"&account={Uri.EscapeDataString username}")
+        |> Option.defaultValue ""
+
+    let endpoint = $"{realm}{separator}service={service}&scope={scope}{account}"
+
+    use request = new HttpRequestMessage(HttpMethod.Get, endpoint)
+
+    auth
+    |> Option.iter (fun (username, token) ->
+        request.Headers.Authorization <-
+            AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{username}:{token}"))))
+
+    use tokenResponse = client.Send request
+    require tokenResponse.IsSuccessStatusCode $"registry authentication failed with HTTP {int tokenResponse.StatusCode}"
+    use document = JsonDocument.Parse(tokenResponse.Content.ReadAsStream())
+    let root = document.RootElement
+    let mutable value = Unchecked.defaultof<JsonElement>
+
+    require
+        ((root.TryGetProperty("token", &value)
+          || root.TryGetProperty("access_token", &value))
+         && value.ValueKind = JsonValueKind.String)
+        "registry authentication returned no token"
+
+    value.GetString()
+    |> Option.ofObj
+    |> Option.defaultWith (fun () -> fail "registry authentication returned an empty token")
+
+let private regSend (client: HttpClient) auth make =
+    let send value =
+        use request = make value
+        client.Send(request, HttpCompletionOption.ResponseHeadersRead)
+
+    let first = send None
+
+    if first.StatusCode = Net.HttpStatusCode.Unauthorized then
+        let access = bearerToken client first auth
+        first.Dispose()
+        send (Some(AuthenticationHeaderValue("Bearer", access)))
+    else
+        first
+
+let private regURL (root: Uri) repository path = Uri(root, $"/v2/{repository}/{path}")
+
+let private req
+    (method: HttpMethod)
+    (endpoint: Uri)
+    (accept: string list option)
+    (authorization: AuthenticationHeaderValue option)
+    =
+    let request = new HttpRequestMessage(method, endpoint)
+
+    authorization
+    |> Option.iter (fun value -> request.Headers.Authorization <- value)
+
+    accept
+    |> Option.iter (fun values ->
+        for value in values do
+            request.Headers.Accept.Add(MediaTypeWithQualityHeaderValue value))
+
+    request
+
+let private mediaTypes =
+    [ "application/vnd.oci.image.index.v1+json"
+      "application/vnd.oci.image.manifest.v1+json"
+      "application/vnd.docker.distribution.manifest.list.v2+json"
+      "application/vnd.docker.distribution.manifest.v2+json" ]
+
+let private indexTypes = Set.ofList [ mediaTypes[0]; mediaTypes[2] ]
+
+let private digestBytes (content: byte array) =
+    let hash = SHA256.HashData content |> Convert.ToHexString
+    $"sha256:{hash.ToLowerInvariant()}"
+
+let private digestFile path =
+    use content = File.OpenRead path
+    let hash = SHA256.HashData content |> Convert.ToHexString
+    $"sha256:{hash.ToLowerInvariant()}"
+
+let private prop (name: string) (value: JsonElement) =
+    let mutable result = Unchecked.defaultof<JsonElement>
+    require (value.TryGetProperty(name, &result)) $"OCI document omitted {name}"
+    result
+
+let private strProp (name: string) value =
+    let result = prop name value
+    require (result.ValueKind = JsonValueKind.String) $"OCI {name} is not a string"
+
+    result.GetString()
+    |> Option.ofObj
+    |> Option.defaultWith (fun () -> fail $"OCI {name} is empty")
+
+let private desc (value: JsonElement) =
+    let size = prop "size" value
+    require (size.ValueKind = JsonValueKind.Number) "OCI descriptor size is not a number"
+
+    { Digest = strProp "digest" value
+      Size = size.GetInt64()
+      MediaType = strProp "mediaType" value }
+
+let private regClient certPath tlsVerify =
+    let client = new HttpClient(httpHandler certPath tlsVerify, true)
+    client.Timeout <- TimeSpan.FromMinutes 15.
+    client
+
+let private getManifest (client: HttpClient) root repository auth digest =
+    use response =
+        regSend client auth (fun authorization ->
+            req HttpMethod.Get (regURL root repository $"manifests/{digest}") (Some mediaTypes) authorization)
+
+    require response.IsSuccessStatusCode $"manifest read failed with HTTP {int response.StatusCode}"
+    let content = response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult()
+    require (digestBytes content = digest) $"manifest content does not match {digest}"
+
+    let mediaType =
+        Option.ofObj response.Content.Headers.ContentType
+        |> Option.bind (fun value -> Option.ofObj value.MediaType)
+        |> Option.defaultWith (fun () -> fail "manifest response omitted media type")
+
+    require (List.contains mediaType mediaTypes) $"unsupported manifest media type {mediaType}"
+    content, mediaType
+
+let private graph client root repository auth rootDigest =
+    let blobs = Dictionary<string, Descriptor>()
+    let manifests = ResizeArray<Manifest>()
+    let documents = Dictionary<string, Manifest>()
+    let complete = HashSet<string>()
+    let active = HashSet<string>()
+    let pending = Stack<Pending>()
+
+    let addBlob value =
+        let item = desc value
+        require (item.Size >= 0L) $"OCI blob {item.Digest} has an invalid size"
+
+        match blobs.TryGetValue item.Digest with
+        | true, found -> require (found.Size = item.Size) $"OCI blob {item.Digest} has conflicting sizes"
+        | _ -> blobs.Add(item.Digest, item)
+
+    pending.Push
+        { Digest = rootDigest
+          Size = None
+          MediaType = None
+          Finish = false }
+
+    while pending.Count > 0 do
+        let item = pending.Pop()
+
+        if item.Finish then
+            active.Remove item.Digest |> ignore
+            complete.Add item.Digest |> ignore
+            manifests.Add documents[item.Digest]
+        elif not (complete.Contains item.Digest) then
+            require (active.Add item.Digest) $"OCI manifest graph contains a cycle at {item.Digest}"
+            let content, mediaType = getManifest client root repository auth item.Digest
+
+            item.Size
+            |> Option.iter (fun size ->
+                require (int64 content.LongLength = size) $"OCI manifest {item.Digest} has an invalid size")
+
+            item.MediaType
+            |> Option.iter (fun value -> require (mediaType = value) $"OCI manifest {item.Digest} changed media type")
+
+            documents.Add(
+                item.Digest,
+                { Digest = item.Digest
+                  MediaType = mediaType
+                  Content = content }
+            )
+
+            pending.Push { item with Finish = true }
+            use document = JsonDocument.Parse content
+            let root = document.RootElement
+            require ((prop "schemaVersion" root).GetInt32() = 2) "OCI schema version is not 2"
+
+            if indexTypes.Contains mediaType then
+                let children = prop "manifests" root
+                require (children.ValueKind = JsonValueKind.Array) $"OCI index {item.Digest} has no manifests"
+
+                children.EnumerateArray()
+                |> Seq.map desc
+                |> Seq.rev
+                |> Seq.iter (fun child ->
+                    pending.Push
+                        { Digest = child.Digest
+                          Size = Some child.Size
+                          MediaType = Some child.MediaType
+                          Finish = false })
+            else
+                addBlob (prop "config" root)
+                let layers = prop "layers" root
+                require (layers.ValueKind = JsonValueKind.Array) $"OCI manifest {item.Digest} has no layers"
+                layers.EnumerateArray() |> Seq.iter addBlob
+
+    blobs.Values |> Seq.toArray, manifests |> Seq.toArray
+
+let private blobPath (root: string) (digest: string) =
+    Path.Combine(root, digest.Substring("sha256:".Length))
+
+let private getBlob client root repository auth temp (item: Descriptor) =
+    let path = blobPath temp item.Digest
+
+    use response =
+        regSend client auth (fun authorization ->
+            req HttpMethod.Get (regURL root repository $"blobs/{item.Digest}") None authorization)
+
+    require response.IsSuccessStatusCode $"blob read failed with HTTP {int response.StatusCode}"
+    use source = response.Content.ReadAsStream()
+
+    let length =
+        use target =
+            new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None)
+
+        source.CopyTo target
+        target.Flush()
+        target.Length
+
+    require (length = item.Size) $"OCI blob {item.Digest} has an invalid size"
+    require (digestFile path = item.Digest) $"OCI blob content does not match {item.Digest}"
+    path
+
+let private blobExists client root repository auth digest =
+    use response =
+        regSend client auth (fun authorization ->
+            req HttpMethod.Head (regURL root repository $"blobs/{digest}") None authorization)
+
+    match response.StatusCode with
+    | Net.HttpStatusCode.OK -> true
+    | Net.HttpStatusCode.NotFound -> false
+    | status -> fail $"blob lookup failed with HTTP {int status}"
+
+let private addDigest (location: Uri) (digest: string) =
+    let builder = UriBuilder location
+    let pair = $"digest={Uri.EscapeDataString digest}"
+
+    builder.Query <-
+        if String.IsNullOrEmpty builder.Query then
+            pair
+        else
+            $"{builder.Query.TrimStart('?')}&{pair}"
+
+    builder.Uri
+
+let private putBlob client root repository auth temp (item: Descriptor) sourceClient sourceRoot sourceRepo sourceAuth =
+    if not (blobExists client root repository auth item.Digest) then
+        let path = getBlob sourceClient sourceRoot sourceRepo sourceAuth temp item
+
+        use started =
+            regSend client auth (fun authorization ->
+                req HttpMethod.Post (regURL root repository "blobs/uploads/") None authorization)
+
+        require
+            (started.StatusCode = Net.HttpStatusCode.Accepted)
+            $"blob upload start failed with HTTP {int started.StatusCode}"
+
+        require (not (isNull started.Headers.Location)) "blob upload omitted location"
+        let baseURL = started.RequestMessage.RequestUri
+
+        let location =
+            if started.Headers.Location.IsAbsoluteUri then
+                started.Headers.Location
+            else
+                Uri(baseURL, started.Headers.Location)
+
+        require
+            (location.Scheme = baseURL.Scheme
+             && location.IdnHost = baseURL.IdnHost
+             && location.Port = baseURL.Port)
+            "blob upload location changed authority"
+
+        let endpoint = addDigest location item.Digest
+
+        use completed =
+            regSend client auth (fun authorization ->
+                let request = req HttpMethod.Put endpoint None authorization
+                request.Content <- new StreamContent(File.OpenRead path)
+                request.Content.Headers.ContentType <- MediaTypeHeaderValue "application/octet-stream"
+                request)
+
+        require
+            (completed.StatusCode = Net.HttpStatusCode.Created)
+            $"blob upload failed with HTTP {int completed.StatusCode}"
+
+let private putManifest client root repository auth reference (item: Manifest) =
+    use response =
+        regSend client auth (fun authorization ->
+            let request =
+                req HttpMethod.Put (regURL root repository $"manifests/{reference}") None authorization
+
+            request.Content <- new ByteArrayContent(item.Content)
+            request.Content.Headers.ContentType <- MediaTypeHeaderValue item.MediaType
+            request)
+
+    require
+        (response.StatusCode = Net.HttpStatusCode.Created)
+        $"manifest upload failed with HTTP {int response.StatusCode}"
+
+    let actual =
+        if response.Headers.Contains "Docker-Content-Digest" then
+            response.Headers.GetValues("Docker-Content-Digest") |> Seq.exactlyOne
+        else
+            fail "manifest upload omitted digest"
+
+    require (actual = item.Digest) $"published manifest digest {actual} does not match {item.Digest}"
+
+let private splitSource (source: string) =
+    let separator = source.IndexOf '/'
+    require (separator > 0 && separator < source.Length - 1) $"invalid upstream repository {source}"
+    source[.. separator - 1], source[separator + 1 ..]
+
+let retentionTag (digest: string) =
+    require (Regex.IsMatch(digest, "^sha256:[0-9a-f]{64}$")) "invalid retention digest"
+    "sha256-" + digest.Substring(7)
+
+let private tagState (client: HttpClient) (endpoint: string) (reference: string) =
+    let uri =
+        $"{endpoint}?specificTag={Uri.EscapeDataString reference}&onlyActiveTags=true"
+
+    use response = client.GetAsync(uri).GetAwaiter().GetResult()
+    require (response.StatusCode = Net.HttpStatusCode.OK) $"tag lookup failed with HTTP {int response.StatusCode}"
+    use content = response.Content.ReadAsStream()
+    use document = JsonDocument.Parse content
+    let tags = document.RootElement.GetProperty("tags").EnumerateArray() |> Seq.toArray
+    require (tags.Length <= 1) "tag lookup returned multiple active versions"
+
+    if tags.Length = 0 then
+        None
+    else
+        let tag = tags[0]
+        require (tag.GetProperty("name").GetString() = reference) "tag lookup returned a different name"
+
+        let hasValue (name: string) =
+            let mutable value = Unchecked.defaultof<JsonElement>
+            tag.TryGetProperty(name, &value) && value.ValueKind <> JsonValueKind.Null
+
+        Some(tag.GetProperty("manifest_digest").GetString(), not (hasValue "end_ts" || hasValue "expiration"))
+
+let private ensureTag (client: HttpClient) endpoint digest allowRetarget reference publish =
+    let existing = tagState client endpoint reference
+
+    match existing with
+    | Some(actual, _) when actual <> digest -> require allowRetarget "retention tag points to a different digest"
+    | _ -> ()
+
+    if existing <> Some(digest, true) then
+        if not (existing |> Option.exists (fun (actual, _) -> actual = digest)) then
+            publish reference
+
+        use payload =
+            new StringContent("{\"expiration\":null}", Encoding.UTF8, "application/json")
+
+        use response =
+            client.PutAsync(endpoint + Uri.EscapeDataString reference, payload).GetAwaiter().GetResult()
+
+        require
+            (response.StatusCode = Net.HttpStatusCode.Created)
+            $"tag expiry update failed with HTTP {int response.StatusCode}"
+
+        require (tagState client endpoint reference = Some(digest, true)) "tag retention was not confirmed"
+
+let copy source registry (api: string) (target: string) digest tag sourceAuth targetAuth certPath tlsVerify =
+    let retained = retentionTag digest
+
+    require
+        (String.IsNullOrEmpty tag
+         || (Regex.IsMatch(tag, "^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$")
+             && (not (Regex.IsMatch(tag, "^sha256-[0-9a-f]{64}$")) || tag = retained)))
+        "invalid alias or alias owned by another digest"
+
+    let sourceHost, sourceRepo = splitSource source
+    let targetScheme = if tlsVerify then "https" else "http"
+    let sourceScheme = if sourceHost = registry then targetScheme else "https"
+    let sourceRoot = Uri($"{sourceScheme}://{sourceHost}")
+    let targetRoot = Uri($"{targetScheme}://{registry}")
+    let temp = Path.Combine(Path.GetTempPath(), $"mirror-{Guid.NewGuid():N}")
+    Directory.CreateDirectory temp |> ignore
+
+    try
+        use sourceClient = regClient certPath tlsVerify
+        use targetClient = regClient certPath tlsVerify
+        use apiClient = regClient certPath tlsVerify
+        apiClient.Timeout <- TimeSpan.FromSeconds 20.
+
+        targetAuth
+        |> Option.iter (fun (_, token) ->
+            apiClient.DefaultRequestHeaders.Authorization <- AuthenticationHeaderValue("Bearer", token))
+
+        let path = target.Split('/') |> Array.map Uri.EscapeDataString |> String.concat "/"
+        let endpoint = $"{api.TrimEnd('/')}/api/v1/repository/{path}/tag/"
+        let contents = lazy (graph sourceClient sourceRoot sourceRepo sourceAuth digest)
+        let mutable uploaded = false
+
+        let publish reference =
+            let blobs, manifests = contents.Value
+
+            if not uploaded then
+                for item in blobs do
+                    putBlob
+                        targetClient
+                        targetRoot
+                        target
+                        targetAuth
+                        temp
+                        item
+                        sourceClient
+                        sourceRoot
+                        sourceRepo
+                        sourceAuth
+
+                for item in manifests[.. manifests.Length - 2] do
+                    putManifest targetClient targetRoot target targetAuth item.Digest item
+
+                uploaded <- true
+
+            putManifest targetClient targetRoot target targetAuth reference manifests[manifests.Length - 1]
+
+        ensureTag apiClient endpoint digest false retained publish
+
+        if not (String.IsNullOrEmpty tag) && tag <> retained then
+            ensureTag apiClient endpoint digest true tag publish
+    finally
+        Directory.Delete(temp, true)
+
+let deleteManifest
+    (registry: string)
+    (repository: string)
+    (digest: string)
+    (username: string)
+    (token: string)
+    (certPath: string)
+    (tlsVerify: bool)
+    =
+    let scheme = if tlsVerify then "https" else "http"
+
+    let path =
+        repository.Split('/') |> Array.map Uri.EscapeDataString |> String.concat "/"
+
+    let endpoint =
+        $"{scheme}://{registry}/v2/{path}/manifests/{Uri.EscapeDataString digest}"
+
+    use handler = httpHandler certPath tlsVerify
+    use client = new HttpClient(handler, Timeout = TimeSpan.FromSeconds 20.)
+
+    let send authorization =
+        let request = new HttpRequestMessage(HttpMethod.Delete, endpoint)
+        request.Headers.Authorization <- authorization
+        let response = client.Send request
+        request.Dispose()
+        response
+
+    let validate (response: HttpResponseMessage) =
+        require
+            (response.StatusCode = Net.HttpStatusCode.Accepted
+             || response.StatusCode = Net.HttpStatusCode.NotFound)
+            $"manifest deletion failed with HTTP {int response.StatusCode}"
+
+    use first = send null
+
+    match first.StatusCode with
+    | Net.HttpStatusCode.Accepted
+    | Net.HttpStatusCode.NotFound -> validate first
+    | Net.HttpStatusCode.Unauthorized ->
+        let accessToken = bearerToken client first (Some(username, token))
+        use response = send (AuthenticationHeaderValue("Bearer", accessToken))
+
+        validate response
+    | status -> fail $"manifest deletion failed with HTTP {int status}"
+
+let private quayError (body: string) =
+    try
+        use document = JsonDocument.Parse body
+        let root = document.RootElement
+
+        [ "error_message"; "message"; "title" ]
+        |> List.tryPick (fun name ->
+            let mutable value = Unchecked.defaultof<JsonElement>
+
+            if root.TryGetProperty(name, &value) && value.ValueKind = JsonValueKind.String then
+                value.GetString() |> Option.ofObj
+            else
+                None)
+    with :? JsonException ->
+        None
+
+let ensureRepo (api: string) (repository: string) (token: string) (certPath: string) (tlsVerify: bool) =
+    let parts = repository.Split('/', 2)
+    require (parts.Length = 2) $"invalid repository {repository}"
+
+    let payload =
+        JsonSerializer.Serialize
+            {| repository = parts[1]
+               ``namespace`` = parts[0]
+               description = ""
+               visibility = "private" |}
+
+    use handler = httpHandler certPath tlsVerify
+    use client = new HttpClient(handler, Timeout = TimeSpan.FromSeconds 20.)
+    client.DefaultRequestHeaders.Authorization <- AuthenticationHeaderValue("Bearer", token)
+    use content = new StringContent(payload, Encoding.UTF8, "application/json")
+
+    use response =
+        client.PostAsync($"{api.TrimEnd('/')}/api/v1/repository", content).GetAwaiter().GetResult()
+
+    if not response.IsSuccessStatusCode then
+        let body = response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
+        let message = quayError body
+        let detail = message |> Option.defaultValue "unavailable"
+
+        require
+            (response.StatusCode = Net.HttpStatusCode.BadRequest
+             && message = Some "Repository already exists")
+            $"repository creation failed with HTTP {int response.StatusCode}: {detail}"

services/mirrored-image/controller/controller.go

@@ -0,0 +1,708 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"reflect"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/dynamic/dynamicinformer"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/client-go/util/workqueue"
+)
+
+const (
+	prepareStage   = "prepare"
+	promotionStage = "promotion"
+	cleanupStage   = "cleanup"
+	scanAccepted   = "Accepted"
+	scanPending    = "Pending"
+	scanRejected   = "Rejected"
+)
+
+var (
+	imageGVR         = schema.GroupVersionResource{Group: "mirror.quorate.dev", Version: "v1alpha1", Resource: "mirroredimages"}
+	workflowGVR      = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflows"}
+	digestRule       = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
+	tagRule          = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
+	retentionTagRule = regexp.MustCompile(`^sha256-[0-9a-f]{64}$`)
+)
+
+type controllerConfig struct {
+	namespace         string
+	pollInterval      time.Duration
+	prepareTemplate   string
+	promotionTemplate string
+	cleanupTemplate   string
+}
+
+type controller struct {
+	client   dynamic.Interface
+	registry registryReader
+	config   controllerConfig
+}
+
+type registryReader interface {
+	scan(context.Context, string, string, string) (string, error)
+	exists(context.Context, string, string) (bool, error)
+	physicallyPresent(context.Context, string, string) (bool, error)
+}
+
+type retryRef struct {
+	digest    int
+	stage     string
+	successor bool
+}
+
+type imageKey struct {
+	namespace string
+	name      string
+}
+
+func (c *controller) run(ctx context.Context) error {
+	factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(c.client, 0, c.config.namespace, nil)
+	informer := factory.ForResource(imageGVR).Informer()
+	queue := workqueue.NewTypedRateLimitingQueueWithConfig(
+		workqueue.DefaultTypedControllerRateLimiter[imageKey](),
+		workqueue.TypedRateLimitingQueueConfig[imageKey]{Name: "mirrored-image"},
+	)
+	defer queue.ShutDown()
+
+	enqueue := func(object any) {
+		image, ok := object.(*unstructured.Unstructured)
+		if ok {
+			queue.Add(imageKey{namespace: image.GetNamespace(), name: image.GetName()})
+		}
+	}
+	_, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    enqueue,
+		UpdateFunc: func(_, newObject any) { enqueue(newObject) },
+	})
+	if err != nil {
+		return fmt.Errorf("register MirroredImage informer: %w", err)
+	}
+
+	factory.Start(ctx.Done())
+	if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		return errors.New("MirroredImage informer did not synchronize")
+	}
+	go func() {
+		<-ctx.Done()
+		queue.ShutDown()
+	}()
+
+	for c.processNext(ctx, queue) {
+	}
+	return ctx.Err()
+}
+
+func (c *controller) processNext(ctx context.Context, queue workqueue.TypedRateLimitingInterface[imageKey]) bool {
+	key, shutdown := queue.Get()
+	if shutdown {
+		return false
+	}
+	defer queue.Done(key)
+
+	found, err := c.reconcileKey(ctx, key.namespace, key.name)
+	if err != nil {
+		slog.Error("reconcile MirroredImage", "namespace", key.namespace, "name", key.name, "error", err)
+		c.recordError(ctx, key.namespace, key.name, err)
+		queue.AddRateLimited(key)
+		return true
+	}
+	queue.Forget(key)
+	if found {
+		queue.AddAfter(key, c.config.pollInterval)
+	}
+	return true
+}
+
+func (c *controller) reconcileKey(ctx context.Context, namespace, name string) (bool, error) {
+	found := false
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		object, err := c.client.Resource(imageGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		found = true
+		return c.reconcile(ctx, object)
+	})
+	return found, err
+}
+
+func (c *controller) reconcile(ctx context.Context, object *unstructured.Unstructured) error {
+	image, err := decodeImage(object)
+	if err != nil {
+		return err
+	}
+	if image.Metadata.DeletionTimestamp != nil {
+		return nil
+	}
+	if err := validateImage(&image); err != nil {
+		return err
+	}
+	retried, err := c.reconcileRetry(ctx, &image)
+	if err != nil {
+		return err
+	}
+	if !retried {
+		if err := c.reconcileDesired(ctx, &image); err != nil {
+			return err
+		}
+	}
+	image.Status.ObservedGeneration = image.Metadata.Generation
+	return c.updateStatus(ctx, object, &image)
+}
+
+func (c *controller) reconcileRetry(ctx context.Context, image *mirroredImage) (bool, error) {
+	request := image.Metadata.Annotations[retryAnnotation]
+	if request == "" || request == image.Status.LastHandledRetry {
+		return false, nil
+	}
+	target, found := retryTarget(image, request)
+	if !found {
+		return false, fmt.Errorf("retry Workflow %q is not a recorded attempt", request)
+	}
+	attempt := attemptAt(image, target)
+	if target.successor {
+		if otherAttemptActive(image, attempt.WorkflowName) {
+			return false, nil
+		}
+		status := &image.Status.Digests[target.digest]
+		if err := c.advanceAttempt(ctx, image, status, target.stage, attempt); err != nil {
+			return false, err
+		}
+		image.Status.LastHandledRetry = request
+		image.Status.Message = "retry " + target.stage + ": " + phaseOrPending(attempt.Phase)
+		return true, nil
+	}
+	if otherAttemptActive(image, request) || !terminalPhase(attempt.Phase) {
+		return false, nil
+	}
+	status := &image.Status.Digests[target.digest]
+	successor := c.newAttempt(image, status, target.stage, request)
+	switch target.stage {
+	case prepareStage:
+		status.Prepare = successor
+		status.Scan = ""
+		status.Promotion = nil
+	case promotionStage:
+		status.Promotion = successor
+	case cleanupStage:
+		status.Cleanup = successor
+	default:
+		return false, fmt.Errorf("unknown retry stage %q", target.stage)
+	}
+	if target.stage != cleanupStage && status.Cleanup != nil {
+		status.Cleanup = c.newAttempt(image, status, cleanupStage, "")
+	}
+	image.Status.Message = "retry " + target.stage + ": Pending"
+	return true, nil
+}
+
+func (c *controller) reconcileDesired(ctx context.Context, image *mirroredImage) error {
+	desired := make(map[string]digestSpec, len(image.Spec.Digests))
+	failedMessage := ""
+	for _, item := range image.Spec.Digests {
+		desired[item.Digest] = item
+		index := digestIndex(image.Status.Digests, item.Digest)
+		if index < 0 {
+			image.Status.Digests = append(image.Status.Digests, digestStatus{Digest: item.Digest, Tag: item.Tag})
+		}
+	}
+
+	for index := 0; index < len(image.Status.Digests); index++ {
+		status := &image.Status.Digests[index]
+		item, wanted := desired[status.Digest]
+		retiring := status.Cleanup != nil || !wanted
+		if retiring {
+			active := activeAttempt(status)
+			if active != nil {
+				stage := activeStage(status)
+				if err := c.advanceAttempt(ctx, image, status, stage, active); err != nil {
+					return err
+				}
+				image.Status.Message = "awaiting " + stage + " before cleanup: " + phaseOrPending(active.Phase)
+				return nil
+			}
+			switch {
+			case status.Cleanup == nil:
+				status.Cleanup = c.newAttempt(image, status, cleanupStage, "")
+				image.Status.Message = "cleanup: Pending"
+				return nil
+			case !terminalPhase(status.Cleanup.Phase):
+				if err := c.advanceAttempt(ctx, image, status, cleanupStage, status.Cleanup); err != nil {
+					return err
+				}
+				image.Status.Message = "cleanup: " + phaseOrPending(status.Cleanup.Phase)
+				return nil
+			case status.Cleanup.Phase != "Succeeded":
+				if failedMessage == "" {
+					failedMessage = "cleanup: " + status.Cleanup.Phase
+				}
+				continue
+			default:
+				complete, err := c.cleanupComplete(ctx, image.Metadata.Name, status.Digest)
+				if err != nil {
+					return err
+				}
+				if !complete {
+					image.Status.Message = "cleanup: awaiting repository GC"
+					return nil
+				}
+				if wanted {
+					*status = digestStatus{Digest: item.Digest, Tag: item.Tag}
+					image.Status.Message = "cleanup completed before remirroring"
+					return nil
+				}
+				image.Status.Digests = append(image.Status.Digests[:index], image.Status.Digests[index+1:]...)
+				image.Status.Message = "cleanup completed"
+				return nil
+			}
+		}
+
+		switch {
+		case status.Cleanup != nil:
+			return fmt.Errorf("digest %q has cleanup state while desired", status.Digest)
+		case status.Prepare == nil:
+			status.Prepare = c.newAttempt(image, status, prepareStage, "")
+			image.Status.Message = "prepare: Pending"
+			return nil
+		case !terminalPhase(status.Prepare.Phase):
+			if err := c.advanceAttempt(ctx, image, status, prepareStage, status.Prepare); err != nil {
+				return err
+			}
+			image.Status.Message = "prepare: " + phaseOrPending(status.Prepare.Phase)
+			return nil
+		case status.Prepare.Phase != "Succeeded":
+			if failedMessage == "" {
+				failedMessage = "prepare: " + status.Prepare.Phase
+			}
+			continue
+		case status.Scan == scanRejected:
+			if failedMessage == "" {
+				failedMessage = "scan: " + status.Scan
+			}
+			continue
+		case status.Scan != scanAccepted:
+			phase, err := c.registry.scan(ctx, "candidates/"+image.Metadata.Name, status.Digest, image.Spec.Policy.RejectAtOrAbove)
+			if err != nil {
+				return err
+			}
+			status.Scan = phase
+			if phase == scanRejected {
+				if failedMessage == "" {
+					failedMessage = "scan: " + phase
+				}
+				continue
+			}
+			image.Status.Message = "scan: " + phase
+			return nil
+		case status.Promotion == nil:
+			status.Promotion = c.newAttempt(image, status, promotionStage, "")
+			image.Status.Message = "promotion: Pending"
+			return nil
+		case !terminalPhase(status.Promotion.Phase):
+			if err := c.advanceAttempt(ctx, image, status, promotionStage, status.Promotion); err != nil {
+				return err
+			}
+			image.Status.Message = "promotion: " + phaseOrPending(status.Promotion.Phase)
+			return nil
+		case status.Promotion.Phase != "Succeeded":
+			if failedMessage == "" {
+				failedMessage = "promotion: " + status.Promotion.Phase
+			}
+		default:
+			present, err := c.registry.exists(ctx, "mirror/"+image.Metadata.Name, status.Digest)
+			if err != nil {
+				return err
+			}
+			if !present {
+				image.Status.Message = "promotion: awaiting mirror"
+				return nil
+			}
+		}
+	}
+
+	if failedMessage == "" {
+		image.Status.Message = "desired digests reconciled"
+	} else {
+		image.Status.Message = failedMessage
+	}
+	return nil
+}
+
+func (c *controller) newAttempt(image *mirroredImage, status *digestStatus, stage, retryOf string) *attemptStatus {
+	attempt := &attemptStatus{RetryOf: retryOf, Phase: "Pending", Generation: image.Metadata.Generation}
+	lineage := []string{retryOf}
+	if stage == promotionStage && status.Prepare != nil {
+		lineage = append(lineage, status.Prepare.WorkflowName)
+	}
+	if stage == cleanupStage {
+		for _, copy := range []*attemptStatus{status.Prepare, status.Promotion} {
+			if copy != nil {
+				lineage = append(lineage, copy.WorkflowName)
+			}
+		}
+	}
+	attempt.WorkflowName = attemptName(image, status.Digest, stage, attempt.Generation, lineage...)
+	return attempt
+}
+
+func (c *controller) advanceAttempt(
+	ctx context.Context,
+	image *mirroredImage,
+	status *digestStatus,
+	stage string,
+	attempt *attemptStatus,
+) error {
+	if attempt.WorkflowName == "" {
+		return errors.New("workflow attempt has no name")
+	}
+	template, parameters, err := c.workflowInput(image, status, stage)
+	if err != nil {
+		return err
+	}
+	if err := c.ensureWorkflow(ctx, image, stage, attempt.WorkflowName, template, parameters); err != nil {
+		return err
+	}
+	phase, err := c.workflowPhase(ctx, image.Metadata.Namespace, attempt.WorkflowName)
+	if err != nil {
+		return err
+	}
+	attempt.Phase = phaseOrPending(phase)
+	return nil
+}
+
+func (c *controller) cleanupComplete(ctx context.Context, name, digest string) (bool, error) {
+	candidate, err := c.registry.physicallyPresent(ctx, "candidates/"+name, digest)
+	if err != nil {
+		return false, err
+	}
+	mirror, err := c.registry.physicallyPresent(ctx, "mirror/"+name, digest)
+	if err != nil {
+		return false, err
+	}
+	return !candidate && !mirror, nil
+}
+
+func (c *controller) workflowInput(
+	image *mirroredImage,
+	status *digestStatus,
+	stage string,
+) (string, map[string]string, error) {
+	parameters := map[string]string{
+		"digest": status.Digest,
+	}
+	switch stage {
+	case prepareStage:
+		parameters["tag"] = status.Tag
+		parameters["source-repository"] = image.Spec.Upstream
+		parameters["target-repository"] = "candidates/" + image.Metadata.Name
+		return c.config.prepareTemplate, parameters, nil
+	case promotionStage:
+		parameters["tag"] = status.Tag
+		parameters["source-repository"] = "candidates/" + image.Metadata.Name
+		parameters["target-repository"] = "mirror/" + image.Metadata.Name
+		return c.config.promotionTemplate, parameters, nil
+	case cleanupStage:
+		parameters["candidate-repository"] = "candidates/" + image.Metadata.Name
+		parameters["mirror-repository"] = "mirror/" + image.Metadata.Name
+		return c.config.cleanupTemplate, parameters, nil
+	default:
+		return "", nil, fmt.Errorf("unknown Workflow stage %q", stage)
+	}
+}
+
+func (c *controller) ensureWorkflow(
+	ctx context.Context,
+	image *mirroredImage,
+	stage, name, template string,
+	parameters map[string]string,
+) error {
+	if stage == promotionStage {
+		_, err := c.client.Resource(workflowGVR).Namespace(image.Metadata.Namespace).Get(ctx, name, metav1.GetOptions{})
+		if err == nil {
+			return nil
+		}
+		if !apierrors.IsNotFound(err) {
+			return err
+		}
+		phase, err := c.registry.scan(ctx, "candidates/"+image.Metadata.Name, parameters["digest"], image.Spec.Policy.RejectAtOrAbove)
+		if err != nil {
+			return err
+		}
+		if phase != scanAccepted {
+			return fmt.Errorf("promotion scan: %s", phase)
+		}
+	}
+	names := make([]string, 0, len(parameters))
+	for name := range parameters {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	arguments := make([]any, 0, len(names))
+	for _, name := range names {
+		arguments = append(arguments, map[string]any{"name": name, "value": parameters[name]})
+	}
+	workflow := &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": "argoproj.io/v1alpha1",
+		"kind":       "Workflow",
+		"metadata": map[string]any{
+			"name":      name,
+			"namespace": image.Metadata.Namespace,
+			"labels": map[string]any{
+				"app.kubernetes.io/name":     "mirrored-image",
+				"app.kubernetes.io/instance": image.Metadata.Name,
+				"mirror.quorate.dev/purpose": stage,
+			},
+			"ownerReferences": []any{map[string]any{
+				"apiVersion":         "mirror.quorate.dev/v1alpha1",
+				"kind":               "MirroredImage",
+				"name":               image.Metadata.Name,
+				"uid":                image.Metadata.UID,
+				"controller":         true,
+				"blockOwnerDeletion": false,
+			}},
+		},
+		"spec": map[string]any{
+			"workflowTemplateRef": map[string]any{"name": template},
+			"arguments":           map[string]any{"parameters": arguments},
+			"synchronization": map[string]any{
+				"mutexes": []any{map[string]any{"name": workflowName("lock", image.Metadata.UID)}},
+			},
+		},
+	}}
+	_, err := c.client.Resource(workflowGVR).Namespace(image.Metadata.Namespace).Create(ctx, workflow, metav1.CreateOptions{})
+	if apierrors.IsAlreadyExists(err) {
+		return nil
+	}
+	return err
+}
+
+func (c *controller) workflowPhase(ctx context.Context, namespace, name string) (string, error) {
+	object, err := c.client.Resource(workflowGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+	phase, _, _ := unstructured.NestedString(object.Object, "status", "phase")
+	return phase, nil
+}
+
+func (c *controller) updateStatus(ctx context.Context, object *unstructured.Unstructured, image *mirroredImage) error {
+	status, err := toMap(image.Status)
+	if err != nil {
+		return err
+	}
+	current, _, err := unstructured.NestedMap(object.Object, "status")
+	if err != nil {
+		return err
+	}
+	if reflect.DeepEqual(current, status) {
+		return nil
+	}
+	updated := object.DeepCopy()
+	updated.Object["status"] = status
+	_, err = c.client.Resource(imageGVR).Namespace(updated.GetNamespace()).UpdateStatus(ctx, updated, metav1.UpdateOptions{})
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	return err
+}
+
+func (c *controller) recordError(ctx context.Context, namespace, name string, reconcileError error) {
+	object, err := c.client.Resource(imageGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return
+	}
+	image, err := decodeImage(object)
+	if err != nil {
+		return
+	}
+	image.Status.Message = reconcileError.Error()
+	image.Status.ObservedGeneration = image.Metadata.Generation
+	_ = c.updateStatus(ctx, object, &image)
+}
+
+func decodeImage(object *unstructured.Unstructured) (mirroredImage, error) {
+	content, err := json.Marshal(object.Object)
+	if err != nil {
+		return mirroredImage{}, err
+	}
+	var image mirroredImage
+	if err := json.Unmarshal(content, &image); err != nil {
+		return mirroredImage{}, err
+	}
+	return image, nil
+}
+
+func validateImage(image *mirroredImage) error {
+	if image.Spec.Upstream == "" {
+		return errors.New("upstream repository is empty")
+	}
+	if image.Spec.Policy.RejectAtOrAbove == "" {
+		return errors.New("severity threshold is empty")
+	}
+	seen := make(map[string]struct{}, len(image.Spec.Digests))
+	for index, item := range image.Spec.Digests {
+		if !digestRule.MatchString(item.Digest) {
+			return fmt.Errorf("invalid digest at index %d", index)
+		}
+		if item.Tag != "" && !tagRule.MatchString(item.Tag) {
+			return fmt.Errorf("invalid tag at index %d", index)
+		}
+		if retentionTagRule.MatchString(item.Tag) && item.Tag != "sha256-"+strings.TrimPrefix(item.Digest, "sha256:") {
+			return fmt.Errorf("tag at index %d is owned by another digest", index)
+		}
+		if _, found := seen[item.Digest]; found {
+			return fmt.Errorf("duplicate digest at index %d", index)
+		}
+		seen[item.Digest] = struct{}{}
+	}
+	return nil
+}
+
+func retryTarget(image *mirroredImage, workflow string) (retryRef, bool) {
+	for index := range image.Status.Digests {
+		status := &image.Status.Digests[index]
+		if status.Prepare != nil {
+			if status.Prepare.WorkflowName == workflow {
+				return retryRef{digest: index, stage: prepareStage}, true
+			}
+			if status.Prepare.RetryOf == workflow {
+				return retryRef{digest: index, stage: prepareStage, successor: true}, true
+			}
+		}
+		if status.Promotion != nil {
+			if status.Promotion.WorkflowName == workflow {
+				return retryRef{digest: index, stage: promotionStage}, true
+			}
+			if status.Promotion.RetryOf == workflow {
+				return retryRef{digest: index, stage: promotionStage, successor: true}, true
+			}
+		}
+		if status.Cleanup != nil {
+			if status.Cleanup.WorkflowName == workflow {
+				return retryRef{digest: index, stage: cleanupStage}, true
+			}
+			if status.Cleanup.RetryOf == workflow {
+				return retryRef{digest: index, stage: cleanupStage, successor: true}, true
+			}
+		}
+	}
+	return retryRef{}, false
+}
+
+func attemptAt(image *mirroredImage, target retryRef) *attemptStatus {
+	status := &image.Status.Digests[target.digest]
+	if target.stage == prepareStage {
+		return status.Prepare
+	}
+	if target.stage == promotionStage {
+		return status.Promotion
+	}
+	return status.Cleanup
+}
+
+func otherAttemptActive(image *mirroredImage, excluded string) bool {
+	for index := range image.Status.Digests {
+		status := &image.Status.Digests[index]
+		for _, attempt := range []*attemptStatus{status.Prepare, status.Promotion, status.Cleanup} {
+			if attempt == status.Cleanup && attempt != nil && attempt.Phase == "Pending" && activeAttempt(status) != nil {
+				continue
+			}
+			if attempt != nil && attempt.WorkflowName != excluded && !terminalPhase(attempt.Phase) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func activeAttempt(status *digestStatus) *attemptStatus {
+	for _, attempt := range []*attemptStatus{status.Prepare, status.Promotion} {
+		if attempt != nil && !terminalPhase(attempt.Phase) {
+			return attempt
+		}
+	}
+	return nil
+}
+
+func activeStage(status *digestStatus) string {
+	if status.Prepare != nil && !terminalPhase(status.Prepare.Phase) {
+		return prepareStage
+	}
+	return promotionStage
+}
+
+func digestIndex(statuses []digestStatus, digest string) int {
+	for index := range statuses {
+		if statuses[index].Digest == digest {
+			return index
+		}
+	}
+	return -1
+}
+
+func toMap(value any) (map[string]any, error) {
+	content, err := json.Marshal(value)
+	if err != nil {
+		return nil, err
+	}
+	var result map[string]any
+	err = json.Unmarshal(content, &result)
+	return result, err
+}
+
+func workflowName(purpose string, fields ...string) string {
+	digest := sha256.Sum256([]byte("v1\x00" + purpose + "\x00" + strings.Join(fields, "\x00")))
+	encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(digest[:15])
+	return "mi-" + purpose + "-" + strings.ToLower(encoded)
+}
+
+func attemptName(image *mirroredImage, digest, stage string, generation int64, lineage ...string) string {
+	fields := []string{image.Metadata.UID, digest}
+	lineageFound := false
+	for _, value := range lineage {
+		if value != "" {
+			fields = append(fields, value)
+			lineageFound = true
+		}
+	}
+	if !lineageFound {
+		fields = append(fields, strconv.FormatInt(generation, 10))
+	}
+	return workflowName(stage, fields...)
+}
+
+func terminalPhase(phase string) bool {
+	return phase == "Succeeded" || phase == "Failed" || phase == "Error"
+}
+
+func phaseOrPending(phase string) string {
+	if phase == "" {
+		return "Pending"
+	}
+	return phase
+}

services/mirrored-image/controller/controller_test.go

@@ -0,0 +1,577 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic/fake"
+)
+
+const testDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
+func TestRetentionTagOwnership(t *testing.T) {
+	t.Parallel()
+	for _, test := range []struct {
+		tag   string
+		valid bool
+	}{
+		{"", true},
+		{"stable", true},
+		{"sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true},
+		{"sha256-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", false},
+		{"sha256-release", true},
+	} {
+		t.Run(test.tag, func(t *testing.T) {
+			image := testImage()
+			image.Spec.Digests = []digestSpec{{Digest: testDigest, Tag: test.tag}}
+			err := validateImage(&image)
+			if (err == nil) != test.valid {
+				t.Fatalf("RetentionTags.AliasAllowed: valid=%t err=%v", test.valid, err)
+			}
+		})
+	}
+}
+
+type fakeRegistry struct {
+	scanPhase string
+	present   map[string]bool
+	stored    map[string]bool
+}
+
+func (f *fakeRegistry) scan(context.Context, string, string, string) (string, error) {
+	return f.scanPhase, nil
+}
+
+func (f *fakeRegistry) exists(_ context.Context, repository, digest string) (bool, error) {
+	return f.present[repository+"@"+digest], nil
+}
+
+func (f *fakeRegistry) physicallyPresent(_ context.Context, repository, digest string) (bool, error) {
+	if f.stored != nil {
+		return f.stored[repository+"@"+digest], nil
+	}
+	return f.present[repository+"@"+digest], nil
+}
+
+func TestCleanupObservedMeansGone(t *testing.T) {
+	t.Parallel()
+	for _, repository := range []string{"candidates/pause", "mirror/pause"} {
+		t.Run(repository, func(t *testing.T) {
+			registry := &fakeRegistry{stored: map[string]bool{repository + "@" + testDigest: true}}
+			instance := testController(newClient(), registry)
+			image := testImage()
+			image.Spec.Digests = nil
+			image.Status.Digests = []digestStatus{{Digest: testDigest, Cleanup: &attemptStatus{WorkflowName: "cleanup", Phase: "Succeeded"}}}
+			reconcileDesired(context.Background(), t, instance, &image)
+			if len(image.Status.Digests) != 1 {
+				t.Fatal("CleanupObservedMeansGone: physical manifest was forgotten")
+			}
+			delete(registry.stored, repository+"@"+testDigest)
+			reconcileDesired(context.Background(), t, instance, &image)
+			if len(image.Status.Digests) != 0 {
+				t.Fatal("physically absent digest was retained")
+			}
+		})
+	}
+}
+
+func TestCopyRetryRearmsCleanup(t *testing.T) {
+	t.Parallel()
+	for _, stage := range []string{prepareStage, promotionStage} {
+		for _, readd := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s/readd=%t", stage, readd), func(t *testing.T) {
+				ctx := context.Background()
+				client := newClient()
+				registry := &fakeRegistry{scanPhase: scanAccepted, stored: map[string]bool{}}
+				instance := testController(client, registry)
+				image := testImage()
+				image.Spec.Digests = nil
+				image.Status.Digests = []digestStatus{{
+					Digest: testDigest, Tag: "stable", Scan: scanAccepted,
+					Prepare:   &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
+					Promotion: &attemptStatus{WorkflowName: "promotion", Phase: "Succeeded"},
+				}}
+				status := &image.Status.Digests[0]
+				status.Cleanup = instance.newAttempt(&image, status, cleanupStage, "")
+				status.Cleanup.Phase = "Succeeded"
+				oldCleanup := status.Cleanup.WorkflowName
+				if _, err := client.Resource(workflowGVR).Namespace("mirrored-image").Create(ctx, workflowObject(oldCleanup, "Succeeded"), metav1.CreateOptions{}); err != nil {
+					t.Fatal(err)
+				}
+				image.Metadata.Annotations = map[string]string{retryAnnotation: stage}
+				for pass := 0; pass < 2; pass++ {
+					handled, err := instance.reconcileRetry(ctx, &image)
+					if err != nil || !handled {
+						t.Fatalf("retry pass %d: handled=%t err=%v", pass, handled, err)
+					}
+					if pass == 0 {
+						if image.Status.LastHandledRetry != "" {
+							t.Fatal("RetryAcknowledgedAfterCreation")
+						}
+						if readd {
+							image.Spec.Digests = []digestSpec{{Digest: testDigest, Tag: "edited"}}
+						}
+					}
+				}
+				if image.Status.LastHandledRetry != stage {
+					t.Fatal("retry was not acknowledged")
+				}
+				copy := status.Prepare
+				if stage == promotionStage {
+					copy = status.Promotion
+				}
+				setPhase(ctx, t, client, copy.WorkflowName, "Succeeded")
+				registry.stored["candidates/pause@"+testDigest] = true
+				reconcileDesired(ctx, t, instance, &image)
+				reconcileDesired(ctx, t, instance, &image)
+				if status.Cleanup == nil || status.Cleanup.WorkflowName == oldCleanup || terminalPhase(status.Cleanup.Phase) {
+					t.Fatal("copy retry reused completed cleanup")
+				}
+				fresh := status.Cleanup.WorkflowName
+				if _, err := client.Resource(workflowGVR).Namespace("mirrored-image").Get(ctx, fresh, metav1.GetOptions{}); err != nil {
+					t.Fatal(err)
+				}
+				if status.Tag != "stable" {
+					t.Fatal("RetryPreservesTagIdentity")
+				}
+				setPhase(ctx, t, client, fresh, "Succeeded")
+				reconcileDesired(ctx, t, instance, &image)
+				reconcileDesired(ctx, t, instance, &image)
+				if len(image.Status.Digests) != 1 {
+					t.Fatal("CleanupObservedMeansGone")
+				}
+				delete(registry.stored, "candidates/pause@"+testDigest)
+				reconcileDesired(ctx, t, instance, &image)
+				if readd {
+					if status.Cleanup != nil || status.Prepare != nil || status.Tag != "edited" {
+						t.Fatal("re-addition did not wait for fresh cleanup")
+					}
+				} else if len(image.Status.Digests) != 0 {
+					t.Fatal("retirement did not complete")
+				}
+			})
+		}
+	}
+}
+
+func TestDigestLifecycle(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	registry := &fakeRegistry{scanPhase: scanAccepted, present: map[string]bool{}}
+	instance := testController(client, registry)
+	image := testImage()
+
+	reconcileDesired(ctx, t, instance, &image)
+	status := &image.Status.Digests[0]
+	if status.Prepare == nil || status.Prepare.WorkflowName == "" {
+		t.Fatal("prepare workflow was not recorded")
+	}
+	reconcileDesired(ctx, t, instance, &image)
+	setPhase(ctx, t, client, status.Prepare.WorkflowName, "Succeeded")
+	reconcileDesired(ctx, t, instance, &image)
+	reconcileDesired(ctx, t, instance, &image)
+	if image.Status.Digests[0].Scan != scanAccepted {
+		t.Fatal("candidate scan was not accepted")
+	}
+	reconcileDesired(ctx, t, instance, &image)
+	status = &image.Status.Digests[0]
+	if status.Promotion == nil || status.Promotion.WorkflowName == "" {
+		t.Fatal("promotion workflow was not recorded")
+	}
+	reconcileDesired(ctx, t, instance, &image)
+	setPhase(ctx, t, client, status.Promotion.WorkflowName, "Succeeded")
+	registry.present["mirror/pause@"+testDigest] = true
+	reconcileDesired(ctx, t, instance, &image)
+
+	image.Metadata.Generation++
+	image.Spec.Digests = nil
+	reconcileDesired(ctx, t, instance, &image)
+	status = &image.Status.Digests[0]
+	if status.Cleanup == nil || status.Cleanup.WorkflowName == "" {
+		t.Fatal("cleanup workflow was not recorded")
+	}
+	reconcileDesired(ctx, t, instance, &image)
+	setPhase(ctx, t, client, status.Cleanup.WorkflowName, "Succeeded")
+	delete(registry.present, "mirror/pause@"+testDigest)
+	reconcileDesired(ctx, t, instance, &image)
+	reconcileDesired(ctx, t, instance, &image)
+	if len(image.Status.Digests) != 0 {
+		t.Fatalf("cleanup retained %d digest statuses", len(image.Status.Digests))
+	}
+}
+
+func TestAttemptIsRecordedFirst(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	instance := testController(client, &fakeRegistry{present: map[string]bool{}})
+	image := testImage()
+
+	reconcileDesired(ctx, t, instance, &image)
+	if image.Status.Digests[0].Prepare == nil {
+		t.Fatal("prepare attempt was not recorded")
+	}
+	workflows, err := client.Resource(workflowGVR).Namespace("mirrored-image").List(ctx, metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(workflows.Items) != 0 {
+		t.Fatal("workflow existed before its status record")
+	}
+
+	reconcileDesired(ctx, t, instance, &image)
+	workflows, err = client.Resource(workflowGVR).Namespace("mirrored-image").List(ctx, metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(workflows.Items) != 1 || workflows.Items[0].GetName() != image.Status.Digests[0].Prepare.WorkflowName {
+		t.Fatal("recorded prepare workflow was not created")
+	}
+}
+
+func TestCleanupWaitsForAbsence(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	registry := &fakeRegistry{present: map[string]bool{"candidates/pause@" + testDigest: true}}
+	instance := testController(client, registry)
+	image := testImage()
+	image.Spec.Digests = nil
+	image.Status.Digests = []digestStatus{{
+		Digest:  testDigest,
+		Cleanup: &attemptStatus{WorkflowName: "cleanup", Phase: "Succeeded"},
+	}}
+
+	reconcileDesired(ctx, t, instance, &image)
+	if len(image.Status.Digests) != 1 || image.Status.Message != "cleanup: awaiting repository GC" {
+		t.Fatal("cleanup completed while the candidate remained present")
+	}
+	delete(registry.present, "candidates/pause@"+testDigest)
+	reconcileDesired(ctx, t, instance, &image)
+	if len(image.Status.Digests) != 0 {
+		t.Fatal("cleanup state remained after both repositories were absent")
+	}
+}
+
+func TestReaddWaitsForCleanup(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	registry := &fakeRegistry{scanPhase: scanAccepted, present: map[string]bool{}}
+	instance := testController(client, registry)
+	image := testImage()
+	oldPrepare := attemptName(&image, testDigest, prepareStage, image.Metadata.Generation, "")
+	image.Metadata.Generation = 3
+	image.Status.Digests = []digestStatus{{
+		Digest:    testDigest,
+		Tag:       "stable",
+		Prepare:   &attemptStatus{WorkflowName: oldPrepare, Phase: "Succeeded", Generation: 1},
+		Scan:      scanAccepted,
+		Promotion: &attemptStatus{WorkflowName: "old-promotion", Phase: "Succeeded"},
+		Cleanup:   &attemptStatus{WorkflowName: "old-cleanup", Phase: "Running"},
+	}}
+	workflow := workflowObject("old-cleanup", "Running")
+	if _, err := client.Resource(workflowGVR).Namespace("mirrored-image").Create(ctx, workflow, metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+
+	reconcileDesired(ctx, t, instance, &image)
+	if image.Status.Digests[0].Prepare.WorkflowName != oldPrepare {
+		t.Fatal("prepare restarted while cleanup was active")
+	}
+	setPhase(ctx, t, client, "old-cleanup", "Succeeded")
+	reconcileDesired(ctx, t, instance, &image)
+	reconcileDesired(ctx, t, instance, &image)
+	status := image.Status.Digests[0]
+	if status.Cleanup != nil || status.Prepare != nil {
+		t.Fatal("completed cleanup did not reset digest state")
+	}
+	reconcileDesired(ctx, t, instance, &image)
+	if image.Status.Digests[0].Prepare == nil {
+		t.Fatal("prepare did not restart after cleanup")
+	}
+	if image.Status.Digests[0].Prepare.WorkflowName == oldPrepare {
+		t.Fatal("prepare reused the workflow from an earlier generation")
+	}
+}
+
+func TestRetryCreatesSuccessor(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	registry := &fakeRegistry{scanPhase: scanAccepted, present: map[string]bool{}}
+	instance := testController(client, registry)
+	image := testImage()
+	image.Status.Digests = []digestStatus{{
+		Digest:  testDigest,
+		Tag:     "stable",
+		Prepare: &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"},
+	}}
+	image.Metadata.Annotations = map[string]string{retryAnnotation: "failed-prepare"}
+
+	retried, err := instance.reconcileRetry(ctx, &image)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !retried {
+		t.Fatal("terminal retry was not handled")
+	}
+	attempt := image.Status.Digests[0].Prepare
+	if attempt.RetryOf != "failed-prepare" || attempt.WorkflowName == "failed-prepare" {
+		t.Fatalf("invalid successor: %#v", attempt)
+	}
+	if image.Status.LastHandledRetry != "" {
+		t.Fatal("retry was acknowledged before the successor existed")
+	}
+	retried, err = instance.reconcileRetry(ctx, &image)
+	if err != nil || !retried {
+		t.Fatalf("create retry successor: retried=%v error=%v", retried, err)
+	}
+	if image.Status.LastHandledRetry != "failed-prepare" {
+		t.Fatal("retry was not acknowledged after the successor existed")
+	}
+	setPhase(ctx, t, client, attempt.WorkflowName, "Succeeded")
+	reconcileDesired(ctx, t, instance, &image)
+	reconcileDesired(ctx, t, instance, &image)
+	reconcileDesired(ctx, t, instance, &image)
+	promotion := image.Status.Digests[0].Promotion
+	if promotion == nil || promotion.WorkflowName == "" {
+		t.Fatal("promotion was not created after the retry")
+	}
+	oldName := attemptName(&image, testDigest, promotionStage, image.Metadata.Generation)
+	if promotion.WorkflowName == oldName {
+		t.Fatal("promotion reused the predecessor prepare lineage")
+	}
+}
+
+func TestRetryNameIgnoresGeneration(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	instance := testController(client, &fakeRegistry{present: map[string]bool{}})
+	image := testImage()
+	image.Status.Digests = []digestStatus{{
+		Digest:  testDigest,
+		Prepare: &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"},
+	}}
+	image.Metadata.Annotations = map[string]string{retryAnnotation: "failed-prepare"}
+
+	if _, err := instance.reconcileRetry(ctx, &image); err != nil {
+		t.Fatal(err)
+	}
+	first := image.Status.Digests[0].Prepare.WorkflowName
+	image.Metadata.Generation++
+	image.Status.Digests[0].Prepare = &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"}
+	image.Status.LastHandledRetry = ""
+	if _, err := instance.reconcileRetry(ctx, &image); err != nil {
+		t.Fatal(err)
+	}
+	if second := image.Status.Digests[0].Prepare.WorkflowName; second != first {
+		t.Fatalf("retry generation changed successor from %q to %q", first, second)
+	}
+}
+
+func TestRunStopsWhenIdle(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	client := newClient()
+	instance := testController(client, &fakeRegistry{present: map[string]bool{}})
+	done := make(chan error, 1)
+	go func() {
+		done <- instance.run(ctx)
+	}()
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+	select {
+	case err := <-done:
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("run returned %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("run did not stop")
+	}
+}
+
+func TestRejectedCandidateIsNotPromoted(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	registry := &fakeRegistry{scanPhase: scanRejected, present: map[string]bool{}}
+	instance := testController(client, registry)
+	image := testImage()
+	image.Status.Digests = []digestStatus{{
+		Digest:  testDigest,
+		Tag:     "stable",
+		Prepare: &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
+	}}
+
+	reconcileDesired(ctx, t, instance, &image)
+	status := image.Status.Digests[0]
+	if status.Scan != scanRejected || status.Promotion != nil {
+		t.Fatal("rejected candidate was promoted")
+	}
+	if image.Status.Message != "scan: Rejected" {
+		t.Fatalf("unexpected status message %q", image.Status.Message)
+	}
+}
+
+func TestTaglessDeclarationIsValid(t *testing.T) {
+	t.Parallel()
+	image := testImage()
+	image.Spec.Digests[0].Tag = ""
+	if err := validateImage(&image); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestTagEditPreservesState(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	registry := &fakeRegistry{present: map[string]bool{"mirror/pause@" + testDigest: true}}
+	instance := testController(client, registry)
+	image := testImage()
+	image.Spec.Digests[0].Tag = "new"
+	image.Status.Digests = []digestStatus{{
+		Digest:    testDigest,
+		Tag:       "stable",
+		Prepare:   &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
+		Scan:      scanAccepted,
+		Promotion: &attemptStatus{WorkflowName: "promotion", Phase: "Succeeded"},
+	}}
+
+	reconcileDesired(ctx, t, instance, &image)
+	status := image.Status.Digests[0]
+	if status.Tag != "stable" || status.Cleanup != nil {
+		t.Fatalf("tag edit changed managed state: %#v", status)
+	}
+	workflows, err := client.Resource(workflowGVR).Namespace("mirrored-image").List(ctx, metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(workflows.Items) != 0 {
+		t.Fatal("tag edit created a workflow")
+	}
+}
+
+func TestFailureMessage(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	client := newClient()
+	registry := &fakeRegistry{present: map[string]bool{}}
+	instance := testController(client, registry)
+	image := testImage()
+	image.Status.Digests = []digestStatus{{
+		Digest:  testDigest,
+		Tag:     "stable",
+		Prepare: &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"},
+	}}
+
+	reconcileDesired(ctx, t, instance, &image)
+	if image.Status.Message != "prepare: Failed" {
+		t.Fatalf("unexpected status message %q", image.Status.Message)
+	}
+}
+
+func TestPromotionRescan(t *testing.T) {
+	for _, phase := range []string{scanPending, scanRejected, scanAccepted} {
+		t.Run(phase, func(t *testing.T) {
+			ctx := context.Background()
+			client := newClient()
+			registry := &fakeRegistry{scanPhase: phase}
+			instance := testController(client, registry)
+			image := testImage()
+			image.Status.Digests = []digestStatus{{
+				Digest: testDigest, Scan: scanAccepted,
+				Prepare:   &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
+				Promotion: &attemptStatus{WorkflowName: "promotion", Phase: "Pending"},
+			}}
+			err := instance.reconcileDesired(ctx, &image)
+			if (err == nil) != (phase == scanAccepted) {
+				t.Fatalf("scan %s: %v", phase, err)
+			}
+			workflows, err := client.Resource(workflowGVR).Namespace(image.Metadata.Namespace).List(ctx, metav1.ListOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if (len(workflows.Items) == 1) != (phase == scanAccepted) {
+				t.Fatalf("scan %s created %d workflows", phase, len(workflows.Items))
+			}
+			if phase == scanAccepted {
+				registry.scanPhase = scanPending
+				if err := instance.reconcileDesired(ctx, &image); err != nil {
+					t.Fatalf("existing workflow observation requires a scan: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func testController(client *fake.FakeDynamicClient, registry registryReader) *controller {
+	return &controller{client: client, registry: registry, config: controllerConfig{
+		prepareTemplate:   "image-prepare",
+		promotionTemplate: "image-promote",
+		cleanupTemplate:   "image-cleanup",
+	}}
+}
+
+func newClient(objects ...runtime.Object) *fake.FakeDynamicClient {
+	return fake.NewSimpleDynamicClientWithCustomListKinds(
+		runtime.NewScheme(),
+		map[schema.GroupVersionResource]string{
+			imageGVR:    "MirroredImageList",
+			workflowGVR: "WorkflowList",
+		},
+		objects...,
+	)
+}
+
+func testImage() mirroredImage {
+	return mirroredImage{
+		Metadata: objectMetadata{Name: "pause", Namespace: "mirrored-image", UID: "image-uid", Generation: 1},
+		Spec: mirroredImageSpec{
+			Upstream: "registry.k8s.io/pause",
+			Policy:   scanPolicy{RejectAtOrAbove: "High"},
+			Digests:  []digestSpec{{Digest: testDigest, Tag: "stable"}},
+		},
+	}
+}
+
+func reconcileDesired(ctx context.Context, t *testing.T, instance *controller, image *mirroredImage) {
+	t.Helper()
+	if err := instance.reconcileDesired(ctx, image); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func setPhase(ctx context.Context, t *testing.T, client *fake.FakeDynamicClient, name, phase string) {
+	t.Helper()
+	resource := client.Resource(workflowGVR).Namespace("mirrored-image")
+	object, err := resource.Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := unstructured.SetNestedField(object.Object, phase, "status", "phase"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := resource.Update(ctx, object, metav1.UpdateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func workflowObject(name, phase string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": "argoproj.io/v1alpha1",
+		"kind":       "Workflow",
+		"metadata":   map[string]any{"name": name, "namespace": "mirrored-image"},
+		"status":     map[string]any{"phase": phase},
+	}}
+}

services/mirrored-image/controller/go.mod

@@ -0,0 +1,57 @@
+module quorate.dev/services/mirrored-image-controller
+
+go 1.27.1
+
+require (
+	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
+	k8s.io/apimachinery v0.37.0
+	k8s.io/client-go v0.37.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-openapi/jsonpointer v1.0.0 // indirect
+	github.com/go-openapi/jsonreference v1.0.0 // indirect
+	github.com/go-openapi/swag v0.27.1 // indirect
+	github.com/go-openapi/swag/cmdutils v0.27.1 // indirect
+	github.com/go-openapi/swag/conv v0.27.1 // indirect
+	github.com/go-openapi/swag/fileutils v0.27.1 // indirect
+	github.com/go-openapi/swag/jsonutils v0.27.1 // indirect
+	github.com/go-openapi/swag/loading v0.27.1 // indirect
+	github.com/go-openapi/swag/mangling v0.27.1 // indirect
+	github.com/go-openapi/swag/netutils v0.27.1 // indirect
+	github.com/go-openapi/swag/pools v0.27.1 // indirect
+	github.com/go-openapi/swag/stringutils v0.27.1 // indirect
+	github.com/go-openapi/swag/typeutils v0.27.1 // indirect
+	github.com/go-openapi/swag/yamlutils v0.27.1 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	k8s.io/api v0.37.0 // indirect
+	k8s.io/klog/v2 v2.140.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect
+	k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+)

services/mirrored-image/controller/go.sum

@@ -0,0 +1,133 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
+github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
+github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s=
+github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y=
+github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY=
+github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0=
+github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg=
+github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE=
+github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE=
+github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM=
+github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU=
+github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs=
+github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM=
+github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8=
+github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU=
+github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY=
+github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY=
+github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE=
+github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA=
+github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w=
+github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU=
+github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ=
+github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E=
+github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE=
+github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8=
+github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM=
+github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc=
+github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ=
+github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA=
+github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo=
+github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0=
+github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo=
+github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
+github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f h1:GvCU5GXhHq+7LeOzx/haG7HSIZokl3/0GkoUFzsRJjg=
+github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f/go.mod h1:q59u9px8b7UTj0nIjEjvmTWekazka6xIt6Uogz5Dm+8=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
+gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4=
+k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk=
+k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0=
+k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE=
+k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ=
+k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0=
+k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
+k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
+k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A=
+k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I=
+k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs=
+k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

services/mirrored-image/controller/inventory.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"fmt"
+
+	apk "github.com/knqyf263/go-apk-version"
+)
+
+type scanInventory struct {
+	Hash          string                        `json:"manifest_hash"`
+	Packages      map[string]*scanPackage       `json:"packages"`
+	Distributions map[string]*scanDistribution  `json:"distributions"`
+	Environments  map[string][]*scanEnvironment `json:"environments"`
+}
+
+type scanPackage struct {
+	ID      string       `json:"id"`
+	Name    string       `json:"name"`
+	Version string       `json:"version"`
+	Kind    string       `json:"kind"`
+	Source  *scanPackage `json:"source"`
+}
+
+type scanDistribution struct {
+	ID         string `json:"id"`
+	DID        string `json:"did"`
+	Name       string `json:"name"`
+	Version    string `json:"version"`
+	PrettyName string `json:"pretty_name"`
+}
+
+type scanEnvironment struct {
+	PackageDB      string   `json:"package_db"`
+	IntroducedIn   string   `json:"introduced_in"`
+	DistributionID string   `json:"distribution_id"`
+	RepositoryIDs  []string `json:"repository_ids"`
+}
+
+type featureKey struct {
+	Name    string
+	Version string
+	Layer   string
+}
+
+func checkInventory(data *securityData, digest string, layers map[string]bool) error {
+	if data == nil || data.Inventory == nil || data.Layer == nil || data.Layer.Features == nil {
+		return fmt.Errorf("scan inventory is unavailable")
+	}
+	inv := data.Inventory
+	if inv.Hash != digest || data.Layer.Name != digest {
+		return fmt.Errorf("scan identity does not match candidate")
+	}
+	if len(inv.Packages) == 0 || len(inv.Distributions) == 0 || len(inv.Environments) != len(inv.Packages) {
+		return fmt.Errorf("scan inventory is incomplete")
+	}
+	for id, distro := range inv.Distributions {
+		if distro == nil || distro.ID != id || distro.DID != "alpine" || distro.Version != "3.23" || distro.Name != "Alpine Linux" || distro.PrettyName != "Alpine Linux v3.23" {
+			return fmt.Errorf("distribution coverage is not qualified")
+		}
+	}
+	expected := map[featureKey]int{}
+	for id, pkg := range inv.Packages {
+		if pkg == nil || pkg.ID != id || pkg.Name == "" || pkg.Version == "" || pkg.Kind != "binary" {
+			return fmt.Errorf("package identity is incomplete")
+		}
+		if _, err := apk.NewVersion(pkg.Version); err != nil {
+			return fmt.Errorf("package version is not matchable")
+		}
+		if pkg.Source == nil || pkg.Source.Name == "" || pkg.Source.Kind != "source" {
+			return fmt.Errorf("source package identity is missing")
+		}
+		envs := inv.Environments[id]
+		if len(envs) == 0 {
+			return fmt.Errorf("package environment is missing")
+		}
+		for _, env := range envs {
+			if env == nil || env.PackageDB != "lib/apk/db/installed" || !layers[env.IntroducedIn] || inv.Distributions[env.DistributionID] == nil || len(env.RepositoryIDs) != 0 {
+				return fmt.Errorf("package environment is not qualified")
+			}
+		}
+		expected[featureKey{pkg.Name, pkg.Version, envs[0].IntroducedIn}]++
+	}
+	for _, feature := range *data.Layer.Features {
+		key := featureKey{feature.Name, feature.Version, feature.AddedBy}
+		if expected[key] == 0 {
+			return fmt.Errorf("scan feature does not match inventory")
+		}
+		expected[key]--
+	}
+	for _, count := range expected {
+		if count != 0 {
+			return fmt.Errorf("scan features omit indexed packages")
+		}
+	}
+	return nil
+}

services/mirrored-image/controller/inventory_test.go

@@ -0,0 +1,55 @@
+package main
+
+import "testing"
+
+func inventoryFixture() *securityData {
+	return &securityData{
+		Layer: &securityLayer{Name: testDigest, Features: &[]securityFeature{{Name: "busybox", Version: "1.37.0-r30", AddedBy: testDigest}}},
+		Inventory: &scanInventory{Hash: testDigest,
+			Packages:      map[string]*scanPackage{"p": {ID: "p", Name: "busybox", Version: "1.37.0-r30", Kind: "binary", Source: &scanPackage{Name: "busybox", Kind: "source"}}},
+			Distributions: map[string]*scanDistribution{"d": {ID: "d", DID: "alpine", Version: "3.23", Name: "Alpine Linux", PrettyName: "Alpine Linux v3.23"}},
+			Environments:  map[string][]*scanEnvironment{"p": {{PackageDB: "lib/apk/db/installed", IntroducedIn: testDigest, DistributionID: "d"}}},
+		},
+	}
+}
+
+func TestInventory(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		mutate func(*securityData)
+	}{
+		{"qualified", func(*securityData) {}},
+		{"missing", func(d *securityData) { d.Inventory = nil }},
+		{"identity", func(d *securityData) { d.Inventory.Hash = childDigest }},
+		{"layer identity", func(d *securityData) { d.Layer.Name = childDigest }},
+		{"empty", func(d *securityData) { d.Inventory.Packages = nil }},
+		{"no distribution", func(d *securityData) { d.Inventory.Distributions = nil }},
+		{"Amazon Linux", func(d *securityData) { d.Inventory.Distributions["d"].DID = "amzn" }},
+		{"RHEL", func(d *securityData) { d.Inventory.Distributions["d"].DID = "rhel" }},
+		{"unknown", func(d *securityData) { d.Inventory.Distributions["d"].DID = "unknown" }},
+		{"unqualified release", func(d *securityData) { d.Inventory.Distributions["d"].Version = "3.2" }},
+		{"missing package", func(d *securityData) { d.Inventory.Packages["p"] = nil }},
+		{"invalid version", func(d *securityData) { d.Inventory.Packages["p"].Version = "invalid" }},
+		{"missing source", func(d *securityData) { d.Inventory.Packages["p"].Source = nil }},
+		{"empty source", func(d *securityData) { d.Inventory.Packages["p"].Source.Name = "" }},
+		{"foreign package", func(d *securityData) { d.Inventory.Packages["p"].Kind = "python" }},
+		{"missing environment", func(d *securityData) { d.Inventory.Environments["p"] = nil }},
+		{"foreign layer", func(d *securityData) { d.Inventory.Environments["p"][0].IntroducedIn = childDigest }},
+		{"foreign database", func(d *securityData) { d.Inventory.Environments["p"][0].PackageDB = "var/lib/rpm" }},
+		{"second environment", func(d *securityData) {
+			d.Inventory.Environments["p"] = append(d.Inventory.Environments["p"], &scanEnvironment{PackageDB: "var/lib/rpm"})
+		}},
+		{"missing features", func(d *securityData) { *d.Layer.Features = nil }},
+		{"wrong feature", func(d *securityData) { (*d.Layer.Features)[0].Version = "1-r1" }},
+		{"duplicate feature", func(d *securityData) { *d.Layer.Features = append(*d.Layer.Features, (*d.Layer.Features)[0]) }},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			data := inventoryFixture()
+			test.mutate(data)
+			err := checkInventory(data, testDigest, map[string]bool{testDigest: true})
+			if (err == nil) != (test.name == "qualified") {
+				t.Fatalf("unexpected inventory verdict: %v", err)
+			}
+		})
+	}
+}

services/mirrored-image/controller/main.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/rest"
+)
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+	if err := run(ctx, os.Args[1:]); err != nil && !errors.Is(err, context.Canceled) {
+		slog.Error("image mirror", "error", err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context, arguments []string) error {
+	if len(arguments) != 1 {
+		return errors.New("expected one command: run")
+	}
+	if arguments[0] != "run" {
+		return fmt.Errorf("unknown command %q", arguments[0])
+	}
+	clientConfig, err := rest.InClusterConfig()
+	if err != nil {
+		return fmt.Errorf("load Kubernetes configuration: %w", err)
+	}
+	client, err := dynamic.NewForConfig(clientConfig)
+	if err != nil {
+		return fmt.Errorf("create Kubernetes client: %w", err)
+	}
+	config, err := loadConfig()
+	if err != nil {
+		return err
+	}
+	registry, err := loadRegistry()
+	if err != nil {
+		return err
+	}
+	return (&controller{client: client, registry: registry, config: config}).run(ctx)
+}
+
+func loadConfig() (controllerConfig, error) {
+	namespace := os.Getenv("POD_NAMESPACE")
+	if namespace == "" {
+		return controllerConfig{}, errors.New("POD_NAMESPACE is required")
+	}
+	pollInterval, err := durationEnv("POLL_INTERVAL", 20*time.Second)
+	if err != nil {
+		return controllerConfig{}, err
+	}
+	return controllerConfig{
+		namespace:         namespace,
+		pollInterval:      pollInterval,
+		prepareTemplate:   valueOrDefault("PREPARE_TEMPLATE", "image-prepare"),
+		promotionTemplate: valueOrDefault("PROMOTION_TEMPLATE", "image-promote"),
+		cleanupTemplate:   valueOrDefault("CLEANUP_TEMPLATE", "image-cleanup"),
+	}, nil
+}
+
+func durationEnv(name string, fallback time.Duration) (time.Duration, error) {
+	value := os.Getenv(name)
+	if value == "" {
+		return fallback, nil
+	}
+	duration, err := time.ParseDuration(value)
+	if err != nil || duration <= 0 {
+		return 0, fmt.Errorf("%s must be a positive duration", name)
+	}
+	return duration, nil
+}
+
+func valueOrDefault(name, fallback string) string {
+	if value := os.Getenv(name); value != "" {
+		return value
+	}
+	return fallback
+}

services/mirrored-image/controller/registry.go

@@ -0,0 +1,400 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+)
+
+const oauthUser = "$oauthtoken"
+
+var challengeRule = regexp.MustCompile(`([A-Za-z]+)="([^"]*)"`)
+
+type quayReader struct {
+	api        *url.URL
+	registry   string
+	client     *http.Client
+	candidates string
+	mirror     string
+}
+
+type manifestDoc struct {
+	Manifests json.RawMessage `json:"manifests"`
+	Layers    []descriptor    `json:"layers"`
+}
+
+type descriptor struct {
+	Digest string `json:"digest"`
+}
+
+type securityDoc struct {
+	Status string        `json:"status"`
+	Data   *securityData `json:"data"`
+}
+
+type securityData struct {
+	Layer     *securityLayer `json:"Layer"`
+	Inventory *scanInventory `json:"Inventory"`
+}
+
+type securityLayer struct {
+	Name     string             `json:"Name"`
+	Features *[]securityFeature `json:"Features"`
+}
+
+type securityFeature struct {
+	Name            string          `json:"Name"`
+	Version         string          `json:"Version"`
+	AddedBy         string          `json:"AddedBy"`
+	Vulnerabilities []vulnerability `json:"Vulnerabilities"`
+}
+
+type vulnerability struct {
+	Severity string `json:"Severity"`
+}
+
+func loadRegistry() (*quayReader, error) {
+	registry := os.Getenv("REGISTRY")
+	if registry == "" {
+		return nil, errors.New("REGISTRY is required")
+	}
+	apiValue := os.Getenv("QUAY_API")
+	if apiValue == "" {
+		return nil, errors.New("QUAY_API is required")
+	}
+	api, err := url.Parse(apiValue)
+	if err != nil || api.Host == "" || (api.Scheme != "http" && api.Scheme != "https") {
+		return nil, errors.New("QUAY_API must be an HTTP or HTTPS origin")
+	}
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	if api.Scheme == "https" {
+		roots, err := x509.SystemCertPool()
+		if err != nil {
+			return nil, fmt.Errorf("load system roots: %w", err)
+		}
+		certificate, err := os.ReadFile(valueOrDefault("REGISTRY_CA", "/registry-ca/ca.crt"))
+		if err != nil {
+			return nil, fmt.Errorf("read registry CA: %w", err)
+		}
+		if !roots.AppendCertsFromPEM(certificate) {
+			return nil, errors.New("registry CA contains no certificate")
+		}
+		transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}
+	}
+	candidates, err := readSecret(valueOrDefault("CANDIDATES_TOKEN", "/candidates-reader/token"))
+	if err != nil {
+		return nil, err
+	}
+	mirror, err := readSecret(valueOrDefault("MIRROR_TOKEN", "/mirror-reader/token"))
+	if err != nil {
+		return nil, err
+	}
+	return &quayReader{
+		api:        api,
+		registry:   registry,
+		client:     &http.Client{Timeout: 20 * time.Second, Transport: transport},
+		candidates: candidates,
+		mirror:     mirror,
+	}, nil
+}
+
+func readSecret(path string) (string, error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("read %s: %w", path, err)
+	}
+	value := strings.TrimSpace(string(content))
+	if value == "" {
+		return "", fmt.Errorf("secret %s is empty", path)
+	}
+	return value, nil
+}
+
+func (q *quayReader) scan(ctx context.Context, repository, digest, threshold string) (string, error) {
+	content, found, err := q.manifest(ctx, repository, digest)
+	if err != nil {
+		return "", err
+	}
+	if !found {
+		return scanPending, nil
+	}
+	if fmt.Sprintf("sha256:%x", sha256.Sum256(content)) != digest {
+		return "", errors.New("candidate digest does not match manifest")
+	}
+	digests, err := scanDigests(content, digest)
+	if err != nil {
+		return "", err
+	}
+	limit, err := severityRank(threshold)
+	if err != nil {
+		return "", err
+	}
+	for _, child := range digests {
+		result, found, err := q.security(ctx, repository, child)
+		if err != nil {
+			return "", err
+		}
+		if !found || result.Status == "queued" || result.Status == "scanning" || result.Status == "unsupported" {
+			return scanPending, nil
+		}
+		if result.Status != "scanned" {
+			return "", fmt.Errorf("clair returned unusable status %q for %s", result.Status, child)
+		}
+		if result.Data == nil || result.Data.Layer == nil || result.Data.Layer.Features == nil {
+			return "", fmt.Errorf("clair scan data is unavailable for %s", child)
+		}
+		manifest := content
+		if child != digest {
+			manifest, found, err = q.manifest(ctx, repository, child)
+			if err != nil || !found {
+				return scanPending, err
+			}
+		}
+		if fmt.Sprintf("sha256:%x", sha256.Sum256(manifest)) != child {
+			return "", errors.New("child digest does not match manifest")
+		}
+		var document manifestDoc
+		if err := json.Unmarshal(manifest, &document); err != nil {
+			return "", errors.New("candidate manifest is invalid")
+		}
+		if len(document.Manifests) != 0 || len(document.Layers) == 0 {
+			return scanRejected, nil
+		}
+		layers := map[string]bool{}
+		for _, layer := range document.Layers {
+			if !digestRule.MatchString(layer.Digest) {
+				return "", errors.New("candidate layer digest is invalid")
+			}
+			layers[layer.Digest] = true
+		}
+		if err := checkInventory(result.Data, child, layers); err != nil {
+			return scanRejected, nil
+		}
+		for _, feature := range *result.Data.Layer.Features {
+			for _, finding := range feature.Vulnerabilities {
+				rank, err := severityRank(finding.Severity)
+				if err != nil {
+					return "", err
+				}
+				if rank == 0 || rank >= limit {
+					return scanRejected, nil
+				}
+			}
+		}
+	}
+	return scanAccepted, nil
+}
+
+func scanDigests(content []byte, digest string) ([]string, error) {
+	var manifest manifestDoc
+	if err := json.Unmarshal(content, &manifest); err != nil {
+		return nil, fmt.Errorf("decode candidate manifest: %w", err)
+	}
+	if len(manifest.Manifests) == 0 {
+		return []string{digest}, nil
+	}
+	var children []descriptor
+	if err := json.Unmarshal(manifest.Manifests, &children); err != nil {
+		return nil, fmt.Errorf("decode candidate index: %w", err)
+	}
+	if len(children) == 0 {
+		return nil, errors.New("candidate index contains no manifests")
+	}
+	digests := make([]string, 0, len(children))
+	for _, child := range children {
+		if !digestRule.MatchString(child.Digest) {
+			return nil, fmt.Errorf("candidate index contains invalid digest %q", child.Digest)
+		}
+		digests = append(digests, child.Digest)
+	}
+	return digests, nil
+}
+
+func severityRank(value string) (int, error) {
+	switch strings.ToLower(value) {
+	case "unknown":
+		return 0, nil
+	case "negligible":
+		return 1, nil
+	case "low":
+		return 2, nil
+	case "medium":
+		return 3, nil
+	case "high":
+		return 4, nil
+	case "critical":
+		return 5, nil
+	default:
+		return 0, fmt.Errorf("unknown Clair severity %q", value)
+	}
+}
+
+func (q *quayReader) exists(ctx context.Context, repository, digest string) (bool, error) {
+	_, found, err := q.manifest(ctx, repository, digest)
+	return found, err
+}
+
+func (q *quayReader) physicallyPresent(ctx context.Context, repository, digest string) (bool, error) {
+	response, err := q.apiGet(ctx, repository, digest, "/security?vulnerabilities=false")
+	if err != nil {
+		return false, err
+	}
+	defer func() { _ = response.Body.Close() }()
+	switch response.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusNotFound:
+		return false, nil
+	default:
+		return false, fmt.Errorf("quay physical manifest lookup failed with HTTP %d", response.StatusCode)
+	}
+}
+
+func (q *quayReader) security(ctx context.Context, repository, digest string) (securityDoc, bool, error) {
+	response, err := q.apiGet(ctx, repository, digest, "/security?vulnerabilities=true")
+	if err != nil {
+		return securityDoc{}, false, err
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode == http.StatusNotFound {
+		return securityDoc{}, false, nil
+	}
+	if response.StatusCode != http.StatusOK {
+		return securityDoc{}, false, fmt.Errorf("quay security lookup failed with HTTP %d", response.StatusCode)
+	}
+	var result securityDoc
+	if err := json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&result); err != nil {
+		return securityDoc{}, false, fmt.Errorf("decode Quay security result: %w", err)
+	}
+	return result, true, nil
+}
+
+func (q *quayReader) apiGet(ctx context.Context, repository, digest, suffix string) (*http.Response, error) {
+	endpoint := *q.api
+	endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/api/v1/repository/" + repository + "/manifest/" + digest
+	endpoint.RawQuery = ""
+	if queryIndex := strings.IndexByte(suffix, '?'); queryIndex >= 0 {
+		endpoint.Path += suffix[:queryIndex]
+		endpoint.RawQuery = suffix[queryIndex+1:]
+	} else {
+		endpoint.Path += suffix
+	}
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("Authorization", "Bearer "+q.token(repository))
+	return q.client.Do(request)
+}
+
+func (q *quayReader) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) {
+	endpoint := fmt.Sprintf("%s://%s/v2/%s/manifests/%s", q.api.Scheme, q.registry, repository, digest)
+	send := func(authorization string) (*http.Response, error) {
+		request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		if err != nil {
+			return nil, err
+		}
+		request.Header.Set("Accept", strings.Join([]string{
+			"application/vnd.oci.image.index.v1+json",
+			"application/vnd.oci.image.manifest.v1+json",
+			"application/vnd.docker.distribution.manifest.list.v2+json",
+			"application/vnd.docker.distribution.manifest.v2+json",
+		}, ", "))
+		if authorization != "" {
+			request.Header.Set("Authorization", authorization)
+		}
+		return q.client.Do(request)
+	}
+	response, err := send("")
+	if err != nil {
+		return nil, false, err
+	}
+	if response.StatusCode == http.StatusUnauthorized {
+		challenge := response.Header.Get("WWW-Authenticate")
+		if err := response.Body.Close(); err != nil {
+			return nil, false, fmt.Errorf("close registry challenge: %w", err)
+		}
+		token, err := q.bearerToken(ctx, challenge, repository)
+		if err != nil {
+			return nil, false, err
+		}
+		response, err = send("Bearer " + token)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode == http.StatusNotFound {
+		return nil, false, nil
+	}
+	if response.StatusCode != http.StatusOK {
+		return nil, false, fmt.Errorf("registry manifest lookup failed with HTTP %d", response.StatusCode)
+	}
+	content, err := io.ReadAll(io.LimitReader(response.Body, 8<<20))
+	if err != nil {
+		return nil, false, fmt.Errorf("read candidate manifest: %w", err)
+	}
+	return content, true, nil
+}
+
+func (q *quayReader) bearerToken(ctx context.Context, challenge, repository string) (string, error) {
+	values := make(map[string]string)
+	for _, match := range challengeRule.FindAllStringSubmatch(challenge, -1) {
+		values[strings.ToLower(match[1])] = match[2]
+	}
+	realm, err := url.Parse(values["realm"])
+	if err != nil || realm.Host == "" {
+		return "", errors.New("registry challenge omitted a valid realm")
+	}
+	query := realm.Query()
+	query.Set("service", values["service"])
+	query.Set("scope", values["scope"])
+	query.Set("account", oauthUser)
+	realm.RawQuery = query.Encode()
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, realm.String(), nil)
+	if err != nil {
+		return "", err
+	}
+	request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(oauthUser+":"+q.token(repository))))
+	response, err := q.client.Do(request)
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = response.Body.Close() }()
+	if response.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("registry authentication failed with HTTP %d", response.StatusCode)
+	}
+	var result struct {
+		Token       string `json:"token"`
+		AccessToken string `json:"access_token"`
+	}
+	if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
+		return "", fmt.Errorf("decode registry token: %w", err)
+	}
+	if result.Token != "" {
+		return result.Token, nil
+	}
+	if result.AccessToken != "" {
+		return result.AccessToken, nil
+	}
+	return "", errors.New("registry authentication returned no token")
+}
+
+func (q *quayReader) token(repository string) string {
+	if strings.HasPrefix(repository, "candidates/") {
+		return q.candidates
+	}
+	return q.mirror
+}

services/mirrored-image/controller/registry_test.go

@@ -0,0 +1,163 @@
+package main
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+const childDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+
+func TestPhysicalPresence(t *testing.T) {
+	t.Parallel()
+	for _, status := range []int{200, 404, 401, 403, 429, 500, 502} {
+		t.Run(fmt.Sprint(status), func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+				if request.URL.Path != "/api/v1/repository/candidates/pause/manifest/"+testDigest+"/security" || request.URL.Query().Get("vulnerabilities") != "false" {
+					t.Errorf("unexpected lookup: %s", request.URL)
+				}
+				if request.Header.Get("Authorization") != "Bearer reader-token" {
+					t.Error("lookup did not use reader authority")
+				}
+				response.WriteHeader(status)
+				_, _ = fmt.Fprint(response, `{"status":"queued"}`)
+			}))
+			defer server.Close()
+			api, err := url.Parse(server.URL)
+			if err != nil {
+				t.Fatal(err)
+			}
+			reader := &quayReader{api: api, client: server.Client(), candidates: "reader-token"}
+			present, err := reader.physicallyPresent(context.Background(), "candidates/pause", testDigest)
+			if present != (status == 200) || (err != nil) != (status != 200 && status != 404) {
+				t.Fatalf("HTTP %d: present=%t err=%v", status, present, err)
+			}
+		})
+	}
+}
+
+func TestQuayScanAndExistence(t *testing.T) {
+	t.Parallel()
+	child := fmt.Sprintf(`{"schemaVersion":2,"layers":[{"digest":%q}]}`, testDigest)
+	childDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(child)))
+	index := fmt.Sprintf(`{"manifests":[{"digest":%q}]}`, childDigest)
+	testDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(index)))
+	severity := "Low"
+	var server *httptest.Server
+	server = httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+		switch {
+		case request.URL.Path == "/v2/candidates/pause/manifests/"+childDigest:
+			_, _ = fmt.Fprint(response, child)
+		case request.URL.Path == "/v2/auth":
+			username, password, found := request.BasicAuth()
+			if !found || username != oauthUser || (password != "candidate-token" && password != "mirror-token") {
+				http.Error(response, "unauthorized", http.StatusUnauthorized)
+				return
+			}
+			response.Header().Set("Content-Type", "application/json")
+			if _, err := fmt.Fprintf(response, `{"token":%q}`, password+"-registry"); err != nil {
+				t.Errorf("write token response: %v", err)
+			}
+		case request.URL.Path == "/v2/candidates/pause/manifests/"+testDigest || request.URL.Path == "/v2/mirror/pause/manifests/"+testDigest:
+			expected := "Bearer candidate-token-registry"
+			if strings.Contains(request.URL.Path, "/mirror/") {
+				expected = "Bearer mirror-token-registry"
+			}
+			if request.Header.Get("Authorization") != expected {
+				response.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/v2/auth",service="fixture",scope="repository:test:pull"`)
+				response.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			if strings.Contains(request.URL.Path, "/mirror/") {
+				response.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
+				if _, err := fmt.Fprint(response, `{"schemaVersion":2}`); err != nil {
+					t.Errorf("write manifest response: %v", err)
+				}
+				return
+			}
+			response.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
+			if _, err := fmt.Fprintf(response, `{"manifests":[{"digest":%q}]}`, childDigest); err != nil {
+				t.Errorf("write index response: %v", err)
+			}
+		case strings.HasSuffix(request.URL.Path, "/security"):
+			if request.Header.Get("Authorization") != "Bearer candidate-token" || request.URL.Query().Get("vulnerabilities") != "true" {
+				http.Error(response, "unauthorized", http.StatusUnauthorized)
+				return
+			}
+			data := inventoryFixture()
+			data.Inventory.Hash = childDigest
+			data.Layer.Name = childDigest
+			(*data.Layer.Features)[0].Vulnerabilities = []vulnerability{{Severity: severity}}
+			if err := json.NewEncoder(response).Encode(securityDoc{Status: "scanned", Data: data}); err != nil {
+				t.Errorf("write scan response: %v", err)
+			}
+		default:
+			http.NotFound(response, request)
+		}
+	}))
+	defer server.Close()
+
+	api, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reader := &quayReader{
+		api:        api,
+		registry:   server.Listener.Addr().String(),
+		client:     server.Client(),
+		candidates: "candidate-token",
+		mirror:     "mirror-token",
+	}
+
+	phase, err := reader.scan(context.Background(), "candidates/pause", testDigest, "High")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if phase != scanAccepted {
+		t.Fatalf("unexpected accepted phase %q", phase)
+	}
+	severity = "Critical"
+	phase, err = reader.scan(context.Background(), "candidates/pause", testDigest, "High")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if phase != scanRejected {
+		t.Fatalf("unexpected rejected phase %q", phase)
+	}
+	severity = "Unknown"
+	phase, err = reader.scan(context.Background(), "candidates/pause", testDigest, "Critical")
+	if err != nil || phase != scanRejected {
+		t.Fatalf("unknown severity: %s %v", phase, err)
+	}
+	present, err := reader.exists(context.Background(), "mirror/pause", testDigest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !present {
+		t.Fatal("promoted manifest was not observed")
+	}
+	present, err = reader.exists(context.Background(), "candidates/missing", testDigest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if present {
+		t.Fatal("missing manifest was observed")
+	}
+}
+
+func TestSingleManifestScanDigest(t *testing.T) {
+	t.Parallel()
+	digests, err := scanDigests([]byte(`{"schemaVersion":2}`), testDigest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(digests) != 1 || digests[0] != testDigest {
+		t.Fatalf("unexpected scan digests %#v", digests)
+	}
+}

services/mirrored-image/controller/types.go

@@ -0,0 +1,61 @@
+package main
+
+import "time"
+
+const (
+	retryAnnotation = "mirror.quorate.dev/retry"
+	purposeLabel    = "mirror.quorate.dev/purpose"
+)
+
+type mirroredImage struct {
+	Metadata objectMetadata      `json:"metadata"`
+	Spec     mirroredImageSpec   `json:"spec"`
+	Status   mirroredImageStatus `json:"status,omitempty"`
+}
+
+type objectMetadata struct {
+	Name              string            `json:"name"`
+	Namespace         string            `json:"namespace"`
+	UID               string            `json:"uid"`
+	Generation        int64             `json:"generation"`
+	Annotations       map[string]string `json:"annotations,omitempty"`
+	DeletionTimestamp *time.Time        `json:"deletionTimestamp,omitempty"`
+}
+
+type mirroredImageSpec struct {
+	Upstream string       `json:"upstream"`
+	Policy   scanPolicy   `json:"policy"`
+	Digests  []digestSpec `json:"digests"`
+}
+
+type scanPolicy struct {
+	RejectAtOrAbove string `json:"rejectAtOrAbove"`
+}
+
+type digestSpec struct {
+	Digest string `json:"digest"`
+	Tag    string `json:"tag,omitempty"`
+}
+
+type mirroredImageStatus struct {
+	ObservedGeneration int64          `json:"observedGeneration,omitempty"`
+	Message            string         `json:"message,omitempty"`
+	LastHandledRetry   string         `json:"lastHandledRetry,omitempty"`
+	Digests            []digestStatus `json:"digests,omitempty"`
+}
+
+type digestStatus struct {
+	Digest    string         `json:"digest"`
+	Tag       string         `json:"tag,omitempty"`
+	Prepare   *attemptStatus `json:"prepare,omitempty"`
+	Scan      string         `json:"scan,omitempty"`
+	Promotion *attemptStatus `json:"promotion,omitempty"`
+	Cleanup   *attemptStatus `json:"cleanup,omitempty"`
+}
+
+type attemptStatus struct {
+	WorkflowName string `json:"workflowName"`
+	RetryOf      string `json:"retryOf,omitempty"`
+	Phase        string `json:"phase"`
+	Generation   int64  `json:"generation"`
+}

services/mirrored-image/k8s/bootstrap/access.yaml

@@ -0,0 +1,148 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: mirrored-image
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: mirror-bootstrap
+      app.kubernetes.io/instance: mirrored-image
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - {port: "53", protocol: UDP}
+            - {port: "53", protocol: TCP}
+    - toEntities: [kube-apiserver]
+      toPorts:
+        - ports:
+            - {port: "443", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: vault
+            app.kubernetes.io/name: vault
+            component: server
+      toPorts:
+        - ports:
+            - {port: "8200", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: zitadel
+            app.kubernetes.io/name: zitadel
+      toPorts:
+        - ports:
+            - {port: "8080", protocol: TCP}
+    - toCIDR: [10.108.252.250/32]
+      toPorts:
+        - ports:
+            - {port: "443", protocol: TCP}
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: quay
+            app: quay
+      toPorts:
+        - ports:
+            - {port: "8443", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: vault
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: vault
+      component: server
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "8200", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: zitadel
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: zitadel
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "8080", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: mirror-bootstrap
+  namespace: quay
+spec:
+  endpointSelector:
+    matchLabels:
+      app: quay
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "8443", protocol: TCP}
+---
+apiVersion: cilium.io/v2
+kind: CiliumClusterwideNetworkPolicy
+metadata:
+  name: mirror-boot-replies
+spec:
+  nodeSelector: {}
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: mirrored-image
+            app.kubernetes.io/name: mirror-bootstrap
+            app.kubernetes.io/instance: mirrored-image
+      toPorts:
+        - ports:
+            - {port: "32768", endPort: 60999, protocol: TCP}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mirror-login-key
+  namespace: zitadel
+rules:
+  - apiGroups: [""]
+    resources: [secrets]
+    resourceNames: [zitadel-login-service-key]
+    verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mirror-login-key
+  namespace: zitadel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mirror-login-key
+subjects:
+  - kind: ServiceAccount
+    name: mirror-bootstrap
+    namespace: mirrored-image

services/mirrored-image/k8s/bootstrap/account.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mirror-bootstrap
+  namespace: mirrored-image
+imagePullSecrets:
+  - name: quay-cluster-pull

services/mirrored-image/k8s/bootstrap/bootstrap-lib.fsx

@@ -0,0 +1,93 @@
+module Bootstrap
+
+open System
+
+type App =
+    { Name: string
+      User: string
+      Org: string
+      Scopes: string list
+      Client: string }
+
+type Session = { User: string; State: obj }
+
+type Store =
+    { Read: string -> Map<string, string>
+      Write: string -> Map<string, string> -> unit }
+
+type Quay =
+    { LoginAuth: unit -> string
+      Login: string -> string -> string -> Session
+      Access: string -> App -> unit
+      Application: string -> App -> string
+      Assign: string -> Session -> App -> unit
+      Issue: Session -> App -> string }
+
+type Config = { AdminPath: string }
+
+let apps =
+    [ { Name = "candidates-reader"
+        User = "candidates-bot"
+        Org = "candidates"
+        Scopes = [ "repo:read" ]
+        Client = "" }
+      { Name = "candidates-writer"
+        User = "candidates-bot"
+        Org = "candidates"
+        Scopes = [ "repo:create"; "repo:write" ]
+        Client = "" }
+      { Name = "mirror-reader"
+        User = "mirror-bot"
+        Org = "mirror"
+        Scopes = [ "repo:read" ]
+        Client = "" }
+      { Name = "mirror-writer"
+        User = "mirror-bot"
+        Org = "mirror"
+        Scopes = [ "repo:create"; "repo:write" ]
+        Client = "" } ]
+
+let personaPath user = $"secret/data/quay/personas/{user}"
+let tokenPath app = $"secret/data/quay/oauth/{app.Name}"
+
+let private field name (values: Map<string, string>) =
+    match Map.tryFind name values with
+    | Some value when not (String.IsNullOrWhiteSpace value) && value <> "PLACEHOLDER" -> value
+    | _ -> invalidOp "credential is absent"
+
+let run config store quay =
+    let pending =
+        apps
+        |> List.filter (fun app ->
+            let values = store.Read(tokenPath app)
+
+            match Map.tryFind "token" values with
+            | Some value when not (String.IsNullOrWhiteSpace value) && value <> "PLACEHOLDER" -> false
+            | _ -> true)
+
+    if not pending.IsEmpty then
+        let admin = store.Read config.AdminPath |> field "token"
+
+        let loginAuth = quay.LoginAuth()
+
+        let sessions =
+            pending
+            |> List.map _.User
+            |> List.distinct
+            |> List.map (fun user ->
+                let password = store.Read(personaPath user) |> field "password"
+                let session = quay.Login user password loginAuth
+
+                let app = pending |> List.find (fun app -> app.User = user)
+
+                quay.Access admin app
+                user, session)
+            |> Map.ofList
+
+        for definition in pending do
+            let client = quay.Application admin definition
+            let app = { definition with Client = client }
+            let session = sessions[app.User]
+            quay.Assign admin session app
+            let token = quay.Issue session app
+            store.Write (tokenPath app) (Map.ofList (List.singleton ("token", token)))

services/mirrored-image/k8s/bootstrap/bootstrap-tests.fsx

@@ -0,0 +1,127 @@
+#load "bootstrap-lib.fsx"
+
+open System
+open System.Collections.Generic
+open Bootstrap
+
+let require condition message =
+    if not condition then
+        invalidOp message
+
+let config = { AdminPath = "secret/data/bootstrap/admin" }
+
+let allPending () =
+    let reads = Dictionary<string, Map<string, string>>()
+
+    for app in apps do
+        reads[tokenPath app] <- Map.ofList (List.singleton ("token", "PLACEHOLDER"))
+
+    reads[config.AdminPath] <- Map.ofList (List.singleton ("token", "admin-value"))
+    reads[personaPath "candidates-bot"] <- Map.ofList (List.singleton ("password", "candidates-value"))
+    reads[personaPath "mirror-bot"] <- Map.ofList (List.singleton ("password", "mirror-value"))
+
+    let writes = ResizeArray<string * Map<string, string>>()
+    let logins = ResizeArray<string>()
+    let access = ResizeArray<string>()
+    let assigned = ResizeArray<App>()
+
+    let store =
+        { Read = fun path -> reads[path]
+          Write = fun path value -> writes.Add(path, value) }
+
+    let quay =
+        { LoginAuth = fun () -> "login-value"
+          Login =
+            fun user _ _ ->
+                logins.Add user
+                { User = user; State = null }
+          Access = fun _ app -> access.Add app.User
+          Application = fun _ app -> $"client-{app.Name}"
+          Assign = fun _ _ app -> assigned.Add app
+          Issue = fun _ app -> $"token-{app.Name}" }
+
+    run config store quay
+    require (Set.ofSeq logins = Set.ofList [ "candidates-bot"; "mirror-bot" ]) "personas were not logged in once"
+    require (Set.ofSeq access = Set.ofList [ "candidates-bot"; "mirror-bot" ]) "persona access was not reconciled once"
+    require (assigned.Count = 4) "OAuth assignments were incomplete"
+    require (writes.Count = 4) "OAuth tokens were not written"
+
+    for app in assigned do
+        require (app.Client = $"client-{app.Name}") "application client ID was not retained"
+
+    let writers = assigned |> Seq.filter (fun app -> app.Name.EndsWith "-writer")
+
+    for app in writers do
+        require (app.Scopes = [ "repo:create"; "repo:write" ]) "writer scopes were broader than prescribed"
+
+let complete () =
+    let mutable reads = 0
+    let mutable calls = 0
+
+    let store =
+        { Read =
+            fun _ ->
+                reads <- reads + 1
+                Map.ofList (List.singleton ("token", "existing"))
+          Write = fun _ _ -> calls <- calls + 1 }
+
+    let quay =
+        { LoginAuth =
+            fun () ->
+                calls <- calls + 1
+                ""
+          Login =
+            fun _ _ _ ->
+                calls <- calls + 1
+                { User = ""; State = null }
+          Access = fun _ _ -> calls <- calls + 1
+          Application =
+            fun _ _ ->
+                calls <- calls + 1
+                ""
+          Assign = fun _ _ _ -> calls <- calls + 1
+          Issue =
+            fun _ _ ->
+                calls <- calls + 1
+                "" }
+
+    run config store quay
+    require (reads = 4) "completed outputs were not inspected exactly once"
+    require (calls = 0) "completed bootstrap used privileged operations"
+
+let partial () =
+    let mutable logins = 0
+    let writes = ResizeArray<string>()
+
+    let store =
+        { Read =
+            fun path ->
+                if path = tokenPath apps[0] then
+                    Map.ofList (List.singleton ("token", "PLACEHOLDER"))
+                elif path = config.AdminPath then
+                    Map.ofList (List.singleton ("token", "value"))
+                elif path = personaPath "candidates-bot" then
+                    Map.ofList (List.singleton ("password", "value"))
+                else
+                    Map.ofList (List.singleton ("token", "existing"))
+          Write = fun path _ -> writes.Add path }
+
+    let quay =
+        { LoginAuth = fun () -> "login-token"
+          Login =
+            fun user _ _ ->
+                logins <- logins + 1
+                { User = user; State = null }
+          Access = fun _ _ -> ()
+          Application = fun _ _ -> "client"
+          Assign = fun _ _ _ -> ()
+          Issue = fun _ _ -> "token" }
+
+    run config store quay
+    require (logins = 1) "partial bootstrap logged in an unrelated persona"
+    require (Seq.toList writes = [ tokenPath apps[0] ]) "partial bootstrap changed unrelated outputs"
+
+allPending ()
+complete ()
+partial ()
+printfn "bootstrap tests passed"