ce7cb702218693d08ea6d83464d2457c9250027c

services/ca-sync/k8s/job.yaml

@@ -0,0 +1,88 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: ca-sync
+  namespace: vault
+spec:
+  schedule: "*/15 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 120
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: ca-sync
+            app.kubernetes.io/instance: ca-sync
+        spec:
+          restartPolicy: Never
+          serviceAccountName: ca-sync
+          imagePullSecrets:
+            - name: ca-sync-pull
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: sync
+              image: registry.quorate.internal/quorate/fsharp-runner@sha256:d7b904f4f0cae9fdba7c7345cdf08c91162245834a8a13658d93396272aeec83
+              command: [/usr/share/dotnet/dotnet, /opt/fsi/fsi.dll, --exec]
+              args:
+                - /source/run.fsx
+                - pki=ca/internal
+                - pki_postgres=ca/postgres
+                - pki_opnsense=ca/opnsense
+              env:
+                - name: VAULT_ADDR
+                  value: https://vault.vault.svc.cluster.local:8200
+                - name: VAULT_ROLE
+                  value: ca-sync
+                - name: VAULT_MOUNT
+                  value: secret
+                - name: JWT_PATH
+                  value: /var/run/secrets/kubernetes.io/serviceaccount/token
+                - name: HOME
+                  value: /tmp
+                - name: SSL_CERT_FILE
+                  value: /ca/ca.crt
+                - name: DOTNET_NOLOGO
+                  value: "true"
+                - name: DOTNET_CLI_TELEMETRY_OPTOUT
+                  value: "1"
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 128Mi
+                limits:
+                  memory: 512Mi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: [ALL]
+              volumeMounts:
+                - name: source
+                  mountPath: /source
+                  readOnly: true
+                - name: ca
+                  mountPath: /ca
+                  readOnly: true
+                - name: tmp
+                  mountPath: /tmp
+          volumes:
+            - name: source
+              configMap:
+                name: ca-sync
+            - name: ca
+              secret:
+                secretName: ca-sync-root
+                items:
+                  - key: ca.crt
+                    path: ca.crt
+            - name: tmp
+              emptyDir: {}

services/ca-sync/k8s/policy.yaml

@@ -0,0 +1,50 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ca-sync
+  namespace: vault
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: ca-sync
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+            - port: "53"
+              protocol: TCP
+          rules:
+            dns:
+              - matchName: vault.vault.svc.cluster.local
+    - toEndpoints:
+        - matchLabels:
+            app.kubernetes.io/name: vault
+            component: server
+      toPorts:
+        - ports:
+            - port: "8200"
+              protocol: TCP
+---
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ca-sync-vault
+  namespace: vault
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: vault
+      component: server
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            app.kubernetes.io/name: ca-sync
+      toPorts:
+        - ports:
+            - port: "8200"
+              protocol: TCP

services/ca-sync/k8s/vault.yaml

@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ca-sync
+  namespace: vault
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ca-sync-vso
+  namespace: vault
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: ca-sync
+  namespace: vault
+spec:
+  method: kubernetes
+  mount: kubernetes
+  kubernetes:
+    role: ca-sync-pull
+    serviceAccount: ca-sync-vso
+  vaultConnectionRef: vault/vault
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: ca-sync-pull
+  namespace: vault
+spec:
+  type: kv-v2
+  mount: secret
+  path: quay/robots/cluster_pull
+  refreshAfter: 1h
+  vaultAuthRef: ca-sync
+  destination:
+    name: ca-sync-pull
+    create: true
+    overwrite: true
+    type: kubernetes.io/dockerconfigjson
+    transformation:
+      excludeRaw: true
+      excludes: [.*]
+      templates:
+        .dockerconfigjson:
+          text: '{"auths":{"registry.quorate.internal":{"auth":"{{ printf "%s:%s" (get .Secrets "username") (get .Secrets "token") | b64enc }}"}}}'
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultDynamicSecret
+metadata:
+  name: ca-sync-root
+  namespace: vault
+spec:
+  vaultAuthRef: ca-sync
+  mount: pki
+  path: cert/ca
+  requestHTTPMethod: GET
+  refreshAfter: 1h
+  destination:
+    create: true
+    overwrite: true
+    name: ca-sync-root
+    transformation:
+      excludeRaw: true
+      excludes: [.*]
+      templates:
+        ca.crt:
+          text: '{{ get .Secrets "certificate" }}'

services/ca-sync/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - k8s/job.yaml
+  - k8s/vault.yaml
+  - k8s/policy.yaml
+configMapGenerator:
+  - name: ca-sync
+    namespace: vault
+    files:
+      - sync.fsx
+      - run.fsx

services/ca-sync/run.fsx

@@ -0,0 +1,49 @@
+#load "sync.fsx"
+
+open System
+open System.IO
+open System.Net.Http
+open Sync
+
+let run () =
+    match settings Environment.GetEnvironmentVariable (fsi.CommandLineArgs |> Array.skip 1) with
+    | Error error ->
+        eprintfn "%s" error
+        1
+    | Ok config ->
+        use client =
+            new HttpClient(BaseAddress = config.Vault, Timeout = TimeSpan.FromSeconds 10.)
+
+        let jwt =
+            try
+                Ok(File.ReadAllText(config.TokenFile).Trim())
+            with
+            | :? IOException
+            | :? UnauthorizedAccessException -> Error "Cannot read JWT_PATH"
+
+        match jwt |> Result.bind (reconcile config client) with
+        | Error error ->
+            eprintfn "%s" error
+            1
+        | Ok results ->
+            results
+            |> List.iter (fun result ->
+                match result.Outcome with
+                | Ok Unchanged -> printfn "unchanged %s" result.Source.Kv
+                | Ok(Publish _) -> printfn "updated %s" result.Source.Kv
+                | Error error -> eprintfn "%s: %s" result.Source.Kv error)
+
+            let updated, failed =
+                results
+                |> List.fold
+                    (fun (updated, failed) result ->
+                        match result.Outcome with
+                        | Ok Unchanged -> updated, failed
+                        | Ok(Publish _) -> updated + 1, failed
+                        | Error _ -> updated, failed + 1)
+                    (0, 0)
+
+            printfn "checked=%d updated=%d failed=%d" results.Length updated failed
+            if failed = 0 then 0 else 1
+
+exit (run ())

services/ca-sync/sync.fsx

@@ -0,0 +1,147 @@
+module Sync
+
+open System
+open System.Net.Http
+open System.Text
+open System.Text.Json
+
+type Source = { Pki: string; Kv: string }
+
+type Settings =
+    { Vault: Uri
+      Role: string
+      TokenFile: string
+      Mount: string
+      Sources: Source list }
+
+type Observation =
+    { Certificate: string
+      Current: string
+      Version: int }
+
+type Decision =
+    | Unchanged
+    | Publish of version: int * certificate: string
+
+type CopyResult =
+    { Source: Source
+      Outcome: Result<Decision, string> }
+
+let private (>>=) value next = Result.bind next value
+
+let sources (values: string array) =
+    let parse (value: string) =
+        match value.Split('=') |> Array.map _.Trim() with
+        | [| pki; kv |] when pki <> "" && kv <> "" -> Ok { Pki = pki; Kv = kv }
+        | _ -> Error "Expected each mapping as PKI=KV"
+
+    let add result value =
+        result
+        >>= fun parsed ->
+            parse value
+            >>= fun source ->
+                if parsed |> List.exists (fun entry -> entry.Kv = source.Kv) then
+                    Error $"Duplicate KV destination: {source.Kv}"
+                else
+                    Ok(source :: parsed)
+
+    if Array.isEmpty values then
+        Error "At least one PKI=KV mapping is required"
+    else
+        values |> Array.fold add (Ok []) |> Result.map List.rev
+
+let settings lookup values =
+    let required name =
+        match lookup name with
+        | value when String.IsNullOrWhiteSpace value -> Error $"Missing environment variable: {name}"
+        | value -> Ok(value.Trim())
+
+    sources values
+    >>= fun mappings ->
+        required "VAULT_ADDR"
+        >>= fun address ->
+            match Uri.TryCreate(address.TrimEnd('/') + "/v1/", UriKind.Absolute) with
+            | true, uri when uri.Scheme = "https" || uri.Scheme = "http" ->
+                required "VAULT_ROLE"
+                >>= fun role ->
+                    required "JWT_PATH"
+                    >>= fun tokenFile ->
+                        required "VAULT_MOUNT"
+                        |> Result.map (fun mount ->
+                            { Vault = uri
+                              Role = role
+                              TokenFile = tokenFile
+                              Mount = mount
+                              Sources = mappings })
+            | _ -> Error "VAULT_ADDR must be an absolute HTTP(S) URL"
+
+let plan observation =
+    if observation.Certificate = observation.Current then
+        Unchanged
+    else
+        Publish(observation.Version, observation.Certificate)
+
+let private send (client: HttpClient) (token: string option) (method: HttpMethod) (path: string) payload decode =
+    let error detail =
+        Error $"{method.Method} {path}: {detail}"
+
+    try
+        use request = new HttpRequestMessage(method, path)
+        token |> Option.iter (fun value -> request.Headers.Add("X-Vault-Token", value))
+
+        payload
+        |> Option.iter (fun (value: obj) ->
+            request.Content <- new StringContent(JsonSerializer.Serialize value, Encoding.UTF8, "application/json"))
+
+        use response = client.Send request
+
+        if not response.IsSuccessStatusCode then
+            error $"HTTP {int response.StatusCode}"
+        else
+            use body = JsonDocument.Parse(response.Content.ReadAsStream())
+            Ok(decode body.RootElement)
+    with
+    | :? HttpRequestException -> error "transport failure"
+    | :? OperationCanceledException -> error "request timed out"
+    | :? JsonException -> error "invalid JSON response"
+    | :? Collections.Generic.KeyNotFoundException -> error "missing response field"
+    | :? InvalidOperationException -> error "invalid response field"
+
+let private observe settings client token source =
+    // Reading KV before PKI makes an intervening publication invalidate CAS.
+    send client token HttpMethod.Get (settings.Mount + "/data/" + source.Kv) None (fun body ->
+        let data = body.GetProperty "data"
+
+        data.GetProperty("data").GetProperty("certificate").GetString(),
+        data.GetProperty("metadata").GetProperty("version").GetInt32())
+    >>= fun (current, version) ->
+        send client token HttpMethod.Get (source.Pki + "/cert/ca") None (fun body ->
+            { Certificate = body.GetProperty("data").GetProperty("certificate").GetString()
+              Current = current
+              Version = version })
+
+let private publish settings client token source decision =
+    match decision with
+    | Unchanged -> Ok Unchanged
+    | Publish(version, certificate) ->
+        send
+            client
+            token
+            HttpMethod.Post
+            (settings.Mount + "/data/" + source.Kv)
+            (Some
+                {| options = {| cas = version |}
+                   data = {| certificate = certificate |} |})
+            (fun _ -> decision)
+
+let reconcile settings client jwt =
+    send client None HttpMethod.Post "auth/kubernetes/login" (Some {| role = settings.Role; jwt = jwt |}) (fun body ->
+        body.GetProperty("auth").GetProperty("client_token").GetString())
+    |> Result.map (fun token ->
+        settings.Sources
+        |> List.map (fun source ->
+            { Source = source
+              Outcome =
+                observe settings client (Some token) source
+                |> Result.map plan
+                >>= publish settings client (Some token) source }))

services/ca-sync/tests.fsx

@@ -0,0 +1,264 @@
+#load "sync.fsx"
+
+open System
+open System.Collections.Generic
+open System.Net
+open System.Net.Http
+open System.Text
+open System.Text.Json
+open System.Threading.Tasks
+open Sync
+
+let unwrap =
+    function
+    | Ok value -> value
+    | Error error -> failwith error
+
+let config =
+    { Vault = Uri "https://synthetic.invalid/v1/"
+      Role = "ca-sync"
+      TokenFile = "synthetic-token"
+      Mount = "secret"
+      Sources =
+        sources [| "pki=ca/internal"; "pki_postgres=ca/postgres"; "pki_opnsense=ca/opnsense" |]
+        |> unwrap }
+
+
+type Fixture() =
+    inherit HttpMessageHandler()
+    let source = Dictionary<string, string>()
+    let stored = Dictionary<string, string>()
+    let versions = Dictionary<string, int>()
+    let requests = ResizeArray<string>()
+    let mutable reject = None
+    let mutable jwt = "synthetic-jwt"
+    let mutable afterRead: (string -> unit) = ignore
+    let mutable afterSource: (string -> unit) = ignore
+
+    do
+        for value in config.Sources do
+            source.Add(value.Pki, "synthetic-" + value.Pki)
+            stored.Add(value.Kv, "PLACEHOLDER")
+            versions.Add(value.Kv, 1)
+
+    member _.AfterRead
+        with set value = afterRead <- value
+
+    member _.AfterSource
+        with set value = afterSource <- value
+
+    member _.Source = source
+    member _.Stored = stored
+    member _.Versions = versions
+    member _.Requests = requests
+
+    member _.Reject
+        with get () = reject
+        and set value = reject <- value
+
+    member _.Jwt
+        with set value = jwt <- value
+
+    member private _.Handle(request: HttpRequestMessage) =
+        let path = request.RequestUri.AbsolutePath.Substring("/v1/".Length)
+        let key = request.Method.Method + " " + path
+        requests.Add key
+
+        let response status payload =
+            new HttpResponseMessage(
+                status,
+                Content = new StringContent(JsonSerializer.Serialize payload, Encoding.UTF8, "application/json")
+            )
+
+        if reject = Some key then
+            response HttpStatusCode.ServiceUnavailable {| errors = [| "synthetic outage" |] |}
+        elif path = "auth/kubernetes/login" then
+            use body =
+                JsonDocument.Parse(request.Content.ReadAsStringAsync().GetAwaiter().GetResult())
+
+            if
+                request.Method <> HttpMethod.Post
+                || body.RootElement.GetProperty("role").GetString() <> config.Role
+                || body.RootElement.GetProperty("jwt").GetString() <> jwt
+            then
+                response HttpStatusCode.Forbidden {| errors = [| "synthetic authentication denied" |] |}
+            else
+                response HttpStatusCode.OK {| auth = {| client_token = "synthetic-vault-token" |} |}
+        elif
+            not (request.Headers.Contains "X-Vault-Token")
+            || request.Headers.GetValues("X-Vault-Token") |> Seq.exactlyOne
+               <> "synthetic-vault-token"
+        then
+            response HttpStatusCode.Forbidden {| errors = [| "synthetic authorization denied" |] |}
+        elif path.EndsWith "/cert/ca" then
+            let mount = path.Substring(0, path.Length - "/cert/ca".Length)
+
+            let snapshot =
+                response HttpStatusCode.OK {| data = {| certificate = source[mount] |} |}
+
+            afterSource mount
+            snapshot
+        elif path.StartsWith "secret/data/ca/" then
+            let name = path.Substring("secret/data/".Length)
+
+            if request.Method = HttpMethod.Get then
+                let snapshot =
+                    response
+                        HttpStatusCode.OK
+                        {| data =
+                            {| data = {| certificate = stored[name] |}
+                               metadata = {| version = versions[name] |} |} |}
+
+                afterRead name
+                snapshot
+            else
+                use body =
+                    JsonDocument.Parse(request.Content.ReadAsStringAsync().GetAwaiter().GetResult())
+
+                let expected = body.RootElement.GetProperty("options").GetProperty("cas").GetInt32()
+
+                if expected <> versions[name] then
+                    response HttpStatusCode.BadRequest {| errors = [| "CAS mismatch" |] |}
+                else
+                    stored[name] <- body.RootElement.GetProperty("data").GetProperty("certificate").GetString()
+                    versions[name] <- versions[name] + 1
+                    response HttpStatusCode.OK {| data = {| version = versions[name] |} |}
+        else
+            failwith $"unexpected request: {key}"
+
+    override self.Send(request, _) = self.Handle request
+    override self.SendAsync(request, _) = Task.FromResult(self.Handle request)
+
+let check condition message =
+    if not condition then
+        failwith message
+
+let changed results =
+    results
+    |> List.sumBy (fun result ->
+        match result.Outcome with
+        | Ok(Publish _) -> 1
+        | Ok Unchanged
+        | Error _ -> 0)
+
+let errors results =
+    results
+    |> List.choose (fun result ->
+        match result.Outcome with
+        | Error error -> Some error
+        | Ok _ -> None)
+
+let run () =
+    for values in
+        [| [||]
+           [| "pki" |]
+           [| "=ca/internal" |]
+           [| "pki=" |]
+           [| "pki=ca/x"; "other=ca/x" |] |] do
+        check (sources values |> Result.isError) "invalid mappings were accepted"
+
+    check ((sources [| " pki = ca/internal " |] |> unwrap)[0] = config.Sources[0]) "mapping whitespace"
+    check (settings (fun _ -> null) [| "pki=ca/x" |] |> Result.isError) "missing configuration was accepted"
+
+    let env =
+        Map
+            [ "VAULT_ADDR", "https://synthetic.invalid"
+              "VAULT_ROLE", "ca-sync"
+              "JWT_PATH", "synthetic-token"
+              "VAULT_MOUNT", "secret" ]
+
+    check (settings (fun name -> env[name]) [| "pki=ca/x" |] |> Result.isOk) "valid configuration was rejected"
+
+    check
+        (plan
+            { Certificate = "same"
+              Current = "same"
+              Version = 9 } = Unchanged)
+        "no-op plan"
+
+    check
+        (plan
+            { Certificate = "new"
+              Current = "old"
+              Version = 9 } = Publish(9, "new"))
+        "CAS plan"
+
+    use fixture = new Fixture()
+    use client = new HttpClient(fixture, BaseAddress = config.Vault)
+    let sync jwt = reconcile config client jwt
+    check (sync "synthetic-jwt" |> unwrap |> changed = 3) "initial publication"
+    check (sync "synthetic-jwt" |> unwrap |> changed = 0) "unchanged publication"
+    check (fixture.Versions.Values |> Seq.forall ((=) 2)) "no-op changed KV versions"
+    fixture.Source["pki_postgres"] <- "synthetic-new-postgres"
+    check (sync "synthetic-jwt" |> unwrap |> changed = 1) "selective CA rotation"
+
+    check
+        (fixture.Versions["ca/internal"] = 2
+         && fixture.Versions["ca/postgres"] = 3
+         && fixture.Versions["ca/opnsense"] = 2)
+        "unrelated versions changed"
+
+    check (sync "synthetic-jwt" |> unwrap |> changed = 0) "rotation is not idempotent"
+
+    let snapshot () =
+        fixture.Versions |> Seq.map (fun pair -> pair.Key, pair.Value) |> Map.ofSeq
+
+    let before = snapshot ()
+    check (sync "synthetic-wrong-jwt" = Error "POST auth/kubernetes/login: HTTP 403") "unsafe or opaque login error"
+    check (snapshot () = before) "failed login wrote KV"
+    fixture.Reject <- Some "GET pki_postgres/cert/ca"
+    fixture.Source["pki_opnsense"] <- "synthetic-new-opnsense"
+    let independent = sync "synthetic-jwt" |> unwrap
+    check (errors independent = [ "GET pki_postgres/cert/ca: HTTP 503" ]) "source error context"
+
+    check
+        (changed independent = 1
+         && fixture.Stored["ca/opnsense"] = "synthetic-new-opnsense")
+        "one failed CA blocked another"
+
+    check (fixture.Versions["ca/postgres"] = before["ca/postgres"]) "failed source changed KV"
+
+    for request in [| "GET secret/data/ca/internal"; "POST secret/data/ca/internal" |] do
+        fixture.Source["pki"] <- "synthetic-new-internal"
+        fixture.Reject <- Some request
+        let before = snapshot ()
+        check (sync "synthetic-jwt" |> unwrap |> errors = [ request + ": HTTP 503" ]) "KV error context"
+        check (snapshot () = before) "failed KV access wrote data"
+
+    fixture.Reject <- None
+    fixture.Jwt <- "synthetic-new-jwt"
+    check (sync "synthetic-new-jwt" |> unwrap |> changed = 1) "fresh login recovery"
+    check (sync "synthetic-new-jwt" |> unwrap |> changed = 0) "recovery no-op"
+
+    fixture.Source["pki"] <- "synthetic-stale"
+
+    fixture.AfterSource <-
+        fun name ->
+            if name = "pki" then
+                fixture.AfterSource <- ignore
+                fixture.Source["pki"] <- "synthetic-latest"
+                check (sync "synthetic-new-jwt" |> unwrap |> changed = 1) "newer poll did not publish"
+
+    let overlapping = sync "synthetic-new-jwt" |> unwrap
+    check (errors overlapping = [ "POST secret/data/ca/internal: HTTP 400" ]) "stale write was accepted"
+    check (fixture.Stored["ca/internal"] = "synthetic-latest") "old poll overwrote new certificate"
+    let before = snapshot ()
+    check (sync "synthetic-new-jwt" |> unwrap |> changed = 0 && snapshot () = before) "CAS recovery is not idempotent"
+
+    fixture.AfterRead <-
+        fun name ->
+            if name = "ca/internal" then
+                fixture.AfterRead <- ignore
+                fixture.Source["pki"] <- "synthetic-after-kv"
+                check (sync "synthetic-new-jwt" |> unwrap |> changed = 1) "overlap after KV read"
+
+    check
+        (sync "synthetic-new-jwt" |> unwrap |> errors = [ "POST secret/data/ca/internal: HTTP 400" ])
+        "KV snapshot overlap accepted"
+
+    check (fixture.Stored["ca/internal"] = "synthetic-after-kv") "KV snapshot overlap regressed"
+
+    printfn
+        "PASS: configuration, pure plans, no-op versions, isolated failures, safe diagnostics, fresh login and overlapping CAS writes"
+
+run ()