ce7cb702218693d08ea6d83464d2457c9250027c

services/bot-redirect/bots.fsx

@@ -0,0 +1,95 @@
+open System
+open System.Text.Json
+open System.Text.RegularExpressions
+
+type Bot =
+    { Slug: string
+      Patterns: string list
+      Examples: string list
+      Signer: string option }
+
+type Header =
+    { Name: string
+      Kind: string
+      Value: string }
+
+let strings (value: JsonElement) name =
+    value.GetProperty(name: string).EnumerateArray()
+    |> Seq.map _.GetString()
+    |> Seq.toList
+
+let decode (value: JsonElement) =
+    { Slug = value.GetProperty("slug").GetString()
+      Patterns = strings value "userAgentPatterns"
+      Examples = strings value "userAgents"
+      Signer =
+        match value.TryGetProperty "signatureAgentUrl" with
+        | true, url when url.ValueKind = JsonValueKind.String -> Some(url.GetString())
+        | _ -> None }
+
+let headers excluded bot =
+    let patterns = bot.Patterns |> List.map (fun value -> value.Trim(' ', '\t'))
+
+    [ for pattern in patterns do
+          { Name = "User-Agent"
+            Kind = "RegularExpression"
+            Value = ".*" + Regex.Escape(pattern) + ".*" }
+      for example in bot.Examples do
+          if
+              not (Set.contains example excluded)
+              && not (patterns |> List.exists example.Contains)
+          then
+              { Name = "User-Agent"
+                Kind = "Exact"
+                Value = example.Trim(' ', '\t') }
+      match bot.Signer with
+      | Some signer ->
+          { Name = "Signature-Agent"
+            Kind = "Exact"
+            Value = JsonSerializer.Serialize signer }
+      | None -> () ]
+
+let routes config matches =
+    let text name =
+        (config: JsonElement).GetProperty(name: string).GetString()
+
+    matches
+    |> List.distinct
+    |> List.chunkBySize 128
+    |> List.mapi (fun index chunk ->
+        {| apiVersion = "gateway.networking.k8s.io/v1"
+           kind = "HTTPRoute"
+           metadata =
+            {| name = $"bot-redirect-{index}"
+               ``namespace`` = text "namespace"
+               labels = Map [ ("app.kubernetes.io/managed-by", "bot-redirect") ] |}
+           spec =
+            {| parentRefs =
+                [| {| name = text "gateway"
+                      ``namespace`` = text "gatewayNamespace"
+                      sectionName = "https" |} |]
+               hostnames = [| text "source" |]
+               rules =
+                chunk
+                |> List.chunkBySize 64
+                |> List.map (fun group ->
+                    {| matches =
+                        group
+                        |> List.map (fun header ->
+                            {| headers =
+                                [| {| name = header.Name
+                                      ``type`` = header.Kind
+                                      value = header.Value |} |] |})
+                       filters =
+                        [| box
+                               {| ``type`` = "RequestRedirect"
+                                  requestRedirect =
+                                   {| hostname = text "target"
+                                      port = 443
+                                      statusCode = 301 |} |}
+                           box
+                               {| ``type`` = "ResponseHeaderModifier"
+                                  responseHeaderModifier =
+                                   {| set =
+                                       [| {| name = "X-Robots-Tag"
+                                             value = "noindex" |} |] |} |} |] |}) |} |})

services/bot-redirect/k8s/network-policy.yaml

@@ -0,0 +1,32 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: bot-redirect
+  namespace: argo
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: bot-redirect
+  egress:
+    - toEntities: [kube-apiserver]
+      toPorts:
+        - ports:
+            - { port: "6443", protocol: TCP }
+            - { port: "443", protocol: TCP }
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - { port: "53", protocol: UDP }
+            - { port: "53", protocol: TCP }
+          rules:
+            dns:
+              - matchName: api.cloudflare.com
+              - matchName: kubernetes.default.svc.cluster.local
+    - toFQDNs:
+        - matchName: api.cloudflare.com
+      toPorts:
+        - ports:
+            - { port: "443", protocol: TCP }

services/bot-redirect/k8s/rbac.yaml

@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: bot-redirect
+  namespace: argo
+imagePullSecrets:
+  - name: bot-redirect-pull
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: bot-redirect-vso
+  namespace: argo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: bot-redirect
+  namespace: forgejo-public
+rules:
+  - apiGroups: [gateway.networking.k8s.io]
+    resources: [httproutes]
+    verbs: [get, list, patch, create, delete]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: bot-redirect
+  namespace: forgejo-public
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: bot-redirect
+subjects:
+  - kind: ServiceAccount
+    name: bot-redirect
+    namespace: argo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: bot-redirect
+  namespace: argo
+rules:
+  - apiGroups: [argoproj.io]
+    resources: [workflowtaskresults]
+    verbs: [create, patch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: bot-redirect
+  namespace: argo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: bot-redirect
+subjects:
+  - kind: ServiceAccount
+    name: bot-redirect
+    namespace: argo

services/bot-redirect/k8s/vso.yaml

@@ -0,0 +1,58 @@
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultAuth
+metadata:
+  name: bot-redirect
+  namespace: argo
+spec:
+  method: kubernetes
+  mount: kubernetes
+  kubernetes:
+    role: bot-redirect
+    serviceAccount: bot-redirect-vso
+  vaultConnectionRef: vault/vault
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: bot-redirect
+  namespace: argo
+spec:
+  type: kv-v2
+  mount: secret
+  path: cloudflare/radar
+  refreshAfter: 1h
+  vaultAuthRef: bot-redirect
+  destination:
+    name: bot-redirect
+    create: true
+    overwrite: true
+    transformation:
+      excludeRaw: true
+      excludes: [.*]
+      templates:
+        token:
+          text: '{{ get .Secrets "api-token" }}'
+---
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: bot-redirect-pull
+  namespace: argo
+spec:
+  type: kv-v2
+  mount: secret
+  path: quay/robots/cluster_pull
+  refreshAfter: 1h
+  vaultAuthRef: bot-redirect
+  destination:
+    name: bot-redirect-pull
+    create: true
+    overwrite: true
+    type: kubernetes.io/dockerconfigjson
+    transformation:
+      excludeRaw: true
+      excludes: [.*]
+      templates:
+        .dockerconfigjson:
+          text: >-
+            {"auths":{"registry.quorate.internal":{"auth":"{{ printf "%s:%s" (get .Secrets "username") (get .Secrets "token") | b64enc }}"}}}

services/bot-redirect/k8s/workflow.yaml

@@ -0,0 +1,47 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: bot-redirect
+  namespace: argo
+spec:
+  entrypoint: reconcile
+  serviceAccountName: bot-redirect
+  podMetadata:
+    labels:
+      app.kubernetes.io/name: bot-redirect
+  volumes:
+    - name: source
+      configMap:
+        name: bot-redirect
+    - name: radar
+      secret:
+        secretName: bot-redirect
+  templates:
+    - name: reconcile
+      container:
+        image: registry.quorate.internal/quorate/fsharp-runner@sha256:d7b904f4f0cae9fdba7c7345cdf08c91162245834a8a13658d93396272aeec83
+        command: [/usr/share/dotnet/dotnet, /opt/fsi/fsi.dll, --exec]
+        args: [/source/run.fsx]
+        env:
+          - name: SSL_CERT_FILE
+            value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        volumeMounts:
+          - name: source
+            mountPath: /source
+            readOnly: true
+          - name: radar
+            mountPath: /radar
+            readOnly: true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: bot-redirect
+  namespace: argo
+spec:
+  schedules: ["17 18 * * *"]
+  timezone: UTC
+  concurrencyPolicy: Forbid
+  workflowSpec:
+    workflowTemplateRef:
+      name: bot-redirect

services/bot-redirect/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - k8s/rbac.yaml
+  - k8s/vso.yaml
+  - k8s/workflow.yaml
+  - k8s/network-policy.yaml
+configMapGenerator:
+  - name: bot-redirect
+    namespace: argo
+    files:
+      - bots.fsx
+      - run.fsx
+      - settings.json
+generatorOptions:
+  disableNameSuffixHash: true

services/bot-redirect/run.fsx

@@ -0,0 +1,97 @@
+#load "bots.fsx"
+
+open System
+open System.IO
+open System.Net.Http
+open System.Net.Http.Headers
+open System.Text
+open System.Text.Json
+open Bots
+
+[<TailCall>]
+let rec pages fetch offset collected =
+    let page: JsonElement =
+        fetch $"bots?botVerificationStatus=VERIFIED&limit=100&offset={offset}"
+
+    let slugs =
+        page.GetProperty("bots").EnumerateArray()
+        |> Seq.map (fun bot -> bot.GetProperty("slug").GetString())
+        |> Seq.toList
+
+    if slugs.Length = 100 then
+        pages fetch (offset + slugs.Length) (slugs :: collected)
+    else
+        slugs :: collected |> List.rev |> List.concat
+
+let run () =
+    use settings =
+        JsonDocument.Parse(File.ReadAllText(Path.Combine(__SOURCE_DIRECTORY__, "settings.json")))
+
+    let config = settings.RootElement
+    let excluded = strings config "excluded" |> Set.ofList
+    use radar = new HttpClient()
+    radar.BaseAddress <- Uri(config.GetProperty("radar").GetString())
+
+    radar.DefaultRequestHeaders.Authorization <-
+        AuthenticationHeaderValue("Bearer", File.ReadAllText("/radar/token").Trim())
+
+    let fetch (path: string) =
+        use request = new HttpRequestMessage(HttpMethod.Get, path)
+        use response = radar.Send request
+        response.EnsureSuccessStatusCode() |> ignore
+        use body = JsonDocument.Parse(response.Content.ReadAsStream())
+
+        if not (body.RootElement.GetProperty("success").GetBoolean()) then
+            failwith $"Radar request failed: {path}"
+
+        body.RootElement.GetProperty("result").Clone()
+
+    let bots =
+        pages fetch 0 []
+        |> List.map (fun slug -> (fetch ("bots/" + slug)).GetProperty("bot") |> decode)
+
+    let desired = bots |> List.collect (headers excluded) |> routes config
+    let ns = config.GetProperty("namespace").GetString()
+    let endpoint = $"/apis/gateway.networking.k8s.io/v1/namespaces/{ns}/httproutes"
+    use kube = new HttpClient()
+    kube.BaseAddress <- Uri "https://kubernetes.default.svc"
+
+    kube.DefaultRequestHeaders.Authorization <-
+        AuthenticationHeaderValue(
+            "Bearer",
+            File.ReadAllText("/var/run/secrets/kubernetes.io/serviceaccount/token").Trim()
+        )
+
+    let send method (path: string) payload =
+        use request = new HttpRequestMessage(method, path)
+
+        payload
+        |> Option.iter (fun value ->
+            request.Content <- new StringContent(value, Encoding.UTF8, "application/apply-patch+yaml"))
+
+        use response = kube.Send request
+        response.EnsureSuccessStatusCode() |> ignore
+        use body = JsonDocument.Parse(response.Content.ReadAsStream())
+        body.RootElement.Clone()
+
+    let names = desired |> List.map (fun route -> route.metadata.name) |> Set.ofList
+
+    desired
+    |> List.iter (fun route ->
+        send
+            HttpMethod.Patch
+            (endpoint + "/" + route.metadata.name + "?fieldManager=bot-redirect")
+            (Some(JsonSerializer.Serialize route))
+        |> ignore)
+
+    let existing =
+        send HttpMethod.Get (endpoint + "?labelSelector=app.kubernetes.io%2Fmanaged-by%3Dbot-redirect") None
+
+    existing.GetProperty("items").EnumerateArray()
+    |> Seq.map (fun route -> route.GetProperty("metadata").GetProperty("name").GetString())
+    |> Seq.filter (fun name -> not (Set.contains name names))
+    |> Seq.iter (fun name -> send HttpMethod.Delete (endpoint + "/" + name) None |> ignore)
+
+    printfn "bots=%d routes=%d" bots.Length desired.Length
+
+run ()

services/bot-redirect/settings.json

@@ -0,0 +1,12 @@
+{
+  "radar": "https://api.cloudflare.com/client/v4/radar/",
+  "namespace": "forgejo-public",
+  "gateway": "gateway-relay",
+  "gatewayNamespace": "kube-system",
+  "source": "git.quorate.dev",
+  "target": "code.quorate.dev",
+  "excluded": [
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 11; moto g power (2022)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Mobile Safari/537.36"
+  ]
+}

services/bot-redirect/tests.fsx

@@ -0,0 +1,52 @@
+#load "bots.fsx"
+
+open System.Text.Json
+open Bots
+
+let bot =
+    { Slug = "test"
+      Patterns = [ " Bot/1.0" ]
+      Examples = [ "Bot/1.0"; "Alternate/1.0"; "Browser" ]
+      Signer = Some "https://agent.example.test/keys" }
+
+let matches = headers (Set.singleton "Browser") bot
+assert (matches.Length = 3)
+
+assert
+    (matches
+     |> List.exists (fun header -> header.Kind = "Exact" && header.Value = "Alternate/1.0"))
+
+assert
+    (matches
+     |> List.exists (fun header ->
+         header.Name = "Signature-Agent"
+         && header.Value = "\"https://agent.example.test/keys\""))
+
+assert (matches |> List.forall (fun header -> header.Value <> "Browser"))
+
+let config =
+    JsonDocument.Parse(
+        """{"namespace":"test","gateway":"relay","gatewayNamespace":"test","source":"git.example.test","target":"code.example.test"}"""
+    )
+
+let many =
+    [ for index in 0..128 ->
+          { Name = "User-Agent"
+            Kind = "Exact"
+            Value = string index } ]
+
+let output = routes config.RootElement many
+assert (output.Length = 2)
+assert (output.Head.spec.rules.Length = 2)
+assert (output.Head.spec.rules.Head.matches.Length = 64)
+assert (output.Tail.Head.spec.rules.Head.matches.Length = 1)
+let encoded = JsonDocument.Parse(JsonSerializer.Serialize output)
+
+assert
+    (encoded.RootElement[0]
+        .GetProperty("metadata")
+        .GetProperty("labels")
+        .GetProperty("app.kubernetes.io/managed-by")
+        .GetString() = "bot-redirect")
+
+printfn "bot matching and route partition tests passed"