ce7cb702218693d08ea6d83464d2457c9250027c

services/bot-redirect/bots.fsx

@@ -0,0 +1,95 @@
+open System
+open System.Text.Json
+open System.Text.RegularExpressions
+
+type Bot =
+    { Slug: string
+      Patterns: string list
+      Examples: string list
+      Signer: string option }
+
+type Header =
+    { Name: string
+      Kind: string
+      Value: string }
+
+let strings (value: JsonElement) name =
+    value.GetProperty(name: string).EnumerateArray()
+    |> Seq.map _.GetString()
+    |> Seq.toList
+
+let decode (value: JsonElement) =
+    { Slug = value.GetProperty("slug").GetString()
+      Patterns = strings value "userAgentPatterns"
+      Examples = strings value "userAgents"
+      Signer =
+        match value.TryGetProperty "signatureAgentUrl" with
+        | true, url when url.ValueKind = JsonValueKind.String -> Some(url.GetString())
+        | _ -> None }
+
+let headers excluded bot =
+    let patterns = bot.Patterns |> List.map (fun value -> value.Trim(' ', '\t'))
+
+    [ for pattern in patterns do
+          { Name = "User-Agent"
+            Kind = "RegularExpression"
+            Value = ".*" + Regex.Escape(pattern) + ".*" }
+      for example in bot.Examples do
+          if
+              not (Set.contains example excluded)
+              && not (patterns |> List.exists example.Contains)
+          then
+              { Name = "User-Agent"
+                Kind = "Exact"
+                Value = example.Trim(' ', '\t') }
+      match bot.Signer with
+      | Some signer ->
+          { Name = "Signature-Agent"
+            Kind = "Exact"
+            Value = JsonSerializer.Serialize signer }
+      | None -> () ]
+
+let routes config matches =
+    let text name =
+        (config: JsonElement).GetProperty(name: string).GetString()
+
+    matches
+    |> List.distinct
+    |> List.chunkBySize 128
+    |> List.mapi (fun index chunk ->
+        {| apiVersion = "gateway.networking.k8s.io/v1"
+           kind = "HTTPRoute"
+           metadata =
+            {| name = $"bot-redirect-{index}"
+               ``namespace`` = text "namespace"
+               labels = Map [ ("app.kubernetes.io/managed-by", "bot-redirect") ] |}
+           spec =
+            {| parentRefs =
+                [| {| name = text "gateway"
+                      ``namespace`` = text "gatewayNamespace"
+                      sectionName = "https" |} |]
+               hostnames = [| text "source" |]
+               rules =
+                chunk
+                |> List.chunkBySize 64
+                |> List.map (fun group ->
+                    {| matches =
+                        group
+                        |> List.map (fun header ->
+                            {| headers =
+                                [| {| name = header.Name
+                                      ``type`` = header.Kind
+                                      value = header.Value |} |] |})
+                       filters =
+                        [| box
+                               {| ``type`` = "RequestRedirect"
+                                  requestRedirect =
+                                   {| hostname = text "target"
+                                      port = 443
+                                      statusCode = 301 |} |}
+                           box
+                               {| ``type`` = "ResponseHeaderModifier"
+                                  responseHeaderModifier =
+                                   {| set =
+                                       [| {| name = "X-Robots-Tag"
+                                             value = "noindex" |} |] |} |} |] |}) |} |})