ce7cb702218693d08ea6d83464d2457c9250027c

services/frigate/.gitignore

@@ -0,0 +1 @@
+.tests/

services/frigate/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+configMapGenerator:
+  - name: frigate-model
+    namespace: frigate
+    files:
+      - model.fsx
+      - run.fsx
+      - settings.json

services/frigate/model.fsx

@@ -0,0 +1,34 @@
+module Model
+
+open System
+open System.IO
+open System.Security.Cryptography
+
+type Outcome =
+    | Cached
+    | Installed
+
+let private checksum path =
+    use input = File.OpenRead path
+    SHA256.HashData input |> Convert.ToHexString |> _.ToLowerInvariant()
+
+let install expected destination (download: Stream -> unit) =
+    if File.Exists destination && checksum destination = expected then
+        Cached
+    else
+        Directory.CreateDirectory(Path.GetDirectoryName destination) |> ignore
+        let temporary = destination + ".incoming"
+
+        try
+            do
+                use output = File.Create temporary
+                download output
+
+            if checksum temporary <> expected then
+                invalidOp "model checksum differs"
+
+            File.Move(temporary, destination, true)
+            Installed
+        finally
+            if File.Exists temporary then
+                File.Delete temporary

services/frigate/run.fsx

@@ -0,0 +1,35 @@
+#r "/opt/fsi/packages/AWSSDK.Core.dll"
+#r "/opt/fsi/packages/AWSSDK.S3.dll"
+#load "model.fsx"
+
+open System.IO
+open System.Text.Json
+open Amazon.Runtime
+open Amazon.S3
+open Amazon.S3.Model
+
+[<CLIMutable>]
+type Settings =
+    { Endpoint: string
+      Bucket: string
+      Key: string
+      Sha256: string
+      Destination: string }
+
+let settings =
+    JsonSerializer.Deserialize<Settings>(File.ReadAllText "/installer/settings.json")
+
+let download (output: Stream) =
+    let accessKey = File.ReadAllText("/s3/accessKeyId").TrimEnd()
+    let secretKey = File.ReadAllText("/s3/secretAccessKey").TrimEnd()
+
+    let config =
+        AmazonS3Config(ServiceURL = settings.Endpoint, AuthenticationRegion = "us-east-1", ForcePathStyle = true)
+
+    use client = new AmazonS3Client(BasicAWSCredentials(accessKey, secretKey), config)
+    let request = GetObjectRequest(BucketName = settings.Bucket, Key = settings.Key)
+    use reply = client.GetObjectAsync(request).GetAwaiter().GetResult()
+    reply.ResponseStream.CopyTo output
+
+let outcome = Model.install settings.Sha256 settings.Destination download
+printfn "Model %A sha256:%s" outcome settings.Sha256

services/frigate/settings.json

@@ -0,0 +1,7 @@
+{
+  "Endpoint": "https://s3.quorate.internal",
+  "Bucket": "artifacts",
+  "Key": "frigate/yolov9-t-320.onnx",
+  "Sha256": "b11e86e5f09446f2db58f79208de2fde0aa76ffa1228cb5f0111bdd3f57d6319",
+  "Destination": "/config/model_cache/yolov9-t-320.onnx"
+}

services/frigate/tests.fsx

@@ -0,0 +1,77 @@
+#load "model.fsx"
+
+open System
+open System.IO
+open System.Security.Cryptography
+open System.Text
+
+let check condition message =
+    if not condition then
+        failwith message
+
+let root =
+    Path.Combine(__SOURCE_DIRECTORY__, ".tests", Guid.NewGuid().ToString("N"))
+
+let destination = Path.Combine(root, "model_cache", "model.onnx")
+let bytes = Encoding.UTF8.GetBytes "synthetic model"
+let expected = SHA256.HashData bytes |> Convert.ToHexString |> _.ToLowerInvariant()
+let mutable downloads = 0
+
+let download (output: Stream) =
+    downloads <- downloads + 1
+    output.Write bytes
+
+let verify () =
+    check (File.ReadAllBytes destination = bytes) "installed bytes differ"
+    check (not (File.Exists(destination + ".incoming"))) "partial file remains"
+
+try
+    check (Model.install expected destination download = Model.Installed) "empty cache was not installed"
+    verify ()
+    check (downloads = 1) "empty cache download count differs"
+
+    let offline (_: Stream) =
+        failwith "cached model attempted a download"
+
+    check (Model.install expected destination offline = Model.Cached) "valid cache was not reused"
+    verify ()
+
+    File.WriteAllText(destination, "corrupt model")
+    check (Model.install expected destination download = Model.Installed) "corrupt cache was not replaced"
+    verify ()
+    check (downloads = 2) "corrupt cache download count differs"
+
+    let stale = Encoding.UTF8.GetBytes "retained model"
+    File.WriteAllBytes(destination, stale)
+
+    let mismatch (output: Stream) =
+        output.Write(Encoding.UTF8.GetBytes "wrong model")
+
+    let mutable rejected = false
+
+    try
+        Model.install expected destination mismatch |> ignore
+    with :? InvalidOperationException ->
+        rejected <- true
+
+    check rejected "checksum mismatch was accepted"
+    check (File.ReadAllBytes destination = stale) "checksum mismatch replaced the retained model"
+    check (not (File.Exists(destination + ".incoming"))) "checksum mismatch left a partial file"
+
+    let interrupted (output: Stream) =
+        output.Write(bytes.AsSpan(0, 4))
+        raise (IOException "synthetic interrupted download")
+
+    rejected <- false
+
+    try
+        Model.install expected destination interrupted |> ignore
+    with :? IOException ->
+        rejected <- true
+
+    check rejected "download error was suppressed"
+    check (File.ReadAllBytes destination = stale) "download error replaced the retained model"
+    check (not (File.Exists(destination + ".incoming"))) "download error left a partial file"
+    printfn "PASS: empty cache, cached/offline model, corrupt cache, checksum mismatch and interrupted download"
+finally
+    Directory.Delete(root, true)