ce7cb702218693d08ea6d83464d2457c9250027c
This commit is contained in:
commit
34c5f8963c
325 changed files with 145336 additions and 0 deletions
221
services/project-delivery/actions/apk.fsx
Normal file
221
services/project-delivery/actions/apk.fsx
Normal file
|
|
@ -0,0 +1,221 @@
|
|||
open System
|
||||
open System.Collections.Generic
|
||||
open System.Formats.Tar
|
||||
open System.IO
|
||||
open System.IO.Compression
|
||||
open System.Security.Cryptography
|
||||
open System.Text
|
||||
|
||||
type PathItem = { Source: string; Name: string }
|
||||
|
||||
type Package =
|
||||
{ Name: string
|
||||
Architecture: string
|
||||
Description: string
|
||||
Url: string
|
||||
License: string }
|
||||
|
||||
type Signing = { KeyPath: string; KeyName: string }
|
||||
|
||||
let private require valid message =
|
||||
if not valid then
|
||||
invalidOp message
|
||||
|
||||
let private createHashed destination write =
|
||||
use algorithm = SHA256.Create()
|
||||
use output = File.Create destination
|
||||
|
||||
use hashingOutput =
|
||||
new CryptoStream(output, algorithm, CryptoStreamMode.Write, true)
|
||||
|
||||
write hashingOutput
|
||||
hashingOutput.FlushFinalBlock()
|
||||
Convert.ToHexString(algorithm.Hash).ToLowerInvariant()
|
||||
|
||||
let private sha1 (path: string) =
|
||||
use stream = File.OpenRead path
|
||||
Convert.ToHexString(SHA1.HashData stream).ToLowerInvariant()
|
||||
|
||||
let private packageParents (items: PathItem list) =
|
||||
let parents (name: string) =
|
||||
let mutable parent = Path.GetDirectoryName name
|
||||
let mutable result = []
|
||||
|
||||
while not <| String.IsNullOrEmpty parent do
|
||||
result <- parent.Replace(Path.DirectorySeparatorChar, '/') :: result
|
||||
parent <- Path.GetDirectoryName parent
|
||||
|
||||
result
|
||||
|
||||
let declared =
|
||||
items
|
||||
|> List.filter (fun item -> Directory.Exists item.Source)
|
||||
|> List.map _.Name
|
||||
|> Set.ofList
|
||||
|
||||
items
|
||||
|> List.collect (fun item -> parents item.Name)
|
||||
|> Set.ofList
|
||||
|> Set.filter (fun name -> not <| declared.Contains name)
|
||||
|> Set.toList
|
||||
|> List.sortBy (fun name -> name.Split('/').Length, name)
|
||||
|
||||
let private packageMeta (entry: TarEntry) mode =
|
||||
entry.Mode <- mode
|
||||
entry.Uid <- 65532
|
||||
entry.Gid <- 65532
|
||||
entry.ModificationTime <- DateTimeOffset.UnixEpoch
|
||||
|
||||
let private packageDirectory (writer: TarWriter) (name: string) =
|
||||
let entry = PaxTarEntry(TarEntryType.Directory, name)
|
||||
|
||||
packageMeta
|
||||
entry
|
||||
(UnixFileMode.UserRead
|
||||
||| UnixFileMode.UserWrite
|
||||
||| UnixFileMode.UserExecute
|
||||
||| UnixFileMode.GroupRead
|
||||
||| UnixFileMode.GroupExecute
|
||||
||| UnixFileMode.OtherRead
|
||||
||| UnixFileMode.OtherExecute)
|
||||
|
||||
writer.WriteEntry entry
|
||||
|
||||
let private packageItem (writer: TarWriter) (item: PathItem) =
|
||||
let attributes = File.GetAttributes item.Source
|
||||
|
||||
if attributes.HasFlag FileAttributes.ReparsePoint then
|
||||
let target: FileSystemInfo =
|
||||
if attributes.HasFlag FileAttributes.Directory then
|
||||
DirectoryInfo item.Source
|
||||
else
|
||||
FileInfo item.Source
|
||||
|
||||
require (not <| isNull target.LinkTarget) "invalid symbolic link"
|
||||
|
||||
let checksum =
|
||||
SHA1.HashData(Encoding.UTF8.GetBytes target.LinkTarget)
|
||||
|> Convert.ToHexString
|
||||
|> _.ToLowerInvariant()
|
||||
|
||||
let entry =
|
||||
PaxTarEntry(TarEntryType.SymbolicLink, item.Name, [ KeyValuePair("APK-TOOLS.checksum.SHA1", checksum) ])
|
||||
|
||||
entry.LinkName <- target.LinkTarget
|
||||
|
||||
packageMeta
|
||||
entry
|
||||
(UnixFileMode.UserRead
|
||||
||| UnixFileMode.UserWrite
|
||||
||| UnixFileMode.UserExecute
|
||||
||| UnixFileMode.GroupRead
|
||||
||| UnixFileMode.GroupExecute
|
||||
||| UnixFileMode.OtherRead
|
||||
||| UnixFileMode.OtherExecute)
|
||||
|
||||
writer.WriteEntry entry
|
||||
elif attributes.HasFlag FileAttributes.Directory then
|
||||
packageDirectory writer item.Name
|
||||
else
|
||||
let entry =
|
||||
PaxTarEntry(
|
||||
TarEntryType.RegularFile,
|
||||
item.Name,
|
||||
[ KeyValuePair("APK-TOOLS.checksum.SHA1", sha1 item.Source) ]
|
||||
)
|
||||
|
||||
entry.DataStream <- File.OpenRead item.Source
|
||||
packageMeta entry (File.GetUnixFileMode item.Source)
|
||||
writer.WriteEntry entry
|
||||
entry.DataStream.Dispose()
|
||||
|
||||
let private writeCutMember path name (content: byte[]) =
|
||||
use tar = new MemoryStream()
|
||||
|
||||
do
|
||||
use writer = new TarWriter(tar, TarEntryFormat.Ustar, leaveOpen = true)
|
||||
let entry = UstarTarEntry(TarEntryType.RegularFile, name)
|
||||
entry.DataStream <- new MemoryStream(content)
|
||||
entry.Mode <- UnixFileMode.UserRead ||| UnixFileMode.GroupRead ||| UnixFileMode.OtherRead
|
||||
entry.ModificationTime <- DateTimeOffset.UnixEpoch
|
||||
writer.WriteEntry entry
|
||||
entry.DataStream.Dispose()
|
||||
|
||||
let archive = tar.ToArray()
|
||||
|
||||
require
|
||||
(archive.Length >= 1024
|
||||
&& archive[(archive.Length - 1024) ..] |> Array.forall ((=) 0uy))
|
||||
"invalid tar terminator"
|
||||
|
||||
use output = File.Create path
|
||||
use gzip = new GZipStream(output, CompressionLevel.NoCompression)
|
||||
gzip.Write(archive, 0, archive.Length - 1024)
|
||||
|
||||
let private signingKey (signing: Signing) =
|
||||
let rsa = RSA.Create()
|
||||
rsa.ImportFromPem(File.ReadAllText signing.KeyPath)
|
||||
rsa
|
||||
|
||||
let private signMember signing (rsa: RSA) input output =
|
||||
let signature =
|
||||
rsa.SignData(File.ReadAllBytes input, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1)
|
||||
|
||||
writeCutMember output $".SIGN.RSA256.{signing.KeyName}" signature
|
||||
|
||||
let signIndex signing path =
|
||||
let signature = $"{path}.signature"
|
||||
let signed = $"{path}.signed"
|
||||
use rsa = signingKey signing
|
||||
signMember signing rsa path signature
|
||||
|
||||
use output = File.Create signed
|
||||
use signatureInput = File.OpenRead signature
|
||||
signatureInput.CopyTo output
|
||||
use indexInput = File.OpenRead path
|
||||
indexInput.CopyTo output
|
||||
output.Dispose()
|
||||
File.Move(signed, path, true)
|
||||
File.Delete signature
|
||||
|
||||
let build (info: Package) signing (repository: string) (items: PathItem list) =
|
||||
let name = info.Name
|
||||
let architecture = info.Architecture
|
||||
Directory.CreateDirectory repository |> ignore
|
||||
let data = Path.Combine(repository, $".{name}-data.tar.gz")
|
||||
let control = Path.Combine(repository, $".{name}-control.tar.gz")
|
||||
let signature = Path.Combine(repository, $".{name}-signature.tar.gz")
|
||||
let package = Path.Combine(repository, $"quorate-{name}-0.0.0-r0.apk")
|
||||
|
||||
let dataDigest =
|
||||
createHashed data (fun output ->
|
||||
use gzip = new GZipStream(output, CompressionLevel.NoCompression, true)
|
||||
use writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen = false)
|
||||
packageParents items |> List.iter (packageDirectory writer)
|
||||
items |> List.iter (packageItem writer))
|
||||
|
||||
let installed =
|
||||
items
|
||||
|> List.filter (fun item -> File.Exists item.Source)
|
||||
|> List.sumBy (fun item -> FileInfo(item.Source).Length)
|
||||
|
||||
let metadata =
|
||||
$"pkgname = quorate-{name}\npkgver = 0.0.0-r0\npkgdesc = {info.Description}\nurl = {info.Url}\nbuilddate = 0\npackager = quorate\nsize = {installed}\narch = {architecture}\norigin = quorate-{name}\nlicense = {info.License}\ndatahash = {dataDigest}\n"
|
||||
|> Encoding.UTF8.GetBytes
|
||||
|
||||
writeCutMember control ".PKGINFO" metadata
|
||||
use rsa = signingKey signing
|
||||
signMember signing rsa control signature
|
||||
|
||||
use output = File.Create package
|
||||
use signatureInput = File.OpenRead signature
|
||||
signatureInput.CopyTo output
|
||||
use controlInput = File.OpenRead control
|
||||
controlInput.CopyTo output
|
||||
use dataInput = File.OpenRead data
|
||||
dataInput.CopyTo output
|
||||
output.Dispose()
|
||||
File.Delete data
|
||||
File.Delete control
|
||||
File.Delete signature
|
||||
package
|
||||
44
services/project-delivery/actions/cache-gc.fsx
Normal file
44
services/project-delivery/actions/cache-gc.fsx
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
module CacheGc
|
||||
|
||||
#r "/opt/fsi/packages/AWSSDK.Core.dll"
|
||||
#r "/opt/fsi/packages/AWSSDK.S3.dll"
|
||||
|
||||
open System
|
||||
open System.Net
|
||||
open System.Threading
|
||||
open Amazon.S3
|
||||
open Amazon.S3.Model
|
||||
|
||||
let prune (client: AmazonS3Client) bucket (manifest: string) payload (cancel: CancellationToken) =
|
||||
let prefix = manifest[.. manifest.Length - "_latest.json".Length - 1]
|
||||
let keep = prefix + payload
|
||||
let objects = prefix + "objects/"
|
||||
let obsolete = ResizeArray<string>()
|
||||
let mutable next = null
|
||||
let mutable more = true
|
||||
|
||||
while more do
|
||||
let request =
|
||||
ListObjectsV2Request(BucketName = bucket, Prefix = objects, ContinuationToken = next)
|
||||
|
||||
let reply = client.ListObjectsV2Async(request, cancel).GetAwaiter().GetResult()
|
||||
|
||||
if reply.HttpStatusCode <> HttpStatusCode.OK then
|
||||
invalidOp "cache payload LIST failed"
|
||||
|
||||
if not (isNull reply.S3Objects) then
|
||||
for item in reply.S3Objects do
|
||||
if item.Key <> keep then
|
||||
obsolete.Add item.Key
|
||||
|
||||
next <- reply.NextContinuationToken
|
||||
more <- reply.IsTruncated.GetValueOrDefault()
|
||||
|
||||
for key in obsolete do
|
||||
let request = DeleteObjectRequest(BucketName = bucket, Key = key)
|
||||
let reply = client.DeleteObjectAsync(request, cancel).GetAwaiter().GetResult()
|
||||
|
||||
if reply.HttpStatusCode <> HttpStatusCode.NoContent then
|
||||
invalidOp "cache payload DELETE failed"
|
||||
|
||||
printfn "cache=cleaned prefix=%s deleted=%d" objects obsolete.Count
|
||||
220
services/project-delivery/actions/cache-store-tests.fsx
Normal file
220
services/project-delivery/actions/cache-store-tests.fsx
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
#load "cache-store.fsx"
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.IO
|
||||
open System.Net
|
||||
open System.Text
|
||||
open System.Text.Json
|
||||
open System.Threading.Tasks
|
||||
open Amazon.Runtime
|
||||
open Amazon.S3
|
||||
open Amazon.S3.Model
|
||||
open Contract
|
||||
|
||||
type MemoryStore() =
|
||||
inherit
|
||||
AmazonS3Client(
|
||||
BasicAWSCredentials("synthetic-access", "synthetic-secret"),
|
||||
AmazonS3Config(ServiceURL = "http://127.0.0.1", AuthenticationRegion = "us-east-1")
|
||||
)
|
||||
|
||||
let objects = Dictionary<string, byte array>()
|
||||
let events = ResizeArray<string>()
|
||||
member _.Objects = objects
|
||||
member _.Events = events
|
||||
member val FailWrites = false with get, set
|
||||
member val FailDeletes = false with get, set
|
||||
|
||||
override _.GetObjectAsync(request: GetObjectRequest, _) =
|
||||
let reply =
|
||||
match objects.TryGetValue request.Key with
|
||||
| true, bytes ->
|
||||
new GetObjectResponse(HttpStatusCode = HttpStatusCode.OK, ResponseStream = new MemoryStream(bytes))
|
||||
|> Task.FromResult
|
||||
| false, _ ->
|
||||
Task.FromException<GetObjectResponse>(
|
||||
AmazonS3Exception("missing synthetic object", StatusCode = HttpStatusCode.NotFound)
|
||||
)
|
||||
|
||||
reply
|
||||
|
||||
override this.PutObjectAsync(request: PutObjectRequest, _) =
|
||||
events.Add("put:" + request.Key)
|
||||
|
||||
if this.FailWrites then
|
||||
raise (AmazonS3Exception("synthetic PUT failure", StatusCode = HttpStatusCode.InternalServerError))
|
||||
|
||||
use output = new MemoryStream()
|
||||
request.InputStream.CopyTo output
|
||||
objects[request.Key] <- output.ToArray()
|
||||
Task.FromResult(PutObjectResponse(HttpStatusCode = HttpStatusCode.OK))
|
||||
|
||||
override _.ListObjectsV2Async(request: ListObjectsV2Request, _) =
|
||||
events.Add("list:" + request.Prefix)
|
||||
|
||||
let keys =
|
||||
objects.Keys
|
||||
|> Seq.filter (fun key -> key.StartsWith(request.Prefix, StringComparison.Ordinal))
|
||||
|> Seq.sortWith (fun left right -> StringComparer.Ordinal.Compare(left, right))
|
||||
|> Seq.filter (fun key ->
|
||||
String.IsNullOrEmpty request.ContinuationToken
|
||||
|| StringComparer.Ordinal.Compare(key, request.ContinuationToken) > 0)
|
||||
|> Seq.toArray
|
||||
|
||||
let selected = keys |> Array.truncate 2
|
||||
let more = selected.Length < keys.Length
|
||||
|
||||
let reply =
|
||||
ListObjectsV2Response(
|
||||
HttpStatusCode = HttpStatusCode.OK,
|
||||
IsTruncated = Nullable more,
|
||||
S3Objects = ResizeArray(selected |> Array.map (fun key -> S3Object(Key = key))),
|
||||
NextContinuationToken = (if more then Array.last selected else null)
|
||||
)
|
||||
|
||||
Task.FromResult reply
|
||||
|
||||
override this.DeleteObjectAsync(request: DeleteObjectRequest, _) =
|
||||
events.Add("delete:" + request.Key)
|
||||
|
||||
if this.FailDeletes then
|
||||
raise (AmazonS3Exception("synthetic DELETE failure", StatusCode = HttpStatusCode.Forbidden))
|
||||
|
||||
objects.Remove request.Key |> ignore
|
||||
Task.FromResult(DeleteObjectResponse(HttpStatusCode = HttpStatusCode.NoContent))
|
||||
|
||||
let check condition message =
|
||||
if not condition then
|
||||
failwith message
|
||||
|
||||
let run () =
|
||||
use client = new MemoryStore()
|
||||
let prefix = "projects/synthetic/workflow/cache/toolchain/"
|
||||
let key = prefix + "_latest.json"
|
||||
let payload = "objects/payload.tar.zst"
|
||||
client.Objects[prefix + payload] <- [| 1uy |]
|
||||
|
||||
for index in 0..5 do
|
||||
client.Objects[prefix + $"objects/old-{index}.tar.zst"] <- [| 2uy |]
|
||||
|
||||
let sibling = "projects/synthetic/workflow/cache/build/objects/keep.tar.zst"
|
||||
let neighbor = "projects/other/workflow/cache/toolchain/objects/keep.tar.zst"
|
||||
client.Objects[sibling] <- [| 3uy |]
|
||||
client.Objects[neighbor] <- [| 4uy |]
|
||||
|
||||
let hashes =
|
||||
Dictionary<string, string>(dict [ "package.json", "abc"; "package-lock.json", "def" ])
|
||||
|
||||
CacheStore.writeManifest client key "version" hashes payload
|
||||
let manifest = CacheStore.readManifest client key
|
||||
check (CacheStore.matches "version" hashes manifest) "manifest round trip"
|
||||
check (client.Objects.ContainsKey(prefix + payload)) "current cache payload preserved"
|
||||
check (client.Objects.ContainsKey sibling && client.Objects.ContainsKey neighbor) "other cache prefixes preserved"
|
||||
check (client.Objects.Count = 4) "all paginated obsolete payloads deleted"
|
||||
let events = client.Events |> Seq.toArray
|
||||
|
||||
let firstDelete =
|
||||
events |> Array.findIndex (fun value -> value.StartsWith("delete:"))
|
||||
|
||||
let lastList =
|
||||
events |> Array.findIndexBack (fun value -> value.StartsWith("list:"))
|
||||
|
||||
check (lastList < firstDelete) "inventory completed before deletion"
|
||||
check (events[0] = "put:" + key) "manifest published before cleanup"
|
||||
|
||||
let reordered =
|
||||
Dictionary<string, string>(dict [ "package-lock.json", "def"; "package.json", "abc" ])
|
||||
|
||||
check (CacheStore.matches "version" reordered manifest) "hash ordering"
|
||||
check (not (CacheStore.matches "other" hashes manifest)) "version mismatch"
|
||||
check (not (CacheStore.matches "version" hashes { manifest with SchemaVersion = 2 })) "schema mismatch"
|
||||
check (not (CacheStore.matches "version" hashes { manifest with InputHashes = null })) "missing hashes"
|
||||
let changed = Dictionary<string, string>(hashes)
|
||||
changed["package.json"] <- "changed"
|
||||
check (not (CacheStore.matches "version" changed manifest)) "changed input"
|
||||
changed.Remove "package.json" |> ignore
|
||||
check (not (CacheStore.matches "version" changed manifest)) "missing input"
|
||||
changed["package.json"] <- "abc"
|
||||
changed["extra"] <- "ghi"
|
||||
check (not (CacheStore.matches "version" changed manifest)) "extra input"
|
||||
use document = JsonDocument.Parse(client.Objects[key])
|
||||
let fields = document.RootElement.EnumerateObject() |> Seq.map _.Name |> Set.ofSeq
|
||||
check (fields = Set.ofList [ "schemaVersion"; "cacheVersion"; "inputHashes"; "payloadKey" ]) "manifest wire fields"
|
||||
check (manifest.PayloadKey = "objects/payload.tar.zst") "payload reference"
|
||||
|
||||
let committed = client.Objects[key]
|
||||
let orphan = prefix + "objects/orphan.tar.zst"
|
||||
client.Objects[orphan] <- [| 5uy |]
|
||||
client.Events.Clear()
|
||||
client.FailWrites <- true
|
||||
|
||||
let failedWrite =
|
||||
try
|
||||
CacheStore.writeManifest client key "next" hashes "objects/next.tar.zst"
|
||||
false
|
||||
with :? AmazonS3Exception ->
|
||||
true
|
||||
|
||||
check failedWrite "manifest failure propagated"
|
||||
check (client.Events.Count = 1) "failed manifest never starts cleanup"
|
||||
check (client.Objects[key] = committed && client.Objects.ContainsKey orphan) "failed publication preserves cache"
|
||||
|
||||
client.FailWrites <- false
|
||||
client.FailDeletes <- true
|
||||
|
||||
let failedDelete =
|
||||
try
|
||||
CacheStore.writeManifest client key "version" hashes payload
|
||||
false
|
||||
with :? AmazonS3Exception ->
|
||||
true
|
||||
|
||||
check failedDelete "cleanup failure propagated"
|
||||
check (client.Objects.ContainsKey(prefix + payload)) "cleanup failure preserves current payload"
|
||||
client.FailDeletes <- false
|
||||
CacheStore.writeManifest client key "version" hashes payload
|
||||
check (not (client.Objects.ContainsKey orphan)) "later export cleans orphan payload"
|
||||
|
||||
let directory =
|
||||
Path.Combine(Path.GetTempPath(), "cache-store-" + Guid.NewGuid().ToString("N"))
|
||||
|
||||
Directory.CreateDirectory directory |> ignore
|
||||
let source = Path.Combine(directory, "source")
|
||||
let target = Path.Combine(directory, "download")
|
||||
|
||||
try
|
||||
let bytes = Encoding.UTF8.GetBytes "synthetic cache payload"
|
||||
File.WriteAllBytes(source, bytes)
|
||||
client.Objects["payload"] <- bytes
|
||||
CacheStore.download client "payload" (sha256 source) target
|
||||
check (File.ReadAllBytes target = bytes) "verified download"
|
||||
check (not (File.Exists(target + ".incoming"))) "completed temporary file"
|
||||
|
||||
let rejected =
|
||||
try
|
||||
CacheStore.download client "payload" (String.replicate 64 "0") target
|
||||
false
|
||||
with :? InvalidOperationException ->
|
||||
true
|
||||
|
||||
check rejected "digest mismatch rejection"
|
||||
check (File.ReadAllBytes target = bytes) "existing destination preserved"
|
||||
check (not (File.Exists(target + ".incoming"))) "failed temporary file"
|
||||
|
||||
let missing =
|
||||
try
|
||||
CacheStore.readManifest client "missing" |> ignore
|
||||
false
|
||||
with :? AmazonS3Exception as error ->
|
||||
error.StatusCode = HttpStatusCode.NotFound
|
||||
|
||||
check missing "missing manifest status preserved for caller"
|
||||
finally
|
||||
File.Delete source
|
||||
File.Delete target
|
||||
Directory.Delete directory
|
||||
|
||||
printfn "cache-store tests passed"
|
||||
|
||||
run ()
|
||||
114
services/project-delivery/actions/cache-store.fsx
Normal file
114
services/project-delivery/actions/cache-store.fsx
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
module CacheStore
|
||||
|
||||
#r "/opt/fsi/packages/AWSSDK.Core.dll"
|
||||
#r "/opt/fsi/packages/AWSSDK.S3.dll"
|
||||
#load "contract.fsx"
|
||||
#load "cache-gc.fsx"
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.IO
|
||||
open System.Net
|
||||
open System.Net.Http
|
||||
open System.Security.Cryptography
|
||||
open System.Security.Cryptography.X509Certificates
|
||||
open System.Text.Json
|
||||
open System.Text.Json.Serialization
|
||||
open System.Threading.Tasks
|
||||
open Amazon.Runtime
|
||||
open Amazon.S3
|
||||
open Amazon.S3.Model
|
||||
open Contract
|
||||
|
||||
[<CLIMutable>]
|
||||
type Manifest =
|
||||
{ [<JsonPropertyName("schemaVersion")>]
|
||||
SchemaVersion: int
|
||||
[<JsonPropertyName("cacheVersion")>]
|
||||
CacheVersion: string
|
||||
[<JsonPropertyName("inputHashes")>]
|
||||
InputHashes: Dictionary<string, string>
|
||||
[<JsonPropertyName("payloadKey")>]
|
||||
PayloadKey: string }
|
||||
|
||||
let bucket = "project-delivery"
|
||||
let private awaitResult (operation: Task<'T>) = operation.GetAwaiter().GetResult()
|
||||
|
||||
type private ClientFactory(root: X509Certificate2, timeout: TimeSpan option) =
|
||||
inherit HttpClientFactory()
|
||||
|
||||
override _.CreateHttpClient(_) =
|
||||
let handler = new HttpClientHandler()
|
||||
|
||||
handler.ServerCertificateCustomValidationCallback <-
|
||||
fun _ certificate chain errors -> validateServerCertificate root certificate chain errors
|
||||
|
||||
let client = new HttpClient(handler, true)
|
||||
timeout |> Option.iter (fun value -> client.Timeout <- value)
|
||||
client
|
||||
|
||||
let connect endpoint timeout (credentials: AWSCredentials) root =
|
||||
let configuration =
|
||||
AmazonS3Config(
|
||||
ServiceURL = endpoint,
|
||||
AuthenticationRegion = "us-east-1",
|
||||
ForcePathStyle = true,
|
||||
HttpClientFactory = new ClientFactory(root, timeout)
|
||||
)
|
||||
|
||||
new AmazonS3Client(credentials, configuration)
|
||||
|
||||
let readManifest (client: AmazonS3Client) key =
|
||||
let request = GetObjectRequest(BucketName = bucket, Key = key)
|
||||
use reply = client.GetObjectAsync(request, cancellationToken) |> awaitResult
|
||||
require (reply.HttpStatusCode = HttpStatusCode.OK) "cache manifest GET failed"
|
||||
use reader = new StreamReader(reply.ResponseStream)
|
||||
reader.ReadToEnd() |> decode<Manifest>
|
||||
|
||||
let matches version (hashes: Dictionary<string, string>) (manifest: Manifest) =
|
||||
manifest.SchemaVersion = 1
|
||||
&& manifest.CacheVersion = version
|
||||
&& not (isNull hashes || isNull manifest.InputHashes)
|
||||
&& hashes.Count = manifest.InputHashes.Count
|
||||
&& hashes
|
||||
|> Seq.forall (fun (KeyValue(key, value)) -> manifest.InputHashes.TryGetValue key = (true, value))
|
||||
|
||||
let download (client: AmazonS3Client) key expectedHash destination =
|
||||
let temporary = destination + ".incoming"
|
||||
|
||||
try
|
||||
directory (Path.GetDirectoryName destination) |> ignore
|
||||
let request = GetObjectRequest(BucketName = bucket, Key = key)
|
||||
use reply = client.GetObjectAsync(request, cancellationToken) |> awaitResult
|
||||
require (reply.HttpStatusCode = HttpStatusCode.OK) "cache payload GET failed"
|
||||
use input = reply.ResponseStream
|
||||
|
||||
let digest =
|
||||
use output = File.Create temporary
|
||||
use algorithm = SHA256.Create()
|
||||
use hashing = new CryptoStream(output, algorithm, CryptoStreamMode.Write, true)
|
||||
input.CopyToAsync(hashing, cancellationToken).GetAwaiter().GetResult()
|
||||
hashing.FlushFinalBlock()
|
||||
Convert.ToHexString(algorithm.Hash).ToLowerInvariant()
|
||||
|
||||
require (digest = expectedHash) "cache payload digest mismatch"
|
||||
File.Move(temporary, destination, true)
|
||||
finally
|
||||
if File.Exists temporary then
|
||||
File.Delete temporary
|
||||
|
||||
let writeManifest (client: AmazonS3Client) key version hashes payloadKey =
|
||||
let manifest =
|
||||
{ SchemaVersion = 1
|
||||
CacheVersion = version
|
||||
InputHashes = hashes
|
||||
PayloadKey = payloadKey }
|
||||
|
||||
use input = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, json))
|
||||
|
||||
let request =
|
||||
PutObjectRequest(BucketName = bucket, Key = key, InputStream = input, ContentType = "application/json")
|
||||
|
||||
let reply = client.PutObjectAsync(request, cancellationToken) |> awaitResult
|
||||
require (reply.HttpStatusCode = HttpStatusCode.OK) "cache manifest PUT failed"
|
||||
CacheGc.prune client bucket key payloadKey cancellationToken
|
||||
71
services/project-delivery/actions/clair-oci/build.fsx
Normal file
71
services/project-delivery/actions/clair-oci/build.fsx
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
#load "projects.fsx"
|
||||
#load "image.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open Contract
|
||||
open Image
|
||||
open Projects
|
||||
|
||||
let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
|
||||
let targets = Environment.GetEnvironmentVariable "TARGETS" |> decode<Target array>
|
||||
let project = selectProject targets
|
||||
let sourceRoot = "/output/source"
|
||||
let output = "/output"
|
||||
let go = "/toolchain/usr/lib/go/bin/go"
|
||||
|
||||
let environment architecture =
|
||||
[ "CGO_ENABLED", "0"
|
||||
"GOCACHE", "/cache/go"
|
||||
"GOENV", "off"
|
||||
"GOOS", "linux"
|
||||
"GOARCH", architecture
|
||||
"GOMODCACHE", "/toolchain/modules"
|
||||
"GOPATH", "/toolchain/gopath"
|
||||
"GOPROXY", "off"
|
||||
"GOROOT", "/toolchain/usr/lib/go"
|
||||
"GOWORK", "off"
|
||||
"HOME", "/tmp" ]
|
||||
|
||||
directory sourceRoot |> ignore
|
||||
|
||||
Directory.EnumerateDirectories("/toolchain/upstream", "*", SearchOption.AllDirectories)
|
||||
|> Seq.iter (fun path ->
|
||||
directory (Path.Combine(sourceRoot, Path.GetRelativePath("/toolchain/upstream", path)))
|
||||
|> ignore)
|
||||
|
||||
Directory.EnumerateFiles("/toolchain/upstream", "*", SearchOption.AllDirectories)
|
||||
|> Seq.iter (fun path -> File.Copy(path, Path.Combine(sourceRoot, Path.GetRelativePath("/toolchain/upstream", path))))
|
||||
|
||||
requireSuccess
|
||||
"/toolchain/usr/bin/patch"
|
||||
[ "-p1"; "-i"; "/source/repository/packages/clair/noop.patch" ]
|
||||
(Some(Path.Combine(sourceRoot, "claircore")))
|
||||
[ ("LD_LIBRARY_PATH", "/toolchain/usr/lib") ]
|
||||
|> ignore
|
||||
|
||||
let build architecture (binary: string) =
|
||||
let path = Path.Combine(output, "bin", architecture, ("usr/bin/" + binary))
|
||||
directory (Path.GetDirectoryName path) |> ignore
|
||||
|
||||
requireSuccess
|
||||
go
|
||||
[ "build"
|
||||
"-mod=readonly"
|
||||
"-trimpath"
|
||||
"-ldflags=-s -w -buildid= -X github.com/quay/clair/v4/cmd.Version=v4.9.0-quorate"
|
||||
"-o"
|
||||
path
|
||||
("./cmd/" + binary) ]
|
||||
(Some(Path.Combine(sourceRoot, "clair")))
|
||||
(environment architecture)
|
||||
|> ignore
|
||||
|
||||
File.SetUnixFileMode(path, executableMode)
|
||||
("usr/bin/" + binary), path
|
||||
|
||||
let digest =
|
||||
assemble "/toolchain/image-base" output revision project.Entrypoint project.WorkingDir (fun architecture ->
|
||||
[ "clair"; "clairctl" ] |> List.map (build architecture))
|
||||
|
||||
printfn "build=image project=%s digest=%s" project.Name digest
|
||||
41
services/project-delivery/actions/clair-oci/projects.fsx
Normal file
41
services/project-delivery/actions/clair-oci/projects.fsx
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
#load "contract.fsx"
|
||||
|
||||
open Contract
|
||||
|
||||
type Project =
|
||||
{ Name: string
|
||||
Workflow: string
|
||||
Repository: string
|
||||
Entrypoint: string list
|
||||
WorkingDir: string }
|
||||
|
||||
let project =
|
||||
{ Name = "clair"
|
||||
Workflow = "clair-build"
|
||||
Repository = "quorate/clair"
|
||||
Entrypoint = [ "/usr/bin/clair" ]
|
||||
WorkingDir = "/run" }
|
||||
|
||||
let selectProject (targets: Target array) =
|
||||
require
|
||||
(targets.Length = 1
|
||||
&& targets[0].Kind = "QuayOCI"
|
||||
&& targets[0].Repository = project.Repository)
|
||||
"invalid delivery target"
|
||||
|
||||
project
|
||||
|
||||
let archives =
|
||||
[ "clair",
|
||||
"f6a412ccbfc4c3db83005348584d437348826763",
|
||||
"1cbe2ae88795eba12169b1ceac5eda53ba56af6fd6fb4e645c04d4a0172c0701"
|
||||
"claircore",
|
||||
"182baaed5fbaa355a30e473d64207c30e69ca92a",
|
||||
"daaff87fe62aec729020e81cbec677deb0e13b2b86ac2326d15b6d0e235fa02a" ]
|
||||
|
||||
let validateProject (_: string) (_: Project) = ()
|
||||
let inputPaths (_: string) (_: Project) : string list = []
|
||||
let toolchainProfileName = "clair"
|
||||
|
||||
let toolchainCachePaths (_: Project) =
|
||||
[ "usr/bin/patch"; "usr/lib"; "image-base"; "modules"; "upstream" ]
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
contents:
|
||||
keyring:
|
||||
- https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
|
||||
repositories:
|
||||
- https://packages.wolfi.dev/os
|
||||
packages:
|
||||
- ca-certificates-bundle=20261020-r0
|
||||
- wolfi-baselayout=20230201-r31
|
||||
accounts:
|
||||
groups:
|
||||
- groupname: application
|
||||
gid: 65532
|
||||
users:
|
||||
- username: application
|
||||
uid: 65532
|
||||
gid: 65532
|
||||
run-as: 65532
|
||||
environment:
|
||||
PATH: /usr/bin:/bin
|
||||
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
||||
98
services/project-delivery/actions/clair-oci/toolchain.fsx
Normal file
98
services/project-delivery/actions/clair-oci/toolchain.fsx
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
#load "projects.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open System.Net.Http
|
||||
open Contract
|
||||
open Projects
|
||||
|
||||
let targets = Environment.GetEnvironmentVariable "TARGETS" |> decode<Target array>
|
||||
let project = selectProject targets
|
||||
let root = "/toolchain"
|
||||
let baseImage = Path.Combine(root, "image-base")
|
||||
let upstream = Path.Combine(root, "upstream")
|
||||
|
||||
do
|
||||
let environment =
|
||||
[ "GOCACHE", "/tmp/go-cache"
|
||||
"GOENV", "off"
|
||||
"GOMODCACHE", Path.Combine(root, "modules")
|
||||
"GOPATH", "/tmp/gopath"
|
||||
"GOPROXY", "https://proxy.golang.org"
|
||||
"GOROOT", Path.Combine(root, "usr/lib/go")
|
||||
"GOSUMDB", "sum.golang.org"
|
||||
"GOWORK", "off"
|
||||
"HOME", "/tmp/home" ]
|
||||
|
||||
requireSuccess
|
||||
"/sbin/apk"
|
||||
[ "add"
|
||||
"--root"
|
||||
root
|
||||
"--arch"
|
||||
"x86_64"
|
||||
"--initdb"
|
||||
"--no-scripts"
|
||||
"--keys-dir"
|
||||
"/etc/apk/keys"
|
||||
"--repository"
|
||||
"https://packages.wolfi.dev/os"
|
||||
"patch=2.8.1-r1"
|
||||
"ca-certificates-bundle=20261020-r0"
|
||||
"glibc-2.43=2.43-r19"
|
||||
"glibc-2.43-locale-posix=2.43-r19"
|
||||
"go-1.27=1.27.3-r0"
|
||||
"ld-linux-2.43=2.43-r19"
|
||||
"libgcc=16.2.0-r2"
|
||||
"libxcrypt=4.5.2-r6"
|
||||
"wolfi-baselayout=20230201-r31" ]
|
||||
None
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
let go = Path.Combine(root, "usr/lib/go/bin/go")
|
||||
directory (Path.Combine(root, "modules")) |> ignore
|
||||
directory upstream |> ignore
|
||||
use client = new HttpClient(Timeout = TimeSpan.FromMinutes 10.0)
|
||||
|
||||
archives
|
||||
|> List.iter (fun (name, revision, digest) ->
|
||||
let archive = Path.Combine("/tmp", name + ".tar.gz")
|
||||
|
||||
use input =
|
||||
client
|
||||
.GetStreamAsync($"https://codeload.github.com/quay/{name}/tar.gz/{revision}")
|
||||
.GetAwaiter()
|
||||
.GetResult()
|
||||
|
||||
use output = File.Create archive
|
||||
input.CopyTo output
|
||||
output.Dispose()
|
||||
require (sha256 archive = digest) $"source archive digest mismatch: {name}"
|
||||
extractGzip archive upstream
|
||||
Directory.Move(Path.Combine(upstream, name + "-" + revision), Path.Combine(upstream, name))
|
||||
File.Delete archive)
|
||||
|
||||
let clair = Path.Combine(upstream, "clair")
|
||||
|
||||
requireSuccess go [ "mod"; "edit"; "-replace=github.com/quay/claircore=../claircore" ] (Some clair) environment
|
||||
|> ignore
|
||||
|
||||
requireSuccess go [ "mod"; "download" ] (Some clair) environment |> ignore
|
||||
|
||||
directory baseImage |> ignore
|
||||
|
||||
requireSuccess
|
||||
"/usr/bin/apko"
|
||||
[ "build"
|
||||
"--arch"
|
||||
"x86_64,arm64"
|
||||
"--sbom=false"
|
||||
"/orchestration/go-oci.apko.yaml"
|
||||
"clair-oci:base"
|
||||
baseImage ]
|
||||
None
|
||||
environment
|
||||
|> ignore
|
||||
|
||||
printfn "toolchain=primary"
|
||||
37
services/project-delivery/actions/cluster-health/build.fsx
Normal file
37
services/project-delivery/actions/cluster-health/build.fsx
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
#load "projects.fsx"
|
||||
#load "image.fsx"
|
||||
|
||||
open System
|
||||
open Contract
|
||||
open Projects
|
||||
|
||||
let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
|
||||
|
||||
let project =
|
||||
Environment.GetEnvironmentVariable "TARGETS"
|
||||
|> decode<Target array>
|
||||
|> selectProject
|
||||
|
||||
require (validRevision revision) "invalid source revision"
|
||||
let environment = ("CARGO_NET_OFFLINE", "true") :: environment
|
||||
|
||||
let files architecture =
|
||||
let target =
|
||||
match architecture with
|
||||
| "amd64" -> "x86_64-unknown-linux-gnu"
|
||||
| "arm64" -> "aarch64-unknown-linux-gnu"
|
||||
| _ -> fail "unsupported architecture"
|
||||
|
||||
requireSuccess
|
||||
"/toolchain/usr/bin/cargo"
|
||||
[ "zigbuild"; "--locked"; "--offline"; "--release"; "--target"; target ]
|
||||
(Some source)
|
||||
environment
|
||||
|> ignore
|
||||
|
||||
[ "usr/local/bin/cluster-health-api", $"/output/target/{target}/release/cluster-health-api" ]
|
||||
|
||||
let digest =
|
||||
Image.assemble "/toolchain/image-base" "/output" revision [ "/usr/local/bin/cluster-health-api" ] "/" files
|
||||
|
||||
printfn "build=image project=%s digest=%s" project.Name digest
|
||||
129
services/project-delivery/actions/cluster-health/cache.fsx
Normal file
129
services/project-delivery/actions/cluster-health/cache.fsx
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
#load "cache-store.fsx"
|
||||
#load "projects.fsx"
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.IO
|
||||
open System.Net
|
||||
open System.Security.Cryptography
|
||||
open System.Security.Cryptography.X509Certificates
|
||||
open System.Text
|
||||
open Amazon.Runtime
|
||||
open Amazon.S3
|
||||
open Amazon.S3.Model
|
||||
open Contract
|
||||
open Projects
|
||||
|
||||
type Action =
|
||||
| Pull
|
||||
| Export
|
||||
|
||||
let project =
|
||||
Environment.GetEnvironmentVariable "TARGETS"
|
||||
|> decode<Target array>
|
||||
|> selectProject
|
||||
|
||||
let hashes =
|
||||
[ "Cargo.toml"; "Cargo.lock" ]
|
||||
|> Seq.map (fun name -> $"services/cluster-health-api/{name}", sha256 (Path.Combine(source, name)))
|
||||
|> dict
|
||||
|> Dictionary<string, string>
|
||||
|
||||
let version =
|
||||
[ "projects.fsx"
|
||||
"toolchain.fsx"
|
||||
"build.fsx"
|
||||
"runtime.apko.yaml"
|
||||
"cache.fsx" ]
|
||||
|> List.map (fun name -> sha256 (Path.Combine("/orchestration", name)))
|
||||
|> List.append [ string host ]
|
||||
|> String.concat "\n"
|
||||
|> Encoding.UTF8.GetBytes
|
||||
|> SHA256.HashData
|
||||
|> Convert.ToHexString
|
||||
|> _.ToLowerInvariant()
|
||||
|
||||
let pull client key prefix archive path =
|
||||
let manifest =
|
||||
try
|
||||
Some(CacheStore.readManifest client key)
|
||||
with :? AmazonS3Exception as error when error.StatusCode = HttpStatusCode.NotFound ->
|
||||
None
|
||||
|
||||
match manifest with
|
||||
| Some value when CacheStore.matches version hashes value ->
|
||||
let digest = Path.GetFileName(value.PayloadKey).Replace(".tar.zst", "")
|
||||
|
||||
require
|
||||
(validDigest $"sha256:{digest}" && value.PayloadKey = $"objects/{digest}.tar.zst")
|
||||
"invalid cache payload key"
|
||||
|
||||
CacheStore.download client (prefix + value.PayloadKey) digest archive
|
||||
|
||||
if path = toolchain then
|
||||
extractZstdRooted archive path
|
||||
else
|
||||
extractZstd archive path
|
||||
|
||||
File.Delete archive
|
||||
true
|
||||
| _ -> false
|
||||
|
||||
let export (client: AmazonS3Client) key prefix archive path paths =
|
||||
let digest = createZstdSelection 2 path paths archive
|
||||
let payload = $"objects/{digest}.tar.zst"
|
||||
use input = File.OpenRead archive
|
||||
|
||||
let request =
|
||||
PutObjectRequest(BucketName = CacheStore.bucket, Key = prefix + payload, InputStream = input)
|
||||
|
||||
client.PutObjectAsync(request, cancellationToken).GetAwaiter().GetResult()
|
||||
|> ignore
|
||||
|
||||
CacheStore.writeManifest client key version hashes payload
|
||||
File.Delete archive
|
||||
|
||||
let transfer client action (name, path, paths) =
|
||||
let prefix = $"projects/{project.Name}/{project.Workflow}/cache/{name}/"
|
||||
let key = prefix + "_latest.json"
|
||||
let archive = $"/tmp/{name}.tar.zst"
|
||||
|
||||
match action with
|
||||
| Pull ->
|
||||
let hit = pull client key prefix archive path
|
||||
|
||||
if name = "toolchain" then
|
||||
let marker =
|
||||
if hit then
|
||||
"/state/toolchain-cache"
|
||||
else
|
||||
"/state/toolchain-primary"
|
||||
|
||||
File.WriteAllText(marker, "")
|
||||
|
||||
printfn "cache=%s component=%s" (if hit then "restored" else "miss") name
|
||||
| Export ->
|
||||
export client key prefix archive path paths
|
||||
printfn "cache=exported component=%s" name
|
||||
|
||||
let run () =
|
||||
let action =
|
||||
match Environment.GetEnvironmentVariable "CACHE_ACTION" with
|
||||
| "pull" -> Pull
|
||||
| "export" -> Export
|
||||
| _ -> fail "invalid cache action"
|
||||
|
||||
let credential name =
|
||||
File.ReadAllText(Path.Combine("/cache-auth", name)).Trim()
|
||||
|
||||
let credentials =
|
||||
BasicAWSCredentials(credential "accesskey", credential "secretkey")
|
||||
|
||||
use root = X509CertificateLoader.LoadCertificateFromFile "/cache-ca/ca.crt"
|
||||
use client = CacheStore.connect "https://s3.quorate.internal" None credentials root
|
||||
|
||||
[ "toolchain", "/toolchain", [ "bin"; "lib"; "lib64"; "usr"; "cargo"; "image-base" ]
|
||||
"build", "/output/target", [ "." ] ]
|
||||
|> List.iter (transfer client action)
|
||||
|
||||
run ()
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#load "workspace.fsx"
|
||||
|
||||
System.IO.Directory.CreateDirectory "/workspace/output/target" |> ignore
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
#load "contract.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open System.Runtime.InteropServices
|
||||
open Contract
|
||||
|
||||
type Project =
|
||||
{ Name: string
|
||||
Workflow: string
|
||||
Repository: string }
|
||||
|
||||
let project =
|
||||
{ Name = "cluster-health-api"
|
||||
Workflow = "cluster-health-build"
|
||||
Repository = "quorate/cluster-health-api" }
|
||||
|
||||
let selectProject (targets: Target array) =
|
||||
match targets with
|
||||
| [| target |] when target.Kind = "QuayOCI" && target.Repository = project.Repository -> project
|
||||
| _ -> fail "invalid delivery target"
|
||||
|
||||
let source = "/source/repository/services/cluster-health-api"
|
||||
let toolchain = "/toolchain"
|
||||
let rustPackage = "rust-1.99=1.99.1-r2"
|
||||
|
||||
type HostAssets =
|
||||
{ ZigTarget: string
|
||||
ZigHash: string
|
||||
StdTarget: string
|
||||
StdArch: string }
|
||||
|
||||
let host = RuntimeInformation.ProcessArchitecture
|
||||
|
||||
let assets =
|
||||
match host with
|
||||
| Architecture.X64 ->
|
||||
{ ZigTarget = "x86_64-unknown-linux-musl"
|
||||
ZigHash = "52be2cb85ad3875682564a29bac1832b2ab3db68b1c504765c92d5249cc9b96f"
|
||||
StdTarget = "aarch64-unknown-linux-gnu"
|
||||
StdArch = "aarch64" }
|
||||
| Architecture.Arm64 ->
|
||||
{ ZigTarget = "aarch64-unknown-linux-gnu"
|
||||
ZigHash = "de42626985f428033078dfc12ac8a59ee713f8e5627235ec883b1c899bee895c"
|
||||
StdTarget = "x86_64-unknown-linux-gnu"
|
||||
StdArch = "x86_64" }
|
||||
| _ -> fail "unsupported toolchain architecture"
|
||||
|
||||
let environment =
|
||||
[ "PATH", "/toolchain/usr/local/bin:/toolchain/usr/bin:/toolchain/bin"
|
||||
"LD_LIBRARY_PATH", "/toolchain/usr/lib"
|
||||
"CARGO_HOME", "/toolchain/cargo"
|
||||
"CARGO_TARGET_DIR", "/output/target"
|
||||
"HOME", "/tmp"
|
||||
"ZIG_GLOBAL_CACHE_DIR", "/tmp/zig"
|
||||
"ZIG_LOCAL_CACHE_DIR", "/tmp/zig-local" ]
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
contents:
|
||||
keyring:
|
||||
- https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
|
||||
repositories:
|
||||
- https://packages.wolfi.dev/os
|
||||
packages:
|
||||
- ca-certificates-bundle=20261020-r0
|
||||
- wolfi-baselayout=20230201-r30
|
||||
- glibc=2.43-r14
|
||||
- libgcc=16.2.0-r1
|
||||
accounts:
|
||||
groups:
|
||||
- groupname: application
|
||||
gid: 65532
|
||||
users:
|
||||
- username: application
|
||||
uid: 65532
|
||||
gid: 65532
|
||||
run-as: 65532
|
||||
environment:
|
||||
SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
|
||||
118
services/project-delivery/actions/cluster-health/toolchain.fsx
Normal file
118
services/project-delivery/actions/cluster-health/toolchain.fsx
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
#r "/opt/fsi/packages/HostExec.dll"
|
||||
#load "projects.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open System.Net.Http
|
||||
open Contract
|
||||
open Projects
|
||||
open Quorate.HostExec
|
||||
|
||||
while not (File.Exists "/state/toolchain-cache" || File.Exists "/state/toolchain-primary") do
|
||||
sleep 100
|
||||
|
||||
if File.Exists "/state/toolchain-cache" then
|
||||
printfn "toolchain=cache"
|
||||
else
|
||||
requireSuccess
|
||||
"/sbin/apk"
|
||||
[ "add"
|
||||
"--root"
|
||||
toolchain
|
||||
"--initdb"
|
||||
"--no-scripts"
|
||||
"--keys-dir"
|
||||
"/etc/apk/keys"
|
||||
"--repository"
|
||||
"https://packages.wolfi.dev/os"
|
||||
rustPackage
|
||||
"libLLVM-22=22.1.8-r2"
|
||||
"libLLVM-20=20.1.8-r18"
|
||||
"zig=0.17.0-r0"
|
||||
"busybox=1.38.0-r1"
|
||||
"gcc=16.2.0-r1"
|
||||
"glibc-dev=2.43-r14"
|
||||
"ca-certificates-bundle=20261020-r0" ]
|
||||
None
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
let root = ToolRoot.Open toolchain
|
||||
|
||||
[ GnuTool.C; GnuTool.Gcc ]
|
||||
|> List.iter (fun tool -> GnuProxy.Install(root, tool) |> ignore)
|
||||
|
||||
let archives = directory "/tmp/archives"
|
||||
use http = new HttpClient()
|
||||
|
||||
let fetch name (url: string) digest =
|
||||
let path = Path.Combine(archives, name)
|
||||
use input = http.GetStreamAsync(url, cancellationToken).GetAwaiter().GetResult()
|
||||
|
||||
do
|
||||
use output = File.Create path
|
||||
input.CopyTo output
|
||||
|
||||
require (sha256 path = digest) $"toolchain checksum mismatch: {name}"
|
||||
path
|
||||
|
||||
let zigbuild =
|
||||
fetch
|
||||
"zigbuild.tar.xz"
|
||||
$"https://github.com/rust-cross/cargo-zigbuild/releases/download/v0.24.0/cargo-zigbuild-{assets.ZigTarget}.tar.xz"
|
||||
assets.ZigHash
|
||||
|
||||
extractXz zigbuild (Path.Combine(archives, "zigbuild"))
|
||||
|
||||
let binary =
|
||||
Directory.EnumerateFiles(Path.Combine(archives, "zigbuild"), "cargo-zigbuild", SearchOption.AllDirectories)
|
||||
|> Seq.exactlyOne
|
||||
|
||||
directory "/toolchain/usr/local/bin" |> ignore
|
||||
File.Copy(binary, "/toolchain/usr/local/bin/cargo-zigbuild")
|
||||
|
||||
let cross = directory "/tmp/cross-std"
|
||||
|
||||
requireSuccess
|
||||
"/sbin/apk"
|
||||
[ "add"
|
||||
"--root"
|
||||
cross
|
||||
"--arch"
|
||||
assets.StdArch
|
||||
"--initdb"
|
||||
"--no-scripts"
|
||||
"--keys-dir"
|
||||
"/etc/apk/keys"
|
||||
"--repository"
|
||||
"https://packages.wolfi.dev/os"
|
||||
rustPackage ]
|
||||
None
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
copyTree
|
||||
(Path.Combine(cross, "usr/lib/rustlib", assets.StdTarget))
|
||||
(Path.Combine(toolchain, "usr/lib/rustlib", assets.StdTarget))
|
||||
|
||||
directory "/toolchain/cargo" |> ignore
|
||||
|
||||
requireSuccess "/toolchain/usr/bin/cargo" [ "fetch"; "--locked" ] (Some source) environment
|
||||
|> ignore
|
||||
|
||||
directory "/toolchain/image-base" |> ignore
|
||||
|
||||
requireSuccess
|
||||
"/usr/bin/apko"
|
||||
[ "build"
|
||||
"--arch"
|
||||
"x86_64,arm64"
|
||||
"--sbom=false"
|
||||
"/orchestration/runtime.apko.yaml"
|
||||
"cluster-health:base"
|
||||
"/toolchain/image-base" ]
|
||||
None
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
printfn "toolchain=primary"
|
||||
723
services/project-delivery/actions/contract.fsx
Normal file
723
services/project-delivery/actions/contract.fsx
Normal file
|
|
@ -0,0 +1,723 @@
|
|||
#load "apk.fsx"
|
||||
#r "System.Formats.Tar"
|
||||
#r "/opt/fsi/packages/SharpCompress.dll"
|
||||
#r "/opt/fsi/packages/ZstdSharp.dll"
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.Diagnostics
|
||||
open System.Formats.Tar
|
||||
open System.IO
|
||||
open System.IO.Compression
|
||||
open System.Net.Http
|
||||
open System.Net.Security
|
||||
open System.Runtime.InteropServices
|
||||
open System.Security.Cryptography
|
||||
open System.Security.Cryptography.X509Certificates
|
||||
open System.Text
|
||||
open System.Text.Json
|
||||
open System.Text.Json.Serialization
|
||||
open System.Threading
|
||||
open System.Threading.Tasks
|
||||
open SharpCompress.Compressors.Xz
|
||||
open ZstdSharp
|
||||
open ZstdSharp.Unsafe
|
||||
|
||||
type FileCap =
|
||||
| NetAdmin
|
||||
| NetRaw
|
||||
|
||||
[<CLIMutable>]
|
||||
type Target =
|
||||
{ [<JsonPropertyName("kind")>]
|
||||
Kind: string
|
||||
[<JsonPropertyName("repository")>]
|
||||
Repository: string }
|
||||
|
||||
[<CLIMutable>]
|
||||
type Assembly =
|
||||
{ [<JsonPropertyName("repository")>]
|
||||
Repository: string
|
||||
[<JsonPropertyName("digest")>]
|
||||
Digest: string }
|
||||
|
||||
[<CLIMutable>]
|
||||
type Published =
|
||||
{ [<JsonPropertyName("kind")>]
|
||||
Kind: string
|
||||
[<JsonPropertyName("repository")>]
|
||||
Repository: string
|
||||
[<JsonPropertyName("reference")>]
|
||||
Reference: string }
|
||||
|
||||
[<CLIMutable>]
|
||||
type Delivery =
|
||||
{ [<JsonPropertyName("sourceRevision")>]
|
||||
SourceRevision: string
|
||||
[<JsonPropertyName("targets")>]
|
||||
Targets: Published array }
|
||||
|
||||
[<CLIMutable>]
|
||||
type Descriptor =
|
||||
{ [<JsonPropertyName("mediaType")>]
|
||||
MediaType: string
|
||||
[<JsonPropertyName("digest")>]
|
||||
Digest: string
|
||||
[<JsonPropertyName("size")>]
|
||||
Size: int64
|
||||
[<JsonPropertyName("annotations")>]
|
||||
Annotations: Dictionary<string, string> }
|
||||
|
||||
[<CLIMutable>]
|
||||
type Index =
|
||||
{ [<JsonPropertyName("schemaVersion")>]
|
||||
SchemaVersion: int
|
||||
[<JsonPropertyName("mediaType")>]
|
||||
MediaType: string
|
||||
[<JsonPropertyName("manifests")>]
|
||||
Manifests: Descriptor array }
|
||||
|
||||
type PathItem = { Source: string; Name: string }
|
||||
|
||||
type ProcessResult =
|
||||
{ Code: int
|
||||
Error: string
|
||||
Output: string }
|
||||
|
||||
let json = JsonSerializerOptions(PropertyNamingPolicy = null, WriteIndented = false)
|
||||
|
||||
let private stopping = new CancellationTokenSource()
|
||||
let mutable private signalExitCode = 143
|
||||
|
||||
let private registerSignal signal exitCode =
|
||||
PosixSignalRegistration.Create(
|
||||
signal,
|
||||
fun context ->
|
||||
context.Cancel <- true
|
||||
signalExitCode <- exitCode
|
||||
stopping.Cancel()
|
||||
)
|
||||
|
||||
let private interrupt = registerSignal PosixSignal.SIGINT 130
|
||||
let private terminate = registerSignal PosixSignal.SIGTERM 143
|
||||
let cancellationToken = stopping.Token
|
||||
|
||||
let private exitForSignal<'T> () =
|
||||
Environment.Exit signalExitCode
|
||||
Unchecked.defaultof<'T>
|
||||
|
||||
let fail message = invalidOp message
|
||||
|
||||
let require valid message =
|
||||
if not valid then
|
||||
fail message
|
||||
|
||||
[<RequireQualifiedAccess>]
|
||||
type BuildKind =
|
||||
| Release
|
||||
| Test
|
||||
|
||||
module BuildKind =
|
||||
let parse =
|
||||
function
|
||||
| "release" -> BuildKind.Release
|
||||
| "test" -> BuildKind.Test
|
||||
| _ -> fail "invalid build kind"
|
||||
|
||||
let value =
|
||||
function
|
||||
| BuildKind.Release -> "release"
|
||||
| BuildKind.Test -> "test"
|
||||
|
||||
let directory (path: string) =
|
||||
Directory.CreateDirectory path |> ignore
|
||||
path
|
||||
|
||||
let sha256 (path: string) =
|
||||
use stream = File.OpenRead path
|
||||
Convert.ToHexString(SHA256.HashData stream).ToLowerInvariant()
|
||||
|
||||
[<TailCall>]
|
||||
let rec isTransient (ex: exn) =
|
||||
match ex with
|
||||
| :? HttpRequestException
|
||||
| :? IOException
|
||||
| :? TaskCanceledException -> true
|
||||
| :? AggregateException as aggregate -> aggregate.Flatten().InnerExceptions |> Seq.exists isTransient
|
||||
| _ ->
|
||||
match ex.InnerException with
|
||||
| null -> false
|
||||
| inner -> isTransient inner
|
||||
|
||||
let exponentialBackoff attempt = int (1000.0 * pown 2.0 (attempt - 1))
|
||||
|
||||
// fsharplint:disable noAsyncRunSynchronouslyInLibrary
|
||||
let wait (work: Async<'T>) =
|
||||
try
|
||||
Async.RunSynchronously(work, cancellationToken = cancellationToken)
|
||||
with :? OperationCanceledException when cancellationToken.IsCancellationRequested ->
|
||||
exitForSignal ()
|
||||
// fsharplint:enable noAsyncRunSynchronouslyInLibrary
|
||||
|
||||
let sleep (milliseconds: int) =
|
||||
try
|
||||
Task.Delay(milliseconds, cancellationToken).GetAwaiter().GetResult()
|
||||
with :? OperationCanceledException when cancellationToken.IsCancellationRequested ->
|
||||
exitForSignal ()
|
||||
|
||||
[<TailCall>]
|
||||
let rec private retryLoop
|
||||
(attempts: int)
|
||||
(backoffMs: int -> int)
|
||||
(retryable: exn -> bool)
|
||||
(label: string)
|
||||
(n: int)
|
||||
(work: unit -> Async<'T>)
|
||||
: Async<'T> =
|
||||
async {
|
||||
try
|
||||
return! work ()
|
||||
with ex when not cancellationToken.IsCancellationRequested && retryable ex && n < attempts ->
|
||||
printfn "%s-retry attempt=%d error=%s" label n (ex.GetType().Name)
|
||||
do! Async.Sleep(backoffMs n)
|
||||
return! retryLoop attempts backoffMs retryable label (n + 1) work
|
||||
}
|
||||
|
||||
let retry attempts backoffMs retryable label work =
|
||||
retryLoop attempts backoffMs retryable label 1 work
|
||||
|
||||
let retrySync attempts backoffMs retryable label (work: unit -> 'T) =
|
||||
retry attempts backoffMs retryable label (fun () -> async { return work () })
|
||||
|> wait
|
||||
|
||||
let downloadVerified (http: HttpClient) (url: string) (destination: string) (digest: string) =
|
||||
if File.Exists destination && sha256 destination = digest then
|
||||
()
|
||||
else
|
||||
directory (Path.GetDirectoryName destination) |> ignore
|
||||
|
||||
retry 5 exponentialBackoff isTransient "download" (fun () ->
|
||||
async {
|
||||
let temporary =
|
||||
Path.Combine(
|
||||
Path.GetDirectoryName destination,
|
||||
$".{Path.GetFileName destination}.{Path.GetRandomFileName()}"
|
||||
)
|
||||
|
||||
try
|
||||
use request = new HttpRequestMessage(HttpMethod.Get, url)
|
||||
|
||||
use! reply =
|
||||
http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
|
||||
|> Async.AwaitTask
|
||||
|
||||
require reply.IsSuccessStatusCode $"download failed: {int reply.StatusCode} {url}"
|
||||
use! input = reply.Content.ReadAsStreamAsync(cancellationToken) |> Async.AwaitTask
|
||||
use output = File.Create temporary
|
||||
do! input.CopyToAsync(output, cancellationToken) |> Async.AwaitTask
|
||||
output.Dispose()
|
||||
require (sha256 temporary = digest) $"digest mismatch: {Path.GetFileName destination}"
|
||||
File.Move(temporary, destination, true)
|
||||
with ex ->
|
||||
if File.Exists temporary then
|
||||
File.Delete temporary
|
||||
|
||||
return raise ex
|
||||
})
|
||||
|> wait
|
||||
|
||||
let validRevision (value: string) =
|
||||
not <| isNull value
|
||||
&& (value.Length = 40 || value.Length = 64)
|
||||
&& value
|
||||
|> Seq.forall (fun character -> Char.IsAsciiHexDigit character && not <| Char.IsUpper character)
|
||||
|
||||
let validDigest (value: string) =
|
||||
not <| isNull value
|
||||
&& value.Length = 71
|
||||
&& value.StartsWith("sha256:", StringComparison.Ordinal)
|
||||
&& value[7..] |> Seq.forall Char.IsAsciiHexDigit
|
||||
|
||||
let decode<'T> (value: string) =
|
||||
let result = JsonSerializer.Deserialize<'T>(value, json)
|
||||
require (not <| isNull (box result)) "JSON document is missing"
|
||||
result
|
||||
|
||||
let run executable arguments workingDirectory environment =
|
||||
let start = ProcessStartInfo executable
|
||||
start.UseShellExecute <- false
|
||||
start.RedirectStandardOutput <- true
|
||||
start.RedirectStandardError <- true
|
||||
|
||||
match workingDirectory with
|
||||
| Some path -> start.WorkingDirectory <- path
|
||||
| None -> ()
|
||||
|
||||
for argument in arguments do
|
||||
start.ArgumentList.Add argument
|
||||
|
||||
for key, value in environment do
|
||||
start.Environment[key] <- value
|
||||
|
||||
use child = Process.Start start
|
||||
let output = child.StandardOutput.ReadToEndAsync()
|
||||
let error = child.StandardError.ReadToEndAsync()
|
||||
|
||||
try
|
||||
child.WaitForExitAsync(cancellationToken).GetAwaiter().GetResult()
|
||||
with :? OperationCanceledException when cancellationToken.IsCancellationRequested ->
|
||||
if not child.HasExited then
|
||||
child.Kill true
|
||||
child.WaitForExit()
|
||||
|
||||
exitForSignal ()
|
||||
|
||||
{ Code = child.ExitCode
|
||||
Error = error.GetAwaiter().GetResult()
|
||||
Output = output.GetAwaiter().GetResult() }
|
||||
|
||||
let requireSuccess executable arguments workingDirectory environment =
|
||||
let result = run executable arguments workingDirectory environment
|
||||
require (result.Code = 0) $"{Path.GetFileName executable} failed: {result.Error.Trim()}"
|
||||
result.Output
|
||||
|
||||
let verifyDigestAs path digestPath name =
|
||||
let fields =
|
||||
File.ReadAllText(digestPath).Trim().Split([| ' '; '\t' |], StringSplitOptions.RemoveEmptyEntries)
|
||||
|
||||
require
|
||||
(fields.Length = 2 && fields[0] = sha256 path && fields[1] = name)
|
||||
$"invalid digest: {Path.GetFileName path}"
|
||||
|
||||
let verifyDigest path digestPath =
|
||||
verifyDigestAs path digestPath (Path.GetFileName path)
|
||||
|
||||
let writeDigest path =
|
||||
File.WriteAllText($"{path}.sha256", $"{sha256 path} {Path.GetFileName path}\n")
|
||||
|
||||
let readApkIndex (path: string) =
|
||||
use input = File.OpenRead path
|
||||
use gzip = new GZipStream(input, CompressionMode.Decompress)
|
||||
use reader = new TarReader(gzip)
|
||||
let mutable content = None
|
||||
let mutable entry = reader.GetNextEntry()
|
||||
|
||||
while not (isNull entry) do
|
||||
if entry.Name = "APKINDEX" then
|
||||
use text = new StreamReader(entry.DataStream)
|
||||
content <- Some(text.ReadToEnd())
|
||||
|
||||
entry <- reader.GetNextEntry()
|
||||
|
||||
content |> Option.defaultWith (fun () -> fail "APKINDEX unavailable")
|
||||
|
||||
let writeApkIndex (path: string) (contents: string seq) =
|
||||
let content =
|
||||
contents
|
||||
|> Seq.map (fun value -> value.Trim())
|
||||
|> String.concat "\n\n"
|
||||
|> fun value -> value + "\n\n"
|
||||
|
||||
use output = File.Create path
|
||||
use gzip = new GZipStream(output, CompressionLevel.NoCompression)
|
||||
use writer = new TarWriter(gzip, TarEntryFormat.Ustar, leaveOpen = false)
|
||||
let entry = UstarTarEntry(TarEntryType.RegularFile, "APKINDEX")
|
||||
entry.DataStream <- new MemoryStream(Encoding.UTF8.GetBytes content)
|
||||
entry.Mode <- UnixFileMode.UserRead ||| UnixFileMode.GroupRead ||| UnixFileMode.OtherRead
|
||||
entry.ModificationTime <- DateTimeOffset.UnixEpoch
|
||||
writer.WriteEntry entry
|
||||
entry.DataStream.Dispose()
|
||||
|
||||
let private descendants (root: string) =
|
||||
seq {
|
||||
let pending = Stack<string>(Directory.EnumerateFileSystemEntries root)
|
||||
|
||||
while pending.Count > 0 do
|
||||
let path = pending.Pop()
|
||||
yield path
|
||||
let attributes = File.GetAttributes path
|
||||
|
||||
if
|
||||
attributes.HasFlag FileAttributes.Directory
|
||||
&& not <| attributes.HasFlag FileAttributes.ReparsePoint
|
||||
then
|
||||
for child in Directory.EnumerateFileSystemEntries path do
|
||||
pending.Push child
|
||||
}
|
||||
|
||||
let private tree root = descendants root |> Seq.append [ root ]
|
||||
|
||||
let copyTree source destination =
|
||||
require (Directory.Exists source) $"missing directory: {source}"
|
||||
|
||||
for path in tree source |> Seq.sort do
|
||||
let relative = Path.GetRelativePath(source, path)
|
||||
|
||||
let target =
|
||||
if relative = "." then
|
||||
destination
|
||||
else
|
||||
Path.Combine(destination, relative)
|
||||
|
||||
let attributes = File.GetAttributes path
|
||||
|
||||
if attributes.HasFlag FileAttributes.ReparsePoint then
|
||||
directory (Path.GetDirectoryName target) |> ignore
|
||||
|
||||
let link: FileSystemInfo =
|
||||
if attributes.HasFlag FileAttributes.Directory then
|
||||
DirectoryInfo path
|
||||
else
|
||||
FileInfo path
|
||||
|
||||
require (not <| isNull link.LinkTarget) "invalid symbolic link"
|
||||
|
||||
let targetLink: FileSystemInfo =
|
||||
if attributes.HasFlag FileAttributes.Directory then
|
||||
DirectoryInfo target
|
||||
else
|
||||
FileInfo target
|
||||
|
||||
targetLink.CreateAsSymbolicLink(link.LinkTarget)
|
||||
elif attributes.HasFlag FileAttributes.Directory then
|
||||
directory target |> ignore
|
||||
File.SetUnixFileMode(target, File.GetUnixFileMode path)
|
||||
else
|
||||
directory (Path.GetDirectoryName target) |> ignore
|
||||
File.Copy(path, target, true)
|
||||
File.SetUnixFileMode(target, File.GetUnixFileMode path)
|
||||
|
||||
let private safePath (root: string) (name: string) =
|
||||
require (not <| Path.IsPathFullyQualified name) "archive path is absolute"
|
||||
let path = Path.GetFullPath(Path.Combine(root, name))
|
||||
let prefix = root + string Path.DirectorySeparatorChar
|
||||
require (path = root || path.StartsWith(prefix, StringComparison.Ordinal)) "archive path escapes destination"
|
||||
path
|
||||
|
||||
let private extract rootedAbsoluteLinks input destination =
|
||||
let root = Path.GetFullPath destination
|
||||
directory root |> ignore
|
||||
use reader = new TarReader(input, leaveOpen = false)
|
||||
let mutable entry = reader.GetNextEntry()
|
||||
|
||||
while not <| isNull entry do
|
||||
let path = safePath root entry.Name
|
||||
|
||||
match entry.EntryType with
|
||||
| TarEntryType.Directory ->
|
||||
directory path |> ignore
|
||||
File.SetUnixFileMode(path, entry.Mode)
|
||||
| TarEntryType.RegularFile
|
||||
| TarEntryType.V7RegularFile
|
||||
| TarEntryType.ContiguousFile ->
|
||||
directory (Path.GetDirectoryName path) |> ignore
|
||||
use output = File.Create path
|
||||
|
||||
if not <| isNull entry.DataStream then
|
||||
entry.DataStream.CopyTo output
|
||||
|
||||
output.Dispose()
|
||||
File.SetUnixFileMode(path, entry.Mode)
|
||||
| TarEntryType.SymbolicLink ->
|
||||
directory (Path.GetDirectoryName path) |> ignore
|
||||
|
||||
let linkName =
|
||||
if Path.IsPathFullyQualified entry.LinkName then
|
||||
require rootedAbsoluteLinks $"archive link is absolute: {entry.Name}"
|
||||
|
||||
Path.GetRelativePath(
|
||||
Path.GetDirectoryName path,
|
||||
safePath root (entry.LinkName.TrimStart(Path.DirectorySeparatorChar))
|
||||
)
|
||||
else
|
||||
entry.LinkName
|
||||
|
||||
let linkTarget =
|
||||
Path.GetFullPath(Path.Combine(Path.GetDirectoryName path, linkName))
|
||||
|
||||
let prefix = root + string Path.DirectorySeparatorChar
|
||||
|
||||
require
|
||||
(linkTarget = root || linkTarget.StartsWith(prefix, StringComparison.Ordinal))
|
||||
"archive link escapes destination"
|
||||
|
||||
File.CreateSymbolicLink(path, linkName) |> ignore
|
||||
| TarEntryType.GlobalExtendedAttributes -> ()
|
||||
| _ -> fail $"unsupported archive entry: {entry.Name}"
|
||||
|
||||
entry <- reader.GetNextEntry()
|
||||
|
||||
let extractGzip (source: string) (destination: string) =
|
||||
use input = File.OpenRead source
|
||||
use gzip = new GZipStream(input, CompressionMode.Decompress)
|
||||
extract false gzip destination
|
||||
|
||||
let extractZstd (source: string) (destination: string) =
|
||||
use input = File.OpenRead source
|
||||
use zstd = new DecompressionStream(input, 131072, false, false)
|
||||
extract false zstd destination
|
||||
|
||||
let extractZstdRooted (source: string) (destination: string) =
|
||||
use input = File.OpenRead source
|
||||
use zstd = new DecompressionStream(input, 131072, false, false)
|
||||
extract true zstd destination
|
||||
|
||||
let extractTar (source: string) (destination: string) =
|
||||
use input = File.OpenRead source
|
||||
extract false input destination
|
||||
|
||||
let extractXz (source: string) (destination: string) =
|
||||
use input = File.OpenRead source
|
||||
use xz = new XZStream(input)
|
||||
extract false xz destination
|
||||
|
||||
let private relativeName (root: string) (path: string) =
|
||||
Path.GetRelativePath(root, path).Replace(Path.DirectorySeparatorChar, '/')
|
||||
|
||||
let private writeNamedEntry (writer: TarWriter) (name: string) (path: string) =
|
||||
let attributes = File.GetAttributes path
|
||||
|
||||
if attributes.HasFlag FileAttributes.ReparsePoint then
|
||||
let entry = PaxTarEntry(TarEntryType.SymbolicLink, name)
|
||||
|
||||
let link: FileSystemInfo =
|
||||
if attributes.HasFlag FileAttributes.Directory then
|
||||
DirectoryInfo path
|
||||
else
|
||||
FileInfo path
|
||||
|
||||
require (not <| isNull link.LinkTarget) "invalid symbolic link"
|
||||
entry.LinkName <- link.LinkTarget
|
||||
|
||||
entry.Mode <-
|
||||
UnixFileMode.UserRead
|
||||
||| UnixFileMode.UserWrite
|
||||
||| UnixFileMode.UserExecute
|
||||
||| UnixFileMode.GroupRead
|
||||
||| UnixFileMode.GroupExecute
|
||||
||| UnixFileMode.OtherRead
|
||||
||| UnixFileMode.OtherExecute
|
||||
|
||||
entry.ModificationTime <- DateTimeOffset.UnixEpoch
|
||||
writer.WriteEntry entry
|
||||
elif attributes.HasFlag FileAttributes.Directory then
|
||||
let entry = PaxTarEntry(TarEntryType.Directory, name)
|
||||
|
||||
entry.Mode <-
|
||||
UnixFileMode.UserRead
|
||||
||| UnixFileMode.UserWrite
|
||||
||| UnixFileMode.UserExecute
|
||||
||| UnixFileMode.GroupRead
|
||||
||| UnixFileMode.GroupExecute
|
||||
||| UnixFileMode.OtherRead
|
||||
||| UnixFileMode.OtherExecute
|
||||
|
||||
entry.ModificationTime <- DateTimeOffset.UnixEpoch
|
||||
writer.WriteEntry entry
|
||||
else
|
||||
let entry = PaxTarEntry(TarEntryType.RegularFile, name)
|
||||
entry.Mode <- File.GetUnixFileMode path
|
||||
entry.ModificationTime <- DateTimeOffset.UnixEpoch
|
||||
entry.DataStream <- File.OpenRead path
|
||||
writer.WriteEntry entry
|
||||
entry.DataStream.Dispose()
|
||||
|
||||
let private writeEntry (writer: TarWriter) (root: string) (path: string) =
|
||||
writeNamedEntry writer (relativeName root path) path
|
||||
|
||||
let private writeTree (writer: TarWriter) (source: string) =
|
||||
tree source
|
||||
|> Seq.sortBy (relativeName source)
|
||||
|> Seq.iter (writeEntry writer source)
|
||||
|
||||
let private createCompressedTar (level: CompressionLevel) (source: string) (destination: string) =
|
||||
use output = File.Create destination
|
||||
use gzip = new GZipStream(output, level)
|
||||
use writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen = false)
|
||||
writeTree writer source
|
||||
|
||||
let createGzip source destination =
|
||||
createCompressedTar CompressionLevel.SmallestSize source destination
|
||||
|
||||
let createOptimalGzip source destination =
|
||||
createCompressedTar CompressionLevel.Optimal source destination
|
||||
|
||||
let private createHashed destination write =
|
||||
use algorithm = SHA256.Create()
|
||||
use output = File.Create destination
|
||||
|
||||
use hashingOutput =
|
||||
new CryptoStream(output, algorithm, CryptoStreamMode.Write, true)
|
||||
|
||||
write hashingOutput
|
||||
hashingOutput.FlushFinalBlock()
|
||||
Convert.ToHexString(algorithm.Hash).ToLowerInvariant()
|
||||
|
||||
let createZstdMappings workers mappings destination =
|
||||
createHashed destination (fun output ->
|
||||
use zstd = new CompressionStream(output, 1, 131072, true)
|
||||
zstd.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, workers)
|
||||
use writer = new TarWriter(zstd, TarEntryFormat.Pax, leaveOpen = false)
|
||||
|
||||
mappings
|
||||
|> Seq.collect (fun (source: string, name: string) ->
|
||||
let paths =
|
||||
if Directory.Exists source then
|
||||
tree source
|
||||
else
|
||||
require (File.Exists source) $"missing archive input: {Path.GetFileName source}"
|
||||
Seq.singleton source
|
||||
|
||||
paths
|
||||
|> Seq.map (fun path ->
|
||||
let relative = relativeName source path
|
||||
|
||||
let entryName = if relative = "." then name else $"{name}/{relative}"
|
||||
|
||||
entryName, path))
|
||||
|> Seq.sortBy fst
|
||||
|> Seq.iter (fun (name, path) -> writeNamedEntry writer name path))
|
||||
|
||||
let createZstdSelection workers root paths destination =
|
||||
createHashed destination (fun output ->
|
||||
use zstd = new CompressionStream(output, 1, 131072, true)
|
||||
zstd.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, workers)
|
||||
use writer = new TarWriter(zstd, TarEntryFormat.Pax, leaveOpen = false)
|
||||
|
||||
let selected = paths |> Seq.toArray
|
||||
|
||||
if selected.Length = 0 then
|
||||
writeNamedEntry writer "." root
|
||||
else
|
||||
selected
|
||||
|> Seq.collect (fun path ->
|
||||
let full = Path.Combine(root, path)
|
||||
let attributes = File.GetAttributes full
|
||||
|
||||
if attributes.HasFlag FileAttributes.ReparsePoint then
|
||||
Seq.singleton full
|
||||
elif Directory.Exists full then
|
||||
tree full
|
||||
else
|
||||
require (File.Exists full) $"missing archive input: {Path.GetFileName full}"
|
||||
Seq.singleton full)
|
||||
|> Seq.distinct
|
||||
|> Seq.sortBy (relativeName root)
|
||||
|> Seq.iter (writeEntry writer root))
|
||||
|
||||
let apkFiles (repository: string) =
|
||||
Directory.EnumerateFiles(repository, "*.apk") |> Seq.toList
|
||||
|
||||
let treePaths (root: string) (destination: string) =
|
||||
require (Directory.Exists root) "missing package tree"
|
||||
|
||||
tree root
|
||||
|> Seq.filter (fun source -> source <> root)
|
||||
|> Seq.sort
|
||||
|> Seq.map (fun source ->
|
||||
{ Source = source
|
||||
Name =
|
||||
Path.Combine(destination, Path.GetRelativePath(root, source)).Replace(Path.DirectorySeparatorChar, '/') })
|
||||
|> Seq.toList
|
||||
|
||||
let private readIndex (path: string) =
|
||||
let bytes = File.ReadAllBytes path
|
||||
let index = JsonSerializer.Deserialize<Index>(bytes, json)
|
||||
require (not <| isNull (box index)) "OCI index is missing"
|
||||
|
||||
require
|
||||
(index.SchemaVersion = 2
|
||||
&& index.MediaType = "application/vnd.oci.image.index.v1+json"
|
||||
&& index.Manifests.Length > 0)
|
||||
"OCI index is invalid"
|
||||
|
||||
bytes, index
|
||||
|
||||
let private writeAtomic (path: string) (bytes: byte array) =
|
||||
let temporary = Path.Combine(Path.GetDirectoryName path, Path.GetRandomFileName())
|
||||
File.WriteAllBytes(temporary, bytes)
|
||||
File.Move(temporary, path, true)
|
||||
|
||||
let tag (layout: string) (reference: string) =
|
||||
require (not <| String.IsNullOrWhiteSpace reference) "OCI reference is empty"
|
||||
let indexPath = Path.Combine(layout, "index.json")
|
||||
let data, index = readIndex indexPath
|
||||
let indexType = "application/vnd.oci.image.index.v1+json"
|
||||
let referenceName = "org.opencontainers.image.ref.name"
|
||||
|
||||
if index.Manifests.Length = 1 then
|
||||
let descriptor = index.Manifests[0]
|
||||
|
||||
let tagged =
|
||||
descriptor.MediaType = indexType
|
||||
&& not <| isNull descriptor.Annotations
|
||||
&& descriptor.Annotations.TryGetValue(referenceName) |> fst
|
||||
|
||||
if tagged && descriptor.Annotations[referenceName] = reference then
|
||||
let digest = descriptor.Digest["sha256:".Length ..]
|
||||
let blob = Path.Combine(layout, "blobs", "sha256", digest)
|
||||
let blobData, _ = readIndex blob
|
||||
require (int64 blobData.Length = descriptor.Size && sha256 blob = digest) "tagged OCI index is invalid"
|
||||
descriptor.Digest
|
||||
else
|
||||
let digest = sha256 indexPath
|
||||
let blob = Path.Combine(layout, "blobs", "sha256", digest)
|
||||
directory (Path.GetDirectoryName blob) |> ignore
|
||||
|
||||
if File.Exists blob then
|
||||
require (File.ReadAllBytes(blob) = data) "OCI index digest collision"
|
||||
else
|
||||
writeAtomic blob data
|
||||
|
||||
let annotations = Dictionary<string, string>()
|
||||
annotations.Add(referenceName, reference)
|
||||
|
||||
let root =
|
||||
{ SchemaVersion = 2
|
||||
MediaType = indexType
|
||||
Manifests =
|
||||
[| { MediaType = indexType
|
||||
Digest = $"sha256:{digest}"
|
||||
Size = int64 data.Length
|
||||
Annotations = annotations } |] }
|
||||
|
||||
writeAtomic indexPath (JsonSerializer.SerializeToUtf8Bytes(root, json))
|
||||
$"sha256:{digest}"
|
||||
else
|
||||
let digest = sha256 indexPath
|
||||
let blob = Path.Combine(layout, "blobs", "sha256", digest)
|
||||
directory (Path.GetDirectoryName blob) |> ignore
|
||||
writeAtomic blob data
|
||||
let annotations = Dictionary<string, string>()
|
||||
annotations.Add(referenceName, reference)
|
||||
|
||||
let root =
|
||||
{ SchemaVersion = 2
|
||||
MediaType = indexType
|
||||
Manifests =
|
||||
[| { MediaType = indexType
|
||||
Digest = $"sha256:{digest}"
|
||||
Size = int64 data.Length
|
||||
Annotations = annotations } |] }
|
||||
|
||||
writeAtomic indexPath (JsonSerializer.SerializeToUtf8Bytes(root, json))
|
||||
$"sha256:{digest}"
|
||||
|
||||
let validateServerCertificate
|
||||
(root: X509Certificate2)
|
||||
(certificate: X509Certificate)
|
||||
(providedChain: X509Chain)
|
||||
(policyErrors: SslPolicyErrors)
|
||||
=
|
||||
not (isNull certificate)
|
||||
&& not (isNull providedChain)
|
||||
&& not (policyErrors.HasFlag SslPolicyErrors.RemoteCertificateNameMismatch)
|
||||
&& (use chain = new X509Chain()
|
||||
chain.ChainPolicy.TrustMode <- X509ChainTrustMode.CustomRootTrust
|
||||
chain.ChainPolicy.CustomTrustStore.Add root |> ignore
|
||||
chain.ChainPolicy.RevocationMode <- X509RevocationMode.NoCheck
|
||||
|
||||
for element in providedChain.ChainElements do
|
||||
chain.ChainPolicy.ExtraStore.Add element.Certificate |> ignore
|
||||
|
||||
chain.Build(new X509Certificate2(certificate)))
|
||||
172
services/project-delivery/actions/forgejo.fsx
Normal file
172
services/project-delivery/actions/forgejo.fsx
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
module Forgejo
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open System.Net
|
||||
open System.Net.Http
|
||||
open System.Net.Http.Headers
|
||||
open System.Text
|
||||
open System.Text.Json
|
||||
open System.Text.Json.Serialization
|
||||
open Contract
|
||||
|
||||
[<CLIMutable>]
|
||||
type Asset =
|
||||
{ [<JsonPropertyName("name")>]
|
||||
Name: string
|
||||
[<JsonPropertyName("browser_download_url")>]
|
||||
DownloadUrl: string }
|
||||
|
||||
[<CLIMutable>]
|
||||
type Release =
|
||||
{ [<JsonPropertyName("id")>]
|
||||
Id: int64
|
||||
[<JsonPropertyName("tag_name")>]
|
||||
TagName: string
|
||||
[<JsonPropertyName("target_commitish")>]
|
||||
TargetCommitish: string
|
||||
[<JsonPropertyName("html_url")>]
|
||||
HtmlUrl: string
|
||||
[<JsonPropertyName("assets")>]
|
||||
Assets: Asset array }
|
||||
|
||||
let publish (name: string) (repository: string) (revision: string) (binaries: string array) =
|
||||
for path in binaries do
|
||||
verifyDigest path (path + ".sha256")
|
||||
|
||||
let sums = Path.Combine("/tmp", "SHA256SUMS")
|
||||
File.WriteAllLines(sums, binaries |> Array.map (fun path -> $"{sha256 path} {Path.GetFileName path}"))
|
||||
let assets = Array.append binaries [| sums |]
|
||||
let token = File.ReadAllText("/publisher-auth/token").Trim()
|
||||
require (not (String.IsNullOrWhiteSpace token)) "publisher credential unavailable"
|
||||
|
||||
let api =
|
||||
"http://forge-git-http.forgejo-public.svc.cluster.local.:3000/api/v1/repos"
|
||||
+ Uri(repository).AbsolutePath
|
||||
|
||||
let client = new HttpClient(Timeout = TimeSpan.FromMinutes 15.0)
|
||||
client.DefaultRequestHeaders.Authorization <- AuthenticationHeaderValue("token", token)
|
||||
|
||||
let requestJson method path (body: string option) =
|
||||
use request = new HttpRequestMessage(method, api + path)
|
||||
|
||||
match body with
|
||||
| Some value -> request.Content <- new StringContent(value, Encoding.UTF8, "application/json")
|
||||
| None -> ()
|
||||
|
||||
use reply = client.Send(request, cancellationToken)
|
||||
reply.StatusCode, reply.Content.ReadAsStringAsync(cancellationToken).GetAwaiter().GetResult()
|
||||
|
||||
let artifactDigest = sha256 sums
|
||||
let tag = $"delivery-{revision}-{artifactDigest}"
|
||||
|
||||
let existingStatus, existingBody =
|
||||
requestJson HttpMethod.Get $"/releases/tags/{Uri.EscapeDataString tag}" None
|
||||
|
||||
let release =
|
||||
if existingStatus = HttpStatusCode.NotFound then
|
||||
let body =
|
||||
{| tag_name = tag
|
||||
target_commitish = revision
|
||||
name = $"{name} {revision[..11]} {artifactDigest[..11]}"
|
||||
body = $"Build of {revision}"
|
||||
draft = false
|
||||
prerelease = false |}
|
||||
|
||||
let createdStatus, createdBody =
|
||||
requestJson HttpMethod.Post "/releases" (Some(JsonSerializer.Serialize(body, json)))
|
||||
|
||||
require (createdStatus = HttpStatusCode.Created) $"release creation failed: {int createdStatus}"
|
||||
createdBody |> decode<Release>
|
||||
else
|
||||
require
|
||||
(int existingStatus >= 200 && int existingStatus < 300)
|
||||
$"release lookup failed: {int existingStatus}"
|
||||
|
||||
existingBody |> decode<Release>
|
||||
|
||||
require
|
||||
(release.TagName = tag
|
||||
&& release.TargetCommitish = revision
|
||||
&& not (String.IsNullOrWhiteSpace release.HtmlUrl))
|
||||
"release identity mismatch"
|
||||
|
||||
let verifyRemote (asset: Asset) local =
|
||||
let download = Uri asset.DownloadUrl
|
||||
|
||||
require
|
||||
(download.Scheme = Uri.UriSchemeHttps
|
||||
&& download.Host = "git.quorate.dev"
|
||||
&& download.IsDefaultPort)
|
||||
"release asset URL is outside Forgejo"
|
||||
|
||||
use reply =
|
||||
client
|
||||
.GetAsync(
|
||||
$"http://forge-git-http.forgejo-public.svc.cluster.local.:3000{download.PathAndQuery}",
|
||||
cancellationToken
|
||||
)
|
||||
.GetAwaiter()
|
||||
.GetResult()
|
||||
|
||||
require reply.IsSuccessStatusCode $"release asset verification failed: {asset.Name}"
|
||||
let temporaryPath = Path.Combine("/tmp", ".verify-" + asset.Name)
|
||||
|
||||
do
|
||||
use input =
|
||||
reply.Content.ReadAsStreamAsync(cancellationToken).GetAwaiter().GetResult()
|
||||
|
||||
use temporary = File.Create temporaryPath
|
||||
input.CopyToAsync(temporary, cancellationToken).GetAwaiter().GetResult()
|
||||
temporary.FlushAsync(cancellationToken).GetAwaiter().GetResult()
|
||||
|
||||
require (sha256 temporaryPath = sha256 local) $"release asset digest mismatch: {asset.Name}"
|
||||
|
||||
for path in assets do
|
||||
let name = Path.GetFileName path
|
||||
|
||||
match release.Assets |> Array.tryFind (fun asset -> asset.Name = name) with
|
||||
| Some asset -> verifyRemote asset path
|
||||
| None ->
|
||||
use content = new MultipartFormDataContent()
|
||||
use stream = File.OpenRead path
|
||||
use file = new StreamContent(stream)
|
||||
file.Headers.ContentType <- MediaTypeHeaderValue("application/octet-stream")
|
||||
content.Add(file, "attachment", name)
|
||||
|
||||
use reply =
|
||||
client
|
||||
.PostAsync(
|
||||
$"{api}/releases/{release.Id}/assets?name={Uri.EscapeDataString name}",
|
||||
content,
|
||||
cancellationToken
|
||||
)
|
||||
.GetAwaiter()
|
||||
.GetResult()
|
||||
|
||||
require
|
||||
(reply.StatusCode = HttpStatusCode.Created)
|
||||
$"release asset upload failed: {name} ({int reply.StatusCode})"
|
||||
|
||||
let uploaded =
|
||||
reply.Content.ReadAsStringAsync(cancellationToken).GetAwaiter().GetResult()
|
||||
|> decode<Asset>
|
||||
|
||||
require (uploaded.Name = name) "uploaded release asset identity mismatch"
|
||||
verifyRemote uploaded path
|
||||
|
||||
directory "/result" |> ignore
|
||||
|
||||
File.WriteAllText(
|
||||
"/result/delivery-result",
|
||||
JsonSerializer.Serialize(
|
||||
{| sourceRevision = revision
|
||||
targets =
|
||||
[| {| kind = "ForgejoRelease"
|
||||
repository = repository
|
||||
reference = release.HtmlUrl |} |] |},
|
||||
json
|
||||
)
|
||||
)
|
||||
|
||||
client.Dispose()
|
||||
68
services/project-delivery/actions/git.fsx
Normal file
68
services/project-delivery/actions/git.fsx
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
module GitTool
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open Contract
|
||||
|
||||
let install root =
|
||||
requireSuccess
|
||||
"/sbin/apk"
|
||||
[ "add"
|
||||
"--root"
|
||||
root
|
||||
"--initdb"
|
||||
"--keys-dir"
|
||||
"/etc/apk/keys"
|
||||
"--repository"
|
||||
"https://packages.wolfi.dev/os"
|
||||
"ca-certificates-bundle=20261020-r0"
|
||||
"cyrus-sasl-heimdal-libs=2.1.28-r54"
|
||||
"gdbm=1.26-r5"
|
||||
"git=2.56.0-r3"
|
||||
"glibc-2.43=2.43-r14"
|
||||
"glibc-2.43-locale-posix=2.43-r14"
|
||||
"heimdal-libs=7.8.0-r51"
|
||||
"keyutils-libs=1.6.3-r39"
|
||||
"krb5-conf=1.0-r9"
|
||||
"krb5-libs=1.22.2-r2"
|
||||
"ld-linux-2.43=2.43-r14"
|
||||
"libbrotlicommon1=1.2.0-r3"
|
||||
"libbrotlidec1=1.2.0-r3"
|
||||
"libcom_err=1.47.4-r1"
|
||||
"libcrypt1-2.43=2.43-r14"
|
||||
"libcrypto3=3.6.3-r4"
|
||||
"libcurl-openssl4=8.21.0-r2"
|
||||
"libedit=3.1-r18"
|
||||
"libexpat1=2.8.3-r0"
|
||||
"libgcc=16.1.0-r4"
|
||||
"libidn2=2.3.8-r8"
|
||||
"libldap-2.7=2.7.0-r0"
|
||||
"libnghttp2-14=1.70.0-r2"
|
||||
"libpcre2-8-0=10.47-r0"
|
||||
"libpsl=0.23.3-r0"
|
||||
"libselinux=3.11-r0"
|
||||
"libsepol=3.11-r0"
|
||||
"libssl3=3.6.3-r4"
|
||||
"libunistring=1.4.2-r2"
|
||||
"libverto=0.3.2-r7"
|
||||
"libxcrypt=4.5.2-r4"
|
||||
"ncurses=6.6.20260815-r0"
|
||||
"ncurses-terminfo-base=6.6.20260815-r0"
|
||||
"nghttp3=1.18.0-r0"
|
||||
"ngtcp2=1.25.0-r1"
|
||||
"openssh-client=10.5_p1-r0"
|
||||
"readline=8.3-r2"
|
||||
"sqlite-libs=3.53.4-r0"
|
||||
"wolfi-baselayout=20230201-r29"
|
||||
"zlib=1.3.2-r4" ]
|
||||
None
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
let environment root =
|
||||
[ "GIT_EXEC_PATH", Path.Combine(root, "usr/libexec/git-core")
|
||||
"LD_LIBRARY_PATH", $"{root}/usr/lib:{root}/lib"
|
||||
"PATH", $"{root}/usr/bin:{root}/bin" ]
|
||||
|
||||
let run root arguments workingDirectory additions =
|
||||
requireSuccess (Path.Combine(root, "usr/bin/git")) arguments workingDirectory (environment root @ additions)
|
||||
80
services/project-delivery/actions/go-oci/build.fsx
Normal file
80
services/project-delivery/actions/go-oci/build.fsx
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
#load "projects.fsx"
|
||||
#load "image.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open Contract
|
||||
open Image
|
||||
open Projects
|
||||
|
||||
let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
|
||||
let targets = Environment.GetEnvironmentVariable "TARGETS" |> decode<Target array>
|
||||
let project = selectProject targets
|
||||
let sourceRoot = "/source/repository"
|
||||
let output = "/output"
|
||||
let go = "/toolchain/usr/lib/go/bin/go"
|
||||
|
||||
let environment architecture =
|
||||
[ "CGO_ENABLED", "0"
|
||||
"GOCACHE", "/tmp/go-cache"
|
||||
"GOENV", "off"
|
||||
"GOOS", "linux"
|
||||
"GOARCH", architecture
|
||||
"GOMODCACHE", "/toolchain/modules"
|
||||
"GOPATH", "/toolchain/gopath"
|
||||
"GOPROXY", "off"
|
||||
"GOROOT", "/toolchain/usr/lib/go"
|
||||
"GOWORK", "off"
|
||||
"HOME", "/tmp" ]
|
||||
|
||||
let test () =
|
||||
project.Modules
|
||||
|> List.iter (fun path ->
|
||||
let result =
|
||||
run
|
||||
go
|
||||
[ "test"; "-mod=readonly"; "./..." ]
|
||||
(Some(moduleRoot sourceRoot path))
|
||||
(("LD_LIBRARY_PATH", "/toolchain/usr/lib") :: environment "amd64")
|
||||
|
||||
require (result.Code = 0) $"go test failed: {result.Error.Trim()}\n{result.Output.Trim()}")
|
||||
|
||||
let build architecture (binary: Binary) =
|
||||
let path = Path.Combine(output, "bin", architecture, binary.Destination)
|
||||
directory (Path.GetDirectoryName path) |> ignore
|
||||
|
||||
requireSuccess
|
||||
go
|
||||
[ "build"
|
||||
"-mod=readonly"
|
||||
"-trimpath"
|
||||
"-ldflags=-s -w -buildid="
|
||||
"-o"
|
||||
path
|
||||
binary.Package ]
|
||||
(Some(moduleRoot sourceRoot binary.Module))
|
||||
(environment architecture)
|
||||
|> ignore
|
||||
|
||||
File.SetUnixFileMode(path, executableMode)
|
||||
binary.Destination, path
|
||||
|
||||
test ()
|
||||
|
||||
let digest =
|
||||
assembleWithCaps project.FileCaps "/toolchain/image-base" output revision project.Entrypoint project.WorkingDir (fun architecture ->
|
||||
let runtime =
|
||||
runtimeAssets project
|
||||
|> List.filter (fun (arch, _, _) -> arch = architecture)
|
||||
|> List.map (fun _ -> "usr/bin/codex", Path.Combine("/toolchain/runtime", architecture, "codex"))
|
||||
|
||||
let licenses =
|
||||
if project.Name = "ostension" then
|
||||
[ "LICENSE"; "NOTICE" ]
|
||||
|> List.map (fun name -> $"usr/share/licenses/ostension/{name}", Path.Combine(sourceRoot, name))
|
||||
else
|
||||
[]
|
||||
|
||||
(project.Binaries |> List.map (build architecture)) @ runtime @ licenses)
|
||||
|
||||
printfn "build=image project=%s digest=%s" project.Name digest
|
||||
84
services/project-delivery/actions/go-oci/cache-tests.fsx
Normal file
84
services/project-delivery/actions/go-oci/cache-tests.fsx
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
#load "contract.fsx"
|
||||
|
||||
open System
|
||||
open System.Formats.Tar
|
||||
open System.IO
|
||||
open Contract
|
||||
|
||||
let verifyTree source restored =
|
||||
let rec verify path =
|
||||
let relative = Path.GetRelativePath(source, path)
|
||||
let destination = Path.Combine(restored, relative)
|
||||
let entry = FileInfo path
|
||||
|
||||
if not (isNull entry.LinkTarget) then
|
||||
require (FileInfo(destination).LinkTarget = entry.LinkTarget) $"link changed: {relative}"
|
||||
1, 1
|
||||
elif Directory.Exists path then
|
||||
require (Directory.Exists destination) $"directory missing: {relative}"
|
||||
|
||||
Directory.EnumerateFileSystemEntries path
|
||||
|> Seq.map verify
|
||||
|> Seq.fold (fun (count, links) (items, symbolic) -> count + items, links + symbolic) (1, 0)
|
||||
else
|
||||
require (sha256 path = sha256 destination) $"content changed: {relative}"
|
||||
require (File.GetUnixFileMode path = File.GetUnixFileMode destination) $"mode changed: {relative}"
|
||||
1, 0
|
||||
|
||||
verify
|
||||
|
||||
let roundTrip output source paths =
|
||||
let archive = Path.Combine(output, "cache.tar")
|
||||
let restored = directory (Path.Combine(output, "restored"))
|
||||
|
||||
do
|
||||
use stream = File.Create archive
|
||||
use writer = new TarWriter(stream, TarEntryFormat.Pax, true)
|
||||
writeSelection writer source paths
|
||||
|
||||
let archived =
|
||||
use stream = File.OpenRead archive
|
||||
use reader = new TarReader(stream)
|
||||
|
||||
Seq.initInfinite (fun _ -> reader.GetNextEntry(false))
|
||||
|> Seq.takeWhile (isNull >> not)
|
||||
|> Seq.map (fun entry -> entry.Name, entry.EntryType, entry.LinkName)
|
||||
|> Seq.toList
|
||||
|
||||
TarFile.ExtractToDirectory(archive, restored, false)
|
||||
|
||||
let count, links =
|
||||
paths
|
||||
|> Seq.map (fun path -> verifyTree source restored (Path.Combine(source, path)))
|
||||
|> Seq.fold (fun (count, links) (items, symbolic) -> count + items, links + symbolic) (0, 0)
|
||||
|
||||
require (archived.Length = count) "archive contains entries outside the selected tree"
|
||||
printfn "PASS restore: %d entries; %d symlinks; contents and modes match; overwrite=false" count links
|
||||
archived
|
||||
|
||||
let synthetic output =
|
||||
let source = directory (Path.Combine(output, "source"))
|
||||
let library = directory (Path.Combine(source, "usr/lib/apk/db"))
|
||||
let bin = directory (Path.Combine(source, "usr/bin"))
|
||||
File.WriteAllText(Path.Combine(library, "installed"), "synthetic package database")
|
||||
let executable = Path.Combine(bin, "nginx")
|
||||
File.WriteAllText(executable, "synthetic executable")
|
||||
File.SetUnixFileMode(executable, UnixFileMode.UserRead ||| UnixFileMode.UserExecute ||| UnixFileMode.GroupRead)
|
||||
File.WriteAllText(Path.Combine(source, "excluded"), "outside selection")
|
||||
Directory.CreateSymbolicLink(Path.Combine(source, "usr/lib64"), "lib") |> ignore
|
||||
Directory.CreateSymbolicLink(Path.Combine(source, "usr/sbin"), "bin") |> ignore
|
||||
File.CreateSymbolicLink(Path.Combine(bin, "web"), "nginx") |> ignore
|
||||
let archived = roundTrip output source [ "usr" ]
|
||||
|
||||
for name, target in [ "usr/lib64", "lib"; "usr/sbin", "bin"; "usr/bin/web", "nginx" ] do
|
||||
require (List.contains (name, TarEntryType.SymbolicLink, target) archived) $"symlink not archived: {name}"
|
||||
|
||||
printfn "PASS selection: directory aliases stay symlinks; unselected files stay excluded"
|
||||
|
||||
let args = fsi.CommandLineArgs |> Array.skip 1
|
||||
require (args.Length = 1 || args.Length >= 3) "usage: cache-tests.fsx <scratchpad output> [<source> <paths>...]"
|
||||
let output = directory (Path.Combine(args[0], Guid.NewGuid().ToString("N")[..7]))
|
||||
synthetic (directory (Path.Combine(output, "synthetic")))
|
||||
|
||||
if args.Length >= 3 then
|
||||
roundTrip (directory (Path.Combine(output, "toolchain"))) args[1] args[2..] |> ignore
|
||||
276
services/project-delivery/actions/go-oci/cache.fsx
Normal file
276
services/project-delivery/actions/go-oci/cache.fsx
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
#r "System.Formats.Tar"
|
||||
#r "/opt/fsi/packages/AWSSDK.Core.dll"
|
||||
#r "/opt/fsi/packages/AWSSDK.S3.dll"
|
||||
#r "/opt/fsi/packages/ZstdSharp.dll"
|
||||
#nowarn "0760"
|
||||
#load "projects.fsx"
|
||||
#load "cache-gc.fsx"
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.Formats.Tar
|
||||
open System.IO
|
||||
open System.Net
|
||||
open System.Net.Http
|
||||
open System.Net.Security
|
||||
open System.Security.Cryptography
|
||||
open System.Security.Cryptography.X509Certificates
|
||||
open System.Text
|
||||
open System.Text.Json
|
||||
open System.Text.Json.Serialization
|
||||
open System.Threading.Tasks
|
||||
open Amazon.Runtime
|
||||
open Amazon.S3
|
||||
open Amazon.S3.Model
|
||||
open Contract
|
||||
open Projects
|
||||
open ZstdSharp
|
||||
|
||||
[<CLIMutable>]
|
||||
type CacheObject =
|
||||
{ [<JsonPropertyName("key")>]
|
||||
Key: string
|
||||
[<JsonPropertyName("sha256")>]
|
||||
Sha256: string }
|
||||
|
||||
[<CLIMutable>]
|
||||
type Assignment =
|
||||
{ [<JsonPropertyName("exportKey")>]
|
||||
ExportKey: string
|
||||
[<JsonPropertyName("cacheVersion")>]
|
||||
CacheVersion: string
|
||||
[<JsonPropertyName("inputHashes")>]
|
||||
InputHashes: Dictionary<string, string> }
|
||||
|
||||
[<CLIMutable>]
|
||||
type CacheManifest =
|
||||
{ [<JsonPropertyName("schemaVersion")>]
|
||||
SchemaVersion: int
|
||||
[<JsonPropertyName("cacheVersion")>]
|
||||
CacheVersion: string
|
||||
[<JsonPropertyName("inputHashes")>]
|
||||
InputHashes: Dictionary<string, string>
|
||||
[<JsonPropertyName("payloadKey")>]
|
||||
PayloadKey: string }
|
||||
|
||||
let bucket = "project-delivery"
|
||||
let sourceRoot = "/source/repository"
|
||||
let buildCache = Environment.GetEnvironmentVariable "CACHE_MODE" = "parallel"
|
||||
let cacheName = if buildCache then "build" else "toolchain"
|
||||
|
||||
let hashText (value: string) =
|
||||
value
|
||||
|> Text.Encoding.UTF8.GetBytes
|
||||
|> SHA256.HashData
|
||||
|> Convert.ToHexString
|
||||
|> _.ToLowerInvariant()
|
||||
|
||||
let credential path = File.ReadAllText(path).Trim()
|
||||
let root = X509CertificateLoader.LoadCertificateFromFile "/cache-ca/ca.crt"
|
||||
|
||||
let validateCertificate (certificate: X509Certificate) (providedChain: X509Chain) (errors: SslPolicyErrors) =
|
||||
not (isNull certificate)
|
||||
&& not (isNull providedChain)
|
||||
&& not (errors.HasFlag SslPolicyErrors.RemoteCertificateNameMismatch)
|
||||
&& (use chain = new X509Chain()
|
||||
chain.ChainPolicy.TrustMode <- X509ChainTrustMode.CustomRootTrust
|
||||
chain.ChainPolicy.CustomTrustStore.Add root |> ignore
|
||||
chain.ChainPolicy.RevocationMode <- X509RevocationMode.NoCheck
|
||||
|
||||
providedChain.ChainElements
|
||||
|> Seq.iter (fun item -> chain.ChainPolicy.ExtraStore.Add item.Certificate |> ignore)
|
||||
|
||||
chain.Build(new X509Certificate2(certificate)))
|
||||
|
||||
type CacheClientFactory() =
|
||||
inherit HttpClientFactory()
|
||||
|
||||
override _.CreateHttpClient(_) =
|
||||
let handler = new HttpClientHandler()
|
||||
|
||||
handler.ServerCertificateCustomValidationCallback <-
|
||||
fun _ certificate chain errors -> validateCertificate certificate chain errors
|
||||
|
||||
new HttpClient(handler, true)
|
||||
|
||||
let configuration = AmazonS3Config()
|
||||
configuration.ServiceURL <- "https://s3.quorate.internal"
|
||||
configuration.AuthenticationRegion <- "us-east-1"
|
||||
configuration.ForcePathStyle <- true
|
||||
configuration.HttpClientFactory <- CacheClientFactory()
|
||||
|
||||
let client =
|
||||
AmazonS3Client(
|
||||
BasicAWSCredentials(credential "/cache-auth/accesskey", credential "/cache-auth/secretkey"),
|
||||
configuration
|
||||
)
|
||||
|
||||
let result (operation: Task<'T>) = operation.GetAwaiter().GetResult()
|
||||
|
||||
let sameHashes (left: Dictionary<string, string>) (right: Dictionary<string, string>) =
|
||||
not (isNull (box left))
|
||||
&& left.Count = right.Count
|
||||
&& left |> Seq.forall (fun item -> right.TryGetValue item.Key = (true, item.Value))
|
||||
|
||||
let assignment (project: Project) =
|
||||
let hashes = Dictionary<string, string>()
|
||||
|
||||
(if buildCache then [] else inputPaths sourceRoot project)
|
||||
|> List.iter (fun path -> hashes[path] <- sha256 (Path.Combine(sourceRoot, path)))
|
||||
|
||||
let version =
|
||||
[ "go-oci-toolchain-v1"
|
||||
project.Name
|
||||
$"profile={toolchainProfileName}"
|
||||
"apko=1.2.39-r0" ]
|
||||
@ ((if buildCache then
|
||||
Directory.EnumerateFiles "/orchestration"
|
||||
|> Seq.map (fun path -> Path.GetRelativePath("/orchestration", path))
|
||||
|> Seq.sort
|
||||
|> Seq.toList
|
||||
else
|
||||
[ "cache.fsx"
|
||||
"contract.fsx"
|
||||
"go-oci.apko.yaml"
|
||||
"projects.fsx"
|
||||
"toolchain.fsx" ])
|
||||
|> List.map (fun item ->
|
||||
let path = Path.Combine("/orchestration", item)
|
||||
$"{item}={sha256 path}"))
|
||||
|> String.concat "\n"
|
||||
|> fun value -> hashText (value + "\n")
|
||||
|
||||
{ ExportKey = $"projects/{project.Name}/{project.Workflow}/cache/{cacheName}/_latest.json"
|
||||
CacheVersion = version
|
||||
InputHashes = hashes }
|
||||
|
||||
let select assignment =
|
||||
try
|
||||
use reply =
|
||||
result (
|
||||
client.GetObjectAsync(
|
||||
GetObjectRequest(BucketName = bucket, Key = assignment.ExportKey),
|
||||
cancellationToken
|
||||
)
|
||||
)
|
||||
|
||||
use reader = new StreamReader(reply.ResponseStream, Encoding.UTF8)
|
||||
let manifest = reader.ReadToEnd() |> decode<CacheManifest>
|
||||
|
||||
if
|
||||
manifest.SchemaVersion <> 1
|
||||
|| manifest.CacheVersion <> assignment.CacheVersion
|
||||
|| not (sameHashes manifest.InputHashes assignment.InputHashes)
|
||||
then
|
||||
printfn "cache=miss component=%s reason=incompatible" cacheName
|
||||
None
|
||||
else
|
||||
let digest =
|
||||
Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension manifest.PayloadKey)
|
||||
|
||||
require
|
||||
(manifest.PayloadKey.StartsWith("objects/", StringComparison.Ordinal)
|
||||
&& manifest.PayloadKey.EndsWith(".tar.zst", StringComparison.Ordinal)
|
||||
&& validSha256 digest)
|
||||
"invalid toolchain cache payload"
|
||||
|
||||
let prefix =
|
||||
assignment.ExportKey[.. assignment.ExportKey.Length - "_latest.json".Length - 1]
|
||||
|
||||
Some
|
||||
{ Key = prefix + manifest.PayloadKey
|
||||
Sha256 = digest }
|
||||
with :? AmazonS3Exception as error when error.StatusCode = HttpStatusCode.NotFound ->
|
||||
printfn "cache=miss component=%s reason=absent" cacheName
|
||||
None
|
||||
|
||||
let download item destination =
|
||||
use reply =
|
||||
result (client.GetObjectAsync(GetObjectRequest(BucketName = bucket, Key = item.Key), cancellationToken))
|
||||
|
||||
use input = reply.ResponseStream
|
||||
use output = File.Create destination
|
||||
input.CopyTo output
|
||||
output.Dispose()
|
||||
require (sha256 destination = item.Sha256) "toolchain cache payload digest mismatch"
|
||||
|
||||
let restore item =
|
||||
let archive = Path.Combine("/tmp", Path.GetRandomFileName() + ".tar.zst")
|
||||
download item archive
|
||||
use input = File.OpenRead archive
|
||||
use zstd = new DecompressionStream(input, 131072, false, false)
|
||||
|
||||
let destination = if buildCache then "/cache" else "/toolchain"
|
||||
|
||||
TarFile.ExtractToDirectory(zstd, destination, false)
|
||||
File.Delete archive
|
||||
printfn "cache=restored component=%s" cacheName
|
||||
|
||||
let upload project assignment =
|
||||
let archive = Path.Combine("/tmp", Path.GetRandomFileName() + ".tar.zst")
|
||||
|
||||
use output = File.Create archive
|
||||
use zstd = new CompressionStream(output, 1, 131072, false)
|
||||
use writer = new TarWriter(zstd, TarEntryFormat.Pax, true)
|
||||
|
||||
if buildCache then
|
||||
Directory.EnumerateFileSystemEntries "/cache"
|
||||
|> Seq.map Path.GetFileName
|
||||
|> Seq.toList
|
||||
|> writeSelection writer "/cache"
|
||||
else
|
||||
writeSelection writer "/toolchain" (toolchainCachePaths project)
|
||||
|
||||
writer.Dispose()
|
||||
zstd.Dispose()
|
||||
output.Dispose()
|
||||
|
||||
let digest = sha256 archive
|
||||
let payloadKey = $"objects/{digest}.tar.zst"
|
||||
|
||||
let prefix =
|
||||
assignment.ExportKey[.. assignment.ExportKey.Length - "_latest.json".Length - 1]
|
||||
|
||||
use payload = File.OpenRead archive
|
||||
|
||||
result (
|
||||
client.PutObjectAsync(
|
||||
PutObjectRequest(BucketName = bucket, Key = prefix + payloadKey, InputStream = payload),
|
||||
cancellationToken
|
||||
)
|
||||
)
|
||||
|> ignore
|
||||
|
||||
let manifest =
|
||||
{ SchemaVersion = 1
|
||||
CacheVersion = assignment.CacheVersion
|
||||
InputHashes = assignment.InputHashes
|
||||
PayloadKey = payloadKey }
|
||||
|
||||
use body = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, json))
|
||||
|
||||
let request =
|
||||
PutObjectRequest(BucketName = bucket, Key = assignment.ExportKey, InputStream = body)
|
||||
|
||||
request.ContentType <- "application/json"
|
||||
result (client.PutObjectAsync(request, cancellationToken)) |> ignore
|
||||
CacheGc.prune client bucket assignment.ExportKey payloadKey cancellationToken
|
||||
File.Delete archive
|
||||
printfn "cache=exported component=%s" cacheName
|
||||
|
||||
let project =
|
||||
Environment.GetEnvironmentVariable "TARGETS"
|
||||
|> decode<Target array>
|
||||
|> selectProject
|
||||
|
||||
let cacheAssignment () =
|
||||
if not buildCache then
|
||||
validateProject sourceRoot project
|
||||
|
||||
assignment project
|
||||
|
||||
match Environment.GetEnvironmentVariable "CACHE_ACTION" with
|
||||
| "pull" when buildCache -> select (assignment project) |> Option.iter restore
|
||||
| "pull" -> Handoff.pull "/source/.ready" "/state" (fun () -> select (cacheAssignment ())) restore
|
||||
| "export" -> upload project (cacheAssignment ())
|
||||
| _ -> fail "invalid cache action"
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue