#r "/opt/fsi/packages/Oci.dll" #r "System.Formats.Tar" open System open System.Collections.Generic open System.Diagnostics open System.Formats.Tar open System.IO open System.IO.Compression open System.Net.Http open System.Threading open Quorate.Oci open System.Security.Cryptography open System.Text open System.Text.Json open System.Text.Json.Nodes open System.Text.Json.Serialization [] type Target = { [] Kind: string [] Repository: string } [] type Published = { Kind: string Repository: string Reference: string } [] type DeliveryResult = { SourceRevision: string Targets: Published array } let fail message = raise (InvalidOperationException message) let validRevision (value: string) = not (String.IsNullOrWhiteSpace value) && (value.Length = 40 || value.Length = 64) && (value |> Seq.forall (fun character -> character >= '0' && character <= '9' || character >= 'a' && character <= 'f')) let directory path = Directory.CreateDirectory path |> ignore path let runWith env executable arguments = let start = ProcessStartInfo executable start.UseShellExecute <- false for key, value in env do start.Environment[key] <- value for argument in arguments do start.ArgumentList.Add argument use child = Process.Start start if isNull child then fail $"failed to start {executable}" child.WaitForExit() if child.ExitCode <> 0 then fail $"{executable} failed with exit code {child.ExitCode}" let run executable arguments = runWith [] executable arguments let remove path = if File.Exists path then File.Delete path elif Directory.Exists path then Directory.Delete(path, true) let root () = let output = directory "/toolchain/image" run "/usr/bin/apko" [ "build" "--arch" "x86_64,arm64" "--lockfile" "/configuration/fsharp-runner.apko.lock.json" "--sbom=false" "/configuration/fsharp-runner.apko.yaml" "fsharp-runner:10.0.14" output ] let digestPath (root: string) (digest: string) = let parts = digest.Split(':', 2) if parts.Length <> 2 || parts[0] <> "sha256" then fail $"invalid digest {digest}" Path.Combine(root, parts[1]) let cleanName (name: string) = let clean = name.Replace('\\', '/').TrimStart('/') let parts = clean.Split('/', StringSplitOptions.RemoveEmptyEntries) |> Array.filter (fun part -> part <> ".") if parts |> Array.exists (fun part -> part = "..") then fail $"unsafe archive path {name}" String.Join('/', parts) let targetPath root relative = let target = Path.GetFullPath(Path.Combine(root, relative)) let prefix = Path.GetFullPath(root) + string Path.DirectorySeparatorChar if target <> Path.GetFullPath(root) && not <| target.StartsWith(prefix, StringComparison.Ordinal) then fail $"unsafe target path {relative}" target let applyEntry root prefix (entry: TarEntry) = let name = cleanName entry.Name if prefix = "" || name = prefix || name.StartsWith(prefix + "/", StringComparison.Ordinal) then let relative = name[prefix.Length ..].TrimStart('/') if relative <> "" then let leaf = Path.GetFileName relative let parent = Path.GetDirectoryName relative if leaf = ".wh..wh..opq" then let target = targetPath root parent if Directory.Exists target then for child in Directory.EnumerateFileSystemEntries target do remove child elif leaf.StartsWith(".wh.", StringComparison.Ordinal) then let target = targetPath root (Path.Combine(parent, leaf[4..])) remove target else let target = targetPath root relative match entry.EntryType with | TarEntryType.Directory -> directory target |> ignore File.SetUnixFileMode(target, entry.Mode) | TarEntryType.RegularFile | TarEntryType.V7RegularFile -> directory (Path.GetDirectoryName target) |> ignore remove target use output = File.Create target if not <| isNull entry.DataStream then entry.DataStream.CopyTo output output.Close() File.SetUnixFileMode(target, entry.Mode) | TarEntryType.SymbolicLink -> directory (Path.GetDirectoryName target) |> ignore remove target File.CreateSymbolicLink(target, entry.LinkName) |> ignore | TarEntryType.HardLink -> let link = cleanName entry.LinkName if link <> prefix && not <| link.StartsWith(prefix + "/", StringComparison.Ordinal) then fail $"hard link escapes selected path {entry.LinkName}" let source = targetPath root (link[prefix.Length ..].TrimStart('/')) directory (Path.GetDirectoryName target) |> ignore remove target File.Copy(source, target) File.SetUnixFileMode(target, File.GetUnixFileMode source) | _ -> () let applyLayer (root: string) (prefix: string) (mediaType: string) (path: string) = use input = File.OpenRead path use content: Stream = if mediaType.EndsWith("gzip", StringComparison.Ordinal) then new GZipStream(input, CompressionMode.Decompress) :> Stream else input :> Stream use reader = new TarReader(content) let mutable entry = reader.GetNextEntry() while not <| isNull entry do applyEntry root prefix entry entry <- reader.GetNextEntry() let extract archive prefix root = applyLayer root prefix "application/vnd.oci.image.layer.v1.tar+gzip" archive let archives = [ "sdk", "amd64", "10.0.403", "b7b3d8461cf98b07d2552ce9199702fa3ad010ac2671a5f8cd5136f7534e50f6855ba7bbb02b11494563f1d04156276a31cda1103b900e5eecd0d9473dc7b9c5" "sdk", "arm64", "10.0.403", "0966aa2b70f4677b8319be9df447a78a0dcd6164836425b55a68fd714f4023feb87b4cac21b9a996b79472ad54487cd0adcd2a9dfd570e2b2a1d26423a2baa9b" "runtime", "amd64", "10.0.14", "634326028645f38731c8c4e71a68356e4ca0d6b6607139a234ab760687f742063edaf33a1f20dda615742d2dc8300ca2ff857619d51fbb38090383b71334af89" "runtime", "arm64", "10.0.14", "99e1d6eb5d3de935853404fd378c77cbf813bd3ae094837b5a589000a492c289bf0d01d8a6bff99b4470572d9f85dc0b41239b5b7801e031197b83d527cdfeb2" ] let archivePath kind architecture = $"/toolchain/archives/{kind}-{architecture}.tar.gz" let acquire () = directory "/toolchain/archives" |> ignore use client = new HttpClient(Timeout = TimeSpan.FromMinutes 10.0) for kind, architecture, version, expected in archives do let family = if kind = "sdk" then "Sdk" else "Runtime" let rid = if architecture = "amd64" then "x64" else "arm64" let url = $"https://builds.dotnet.microsoft.com/dotnet/{family}/{version}/dotnet-{kind}-{version}-linux-{rid}.tar.gz" let target = archivePath kind architecture use reply = client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead).GetAwaiter().GetResult() reply.EnsureSuccessStatusCode() |> ignore use input = reply.Content.ReadAsStream() use output = File.Create target input.CopyTo output output.Dispose() use content = File.OpenRead target let actual = SHA512.HashData content |> Convert.ToHexString |> _.ToLowerInvariant() if actual <> expected then fail $"archive checksum mismatch: {kind}/{architecture}" if kind = "sdk" then extract target "" (directory $"/toolchain/sdk/{architecture}") printfn "toolchain=%s architecture=%s version=%s" kind architecture version root () let restore () = let architecture = match Runtime.InteropServices.RuntimeInformation.ProcessArchitecture with | Runtime.InteropServices.Architecture.X64 -> "amd64" | Runtime.InteropServices.Architecture.Arm64 -> "arm64" | other -> fail $"unsupported build architecture: {other}" let sdk = $"/toolchain/sdk/{architecture}" let dotnet = Path.Combine(sdk, "dotnet") let project = "/output/project" let package = "/source/repository/packages/fsharp-runner" remove project directory project |> ignore File.Copy(Path.Combine(package, "runner.csproj"), Path.Combine(project, "runner.csproj")) File.Copy(Path.Combine(package, "packages.lock.json"), Path.Combine(project, "packages.lock.json")) File.Copy(Path.Combine(package, "global.json"), "/output/global.json", true) File.Copy(Path.Combine(package, "Directory.Build.props"), "/output/Directory.Build.props", true) let oci = "/output/oci" remove oci directory oci |> ignore File.Copy(Path.Combine(package, "oci/Oci.csproj"), Path.Combine(oci, "Oci.csproj")) File.Copy(Path.Combine(package, "oci/OciTypes.cs"), Path.Combine(oci, "OciTypes.cs")) File.Copy(Path.Combine(package, "oci/OciLayout.cs"), Path.Combine(oci, "OciLayout.cs")) File.Copy(Path.Combine(package, "oci/RegistryAuth.cs"), Path.Combine(oci, "RegistryAuth.cs")) File.Copy(Path.Combine(package, "oci/Registry.cs"), Path.Combine(oci, "Registry.cs")) File.Copy(Path.Combine(package, "oci/Transport.cs"), Path.Combine(oci, "Transport.cs")) File.Copy(Path.Combine(package, "oci/Publisher.cs"), Path.Combine(oci, "Publisher.cs")) File.Copy(Path.Combine(package, "oci/AssemblyInfo.cs"), Path.Combine(oci, "AssemblyInfo.cs")) let host = "/output/hostexec" remove host directory host |> ignore File.Copy(Path.Combine(package, "hostexec/HostExec.csproj"), Path.Combine(host, "HostExec.csproj")) File.Copy(Path.Combine(package, "hostexec/AssemblyInfo.cs"), Path.Combine(host, "AssemblyInfo.cs")) File.Copy(Path.Combine(package, "hostexec/GnuProxy.cs"), Path.Combine(host, "GnuProxy.cs")) File.Copy(Path.Combine(package, "hostexec/Proxy.cs"), Path.Combine(host, "Proxy.cs")) File.Copy(Path.Combine(package, "hostexec/Program.cs"), Path.Combine(host, "Program.cs")) let ociTests = "/output/oci-tests" remove ociTests directory ociTests |> ignore File.Copy(Path.Combine(package, "oci-tests/Oci.Tests.csproj"), Path.Combine(ociTests, "Oci.Tests.csproj")) File.Copy(Path.Combine(package, "oci-tests/Program.cs"), Path.Combine(ociTests, "Program.cs")) let hostTests = "/output/hostexec-tests" remove hostTests directory hostTests |> ignore File.Copy( Path.Combine(package, "hostexec-tests/HostExec.Tests.csproj"), Path.Combine(hostTests, "HostExec.Tests.csproj") ) File.Copy(Path.Combine(package, "hostexec-tests/Program.cs"), Path.Combine(hostTests, "Program.cs")) let hostProbe = directory (Path.Combine(hostTests, "probe")) File.Copy( Path.Combine(package, "hostexec-tests/probe/HostExec.Probe.csproj"), Path.Combine(hostProbe, "HostExec.Probe.csproj") ) File.Copy(Path.Combine(package, "hostexec-tests/probe/Program.cs"), Path.Combine(hostProbe, "Program.cs")) let dotnetEnvironment = [ "DOTNET_CLI_HOME", "/tmp" "DOTNET_CLI_TELEMETRY_OPTOUT", "1" "DOTNET_MULTILEVEL_LOOKUP", "0" "DOTNET_NOLOGO", "1" "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "1" "DOTNET_ROOT", sdk "HOME", "/tmp" "NUGET_PACKAGES", "/cache/nuget" "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" ] runWith dotnetEnvironment dotnet [ "restore" "--locked-mode" "--packages" "/cache/nuget" (Path.Combine(project, "runner.csproj")) ] runWith dotnetEnvironment dotnet [ "build" "--configuration" "Release" "--no-restore" "--output" "/output/managed-out" (Path.Combine(project, "runner.csproj")) ] runWith dotnetEnvironment dotnet [ "publish" "--configuration" "Release" "--output" "/output/oci-out" (Path.Combine(oci, "Oci.csproj")) ] runWith dotnetEnvironment dotnet [ "publish" "--configuration" "Release" "--output" "/output/host-out" (Path.Combine(host, "HostExec.csproj")) ] for (project, projectFile, executable, output) in [ ociTests, "Oci.Tests.csproj", "Oci.Tests", "/output/oci-tests-out" hostTests, "HostExec.Tests.csproj", "HostExec.Tests", "/output/hostexec-tests-out" ] do printfn "test=restore name=%s" executable runWith dotnetEnvironment dotnet [ "restore" "--packages" "/cache/nuget" "-p:RunnerSourceRoot=/output" (Path.Combine(project, projectFile)) ] printfn "test=build name=%s" executable runWith dotnetEnvironment dotnet [ "build" "--configuration" "Release" "--no-restore" "--output" output "-p:RunnerSourceRoot=/output" (Path.Combine(project, projectFile)) ] printfn "test=run name=%s" executable runWith dotnetEnvironment (Path.Combine(output, executable)) [ "--progress"; "off"; "--no-ansi" ] let copy (source: string) (target: string) = directory (Path.GetDirectoryName target) |> ignore File.Copy(source, target, true) File.SetUnixFileMode(target, File.GetUnixFileMode source) let stage architecture = let root = $"/output/stage/{architecture}" remove root directory root |> ignore directory (Path.Combine(root, "work")) |> ignore extract (archivePath "runtime" architecture) "" (Path.Combine(root, "usr/share/dotnet")) extract (archivePath "sdk" architecture) "sdk/10.0.403/FSharp" (Path.Combine(root, "opt/fsi")) let packages = Path.Combine(root, "opt/fsi/packages") copy "/output/oci-out/Oci.dll" (Path.Combine(packages, "Oci.dll")) for name in [ "HostExec.dll"; "HostExec.deps.json"; "HostExec.runtimeconfig.json" ] do copy (Path.Combine("/output/host-out", name)) (Path.Combine(packages, name)) for name in [ "AWSSDK.Core.dll"; "AWSSDK.S3.dll"; "SharpCompress.dll"; "ZstdSharp.dll" ] do copy (Path.Combine("/output/managed-out", name)) (Path.Combine(packages, name)) root let entries (root: string) = let pending = Stack() let found = ResizeArray() pending.Push root while pending.Count > 0 do let current = pending.Pop() for path in Directory.EnumerateFileSystemEntries current do found.Add path if Directory.Exists path && isNull (FileInfo(path).LinkTarget) then pending.Push path found |> Seq.sortBy (fun path -> Path.GetRelativePath(root, path)) |> Seq.toArray let addTarEntry (writer: TarWriter) (root: string) (path: string) = let name = Path.GetRelativePath(root, path).Replace('\\', '/') let info = FileInfo path let entry = if not <| isNull info.LinkTarget then let item = PaxTarEntry(TarEntryType.SymbolicLink, name) item.LinkName <- info.LinkTarget item :> TarEntry elif Directory.Exists path then PaxTarEntry(TarEntryType.Directory, name + "/") :> TarEntry else let item = PaxTarEntry(TarEntryType.RegularFile, name) item.DataStream <- File.OpenRead path item :> TarEntry entry.Gid <- 0 entry.Uid <- 0 entry.ModificationTime <- DateTimeOffset.UnixEpoch entry.Mode <- File.GetUnixFileMode path writer.WriteEntry entry entry.DataStream |> Option.ofObj |> Option.iter _.Dispose() let whiteout (writer: TarWriter) name = let entry = PaxTarEntry(TarEntryType.RegularFile, name) entry.Gid <- 0 entry.Uid <- 0 entry.ModificationTime <- DateTimeOffset.UnixEpoch entry.Mode <- UnixFileMode.None entry.DataStream <- Stream.Null writer.WriteEntry entry let hash path = use input = File.OpenRead path SHA256.HashData input |> Convert.ToHexString |> _.ToLowerInvariant() let createLayer architecture = let root = stage architecture let tar = $"/output/{architecture}.tar" let gzip = $"/output/{architecture}.tar.gz" use output = File.Create tar use writer = new TarWriter(output, TarEntryFormat.Pax, true) for path in entries root do addTarEntry writer root path whiteout writer ".wh.dev" whiteout writer "usr/share/.wh.bash-completion" whiteout writer "usr/share/.wh.doc" whiteout writer "usr/share/.wh.man" writer.Dispose() output.Dispose() use input = File.OpenRead tar use compressed = File.Create gzip use encoder = new GZipStream(compressed, CompressionLevel.SmallestSize, true) input.CopyTo encoder encoder.Dispose() compressed.Dispose() let diff = hash tar let digest = hash gzip File.Delete tar diff, digest, FileInfo(gzip).Length, gzip let node path = let parsed = JsonNode.Parse(File.ReadAllText path) if isNull parsed then fail $"invalid JSON {path}" parsed let text (value: JsonNode) = value.GetValue() let jsonArray values = let result = JsonArray() for value in values do result.Add(value: string) result let blob (root: string) (bytes: byte array) = let digest = SHA256.HashData bytes |> Convert.ToHexString |> _.ToLowerInvariant() let path = Path.Combine(root, "blobs/sha256", digest) File.WriteAllBytes(path, bytes) $"sha256:{digest}", int64 bytes.Length let addLayer architecture = let image = "/output/layout" let index = node (Path.Combine(image, "index.json")) let descriptor = index["manifests"].AsArray() |> Seq.find (fun item -> text (item["platform"]["architecture"]) = architecture) let manifestPath = digestPath (Path.Combine(image, "blobs/sha256")) (text (descriptor["digest"])) let manifest = node manifestPath let configDescriptor = manifest["config"] let configPath = digestPath (Path.Combine(image, "blobs/sha256")) (text (configDescriptor["digest"])) let config = node configPath let diff, digest, size, layerPath = createLayer architecture let layerTarget = Path.Combine(image, "blobs/sha256", digest) File.Move(layerPath, layerTarget, true) let layer = JsonObject() layer["mediaType"] <- "application/vnd.oci.image.layer.v1.tar+gzip" layer["digest"] <- $"sha256:{digest}" layer["size"] <- size (manifest["layers"].AsArray()).Add(layer) let diffId = $"sha256:{diff}" let rootfs = config["rootfs"] let imageConfig = config["config"] (rootfs["diff_ids"].AsArray()).Add(diffId) imageConfig["Env"] <- jsonArray [| "DOTNET_MULTILEVEL_LOOKUP=0" "DOTNET_ROOT=/usr/share/dotnet" "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1" "HOME=/tmp" |] imageConfig["Entrypoint"] <- jsonArray [| "/usr/share/dotnet/dotnet"; "/opt/fsi/fsi.dll"; "--exec" |] imageConfig["User"] <- "65532:65532" imageConfig["WorkingDir"] <- "/work" let options = JsonSerializerOptions(WriteIndented = false) let configBytes = JsonSerializer.SerializeToUtf8Bytes(config, options) let configDigest, configSize = blob image configBytes configDescriptor["digest"] <- configDigest configDescriptor["size"] <- configSize let manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifest, options) let manifestDigest, manifestSize = blob image manifestBytes descriptor["digest"] <- manifestDigest descriptor["size"] <- manifestSize File.WriteAllBytes(Path.Combine(image, "index.json"), JsonSerializer.SerializeToUtf8Bytes(index, options)) let nameIndex () = let image = "/output/layout" let indexPath = Path.Combine(image, "index.json") let index = node indexPath let nested = JsonObject() nested["schemaVersion"] <- 2 nested["mediaType"] <- "application/vnd.oci.image.index.v1+json" nested["manifests"] <- index["manifests"].DeepClone() let options = JsonSerializerOptions(WriteIndented = false) let nestedBytes = JsonSerializer.SerializeToUtf8Bytes(nested, options) let nestedDigest, nestedSize = blob image nestedBytes File.WriteAllText("/output/image-digest", nestedDigest + "\n") let descriptor = JsonObject() descriptor["mediaType"] <- "application/vnd.oci.image.index.v1+json" descriptor["digest"] <- nestedDigest descriptor["size"] <- nestedSize let annotations = JsonObject() annotations["org.opencontainers.image.ref.name"] <- "10.0.14" descriptor["annotations"] <- annotations index["manifests"] <- JsonArray(descriptor) File.WriteAllBytes(indexPath, JsonSerializer.SerializeToUtf8Bytes(index, options)) let assemble () = addLayer "amd64" addLayer "arm64" nameIndex () let publish () = let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION" let targets = JsonSerializer.Deserialize(Environment.GetEnvironmentVariable "TARGETS") if not <| validRevision revision then fail "invalid source revision" if isNull targets || targets.Length <> 1 then fail "F# Runner delivery requires one target" let target = targets[0] if target.Kind <> "QuayOCI" || target.Repository <> "quorate/fsharp-runner" then fail "invalid F# Runner delivery target" let auth = RegistryAuth.Load("/auth/.dockerconfigjson", "registry.quorate.internal") use certificate = Registry.LoadRoot "/registry-ca/ca.crt" use registry = new Registry(Uri("https://registry.quorate.internal"), auth, certificate) let digest = registry .PushAsync(target.Repository, "/handoff/layout", "10.0.14", $"delivery-{revision}", CancellationToken.None) .GetAwaiter() .GetResult() let options = JsonSerializerOptions(PropertyNamingPolicy = JsonNamingPolicy.CamelCase) File.WriteAllText( "/result/delivery-result", JsonSerializer.Serialize( { SourceRevision = revision Targets = [| { Kind = target.Kind Repository = target.Repository Reference = $"registry.quorate.internal/{target.Repository}@{digest}" } |] }, options ) ) let build () = for path in Directory.EnumerateFiles("/toolchain/image", "*", SearchOption.AllDirectories) do copy path (Path.Combine("/output/layout", Path.GetRelativePath("/toolchain/image", path))) restore () assemble () match fsi.CommandLineArgs |> Array.skip 1 |> Array.toList with | [ "acquire" ] -> acquire () | [ "build" ] -> build () | [ "restore" ] -> restore () | [ "assemble" ] -> assemble () | [ "publish" ] -> publish () | _ -> fail "invalid operation"