ce7cb702218693d08ea6d83464d2457c9250027c
This commit is contained in:
commit
34c5f8963c
19 changed files with 145336 additions and 0 deletions
222
services/quay-custom/actions/base.fsx
Normal file
222
services/quay-custom/actions/base.fsx
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
#load "contract.fsx"
|
||||
|
||||
open System
|
||||
open System.Collections.Generic
|
||||
open System.IO
|
||||
open System.Net
|
||||
open System.Net.Http
|
||||
open System.Net.Http.Headers
|
||||
open System.Security.Cryptography
|
||||
open System.Text
|
||||
open System.Text.Json
|
||||
open System.Text.Json.Nodes
|
||||
open System.Text.RegularExpressions
|
||||
open Contract
|
||||
|
||||
type private Desc =
|
||||
{ Digest: string
|
||||
Size: int64
|
||||
MediaType: string
|
||||
Platform: JsonNode option }
|
||||
|
||||
let private mediaTypes =
|
||||
[ "application/vnd.oci.image.index.v1+json"
|
||||
"application/vnd.oci.image.manifest.v1+json"
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json"
|
||||
"application/vnd.docker.distribution.manifest.v2+json" ]
|
||||
|
||||
let private digestBytes (content: byte array) =
|
||||
let hash = SHA256.HashData content |> Convert.ToHexString
|
||||
$"sha256:{hash.ToLowerInvariant()}"
|
||||
|
||||
let private prop (name: string) (value: JsonElement) =
|
||||
let mutable result = Unchecked.defaultof<JsonElement>
|
||||
require (value.TryGetProperty(name, &result)) $"OCI document omitted {name}"
|
||||
result
|
||||
|
||||
let private strProp name value =
|
||||
let found = prop name value
|
||||
require (found.ValueKind = JsonValueKind.String) $"OCI {name} is not a string"
|
||||
|
||||
found.GetString()
|
||||
|> Option.ofObj
|
||||
|> Option.defaultWith (fun () -> fail $"OCI {name} is empty")
|
||||
|
||||
let private desc (value: JsonElement) =
|
||||
let size = prop "size" value
|
||||
require (size.ValueKind = JsonValueKind.Number) "OCI descriptor size is not a number"
|
||||
let mutable platform = Unchecked.defaultof<JsonElement>
|
||||
|
||||
{ Digest = strProp "digest" value
|
||||
Size = size.GetInt64()
|
||||
MediaType = strProp "mediaType" value
|
||||
Platform =
|
||||
if value.TryGetProperty("platform", &platform) then
|
||||
Some(JsonNode.Parse(platform.GetRawText()))
|
||||
else
|
||||
None }
|
||||
|
||||
let private request (method: HttpMethod) (endpoint: string) accept authorization =
|
||||
let value = new HttpRequestMessage(method, endpoint)
|
||||
|
||||
accept
|
||||
|> Option.iter (fun types ->
|
||||
types
|
||||
|> List.iter (fun item -> value.Headers.Accept.Add(MediaTypeWithQualityHeaderValue item)))
|
||||
|
||||
authorization |> Option.iter (fun item -> value.Headers.Authorization <- item)
|
||||
value
|
||||
|
||||
let private challenge (header: AuthenticationHeaderValue) =
|
||||
Regex.Matches(header.Parameter, "([A-Za-z]+)=\"([^\"]*)\"")
|
||||
|> Seq.cast<Match>
|
||||
|> Seq.map (fun item -> item.Groups[1].Value.ToLowerInvariant(), item.Groups[2].Value)
|
||||
|> Map.ofSeq
|
||||
|
||||
let private authToken (client: HttpClient) (response: HttpResponseMessage) =
|
||||
let header =
|
||||
response.Headers.WwwAuthenticate
|
||||
|> Seq.tryFind (fun item -> String.Equals(item.Scheme, "Bearer", StringComparison.OrdinalIgnoreCase))
|
||||
|> Option.defaultWith (fun () -> fail "registry did not return a Bearer challenge")
|
||||
|
||||
let values = challenge header
|
||||
|
||||
let value name =
|
||||
values
|
||||
|> Map.tryFind name
|
||||
|> Option.defaultWith (fun () -> fail $"registry challenge omitted {name}")
|
||||
|
||||
let realm = value "realm"
|
||||
let service = value "service" |> Uri.EscapeDataString
|
||||
let scope = value "scope" |> Uri.EscapeDataString
|
||||
let endpoint = $"{realm}?service={service}&scope={scope}"
|
||||
|
||||
use tokenRequest = request HttpMethod.Get endpoint None None
|
||||
use tokenResponse = client.Send tokenRequest
|
||||
require tokenResponse.IsSuccessStatusCode $"registry authentication failed with HTTP {int tokenResponse.StatusCode}"
|
||||
use document = JsonDocument.Parse(tokenResponse.Content.ReadAsStream())
|
||||
let root = document.RootElement
|
||||
let mutable token = Unchecked.defaultof<JsonElement>
|
||||
|
||||
require
|
||||
((root.TryGetProperty("token", &token)
|
||||
|| root.TryGetProperty("access_token", &token))
|
||||
&& token.ValueKind = JsonValueKind.String)
|
||||
"registry authentication returned no token"
|
||||
|
||||
token.GetString()
|
||||
|> Option.ofObj
|
||||
|> Option.defaultWith (fun () -> fail "registry token is empty")
|
||||
|
||||
let private send (client: HttpClient) (token: string option ref) make =
|
||||
let authorization =
|
||||
token.Value
|
||||
|> Option.map (fun value -> AuthenticationHeaderValue("Bearer", value))
|
||||
|
||||
use firstRequest = make authorization
|
||||
let first = client.Send(firstRequest, HttpCompletionOption.ResponseHeadersRead)
|
||||
|
||||
if first.StatusCode = HttpStatusCode.Unauthorized then
|
||||
let value = authToken client first
|
||||
token.Value <- Some value
|
||||
first.Dispose()
|
||||
use nextRequest = make (Some(AuthenticationHeaderValue("Bearer", value)))
|
||||
client.Send(nextRequest, HttpCompletionOption.ResponseHeadersRead)
|
||||
else
|
||||
first
|
||||
|
||||
let private regURL registry repository path =
|
||||
$"https://{registry}/v2/{repository}/{path}"
|
||||
|
||||
let private manifest client token registry repository digest =
|
||||
use response =
|
||||
send client token (fun authorization ->
|
||||
request HttpMethod.Get (regURL registry repository $"manifests/{digest}") (Some mediaTypes) authorization)
|
||||
|
||||
require response.IsSuccessStatusCode $"manifest read failed with HTTP {int response.StatusCode}"
|
||||
let content = response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult()
|
||||
require (digestBytes content = digest) $"manifest content does not match {digest}"
|
||||
content
|
||||
|
||||
let private blob client token registry repository root (item: Desc) =
|
||||
let path = Path.Combine(root, item.Digest[7..])
|
||||
|
||||
if not (File.Exists path) then
|
||||
use response =
|
||||
send client token (fun authorization ->
|
||||
request HttpMethod.Get (regURL registry repository $"blobs/{item.Digest}") None authorization)
|
||||
|
||||
require response.IsSuccessStatusCode $"blob read failed with HTTP {int response.StatusCode}"
|
||||
use source = response.Content.ReadAsStream()
|
||||
use target = File.Create path
|
||||
source.CopyTo target
|
||||
target.Dispose()
|
||||
require (FileInfo(path).Length = item.Size) $"OCI blob {item.Digest} has an invalid size"
|
||||
use content = File.OpenRead path
|
||||
let actual = SHA256.HashData content |> Convert.ToHexString |> _.ToLowerInvariant()
|
||||
require ($"sha256:{actual}" = item.Digest) $"OCI blob content does not match {item.Digest}"
|
||||
|
||||
let private save root digest content =
|
||||
require (digestBytes content = digest) $"OCI content does not match {digest}"
|
||||
File.WriteAllBytes(Path.Combine(root, digest[7..]), content)
|
||||
|
||||
let private platform (item: Desc) =
|
||||
item.Platform
|
||||
|> Option.map (fun node -> node["os"].GetValue<string>(), node["architecture"].GetValue<string>())
|
||||
|
||||
let pullFrom registry repository digest destination =
|
||||
use client = new HttpClient(Timeout = TimeSpan.FromMinutes 15.0)
|
||||
let token = ref None
|
||||
let blobs = Path.Combine(destination, "blobs/sha256")
|
||||
Directory.CreateDirectory blobs |> ignore
|
||||
let rootContent = manifest client token registry repository digest
|
||||
use rootDocument = JsonDocument.Parse rootContent
|
||||
let root = rootDocument.RootElement
|
||||
let children = prop "manifests" root
|
||||
require (children.ValueKind = JsonValueKind.Array) "OCI base is not a multi-platform index"
|
||||
|
||||
let selected =
|
||||
children.EnumerateArray()
|
||||
|> Seq.map desc
|
||||
|> Seq.filter (fun item ->
|
||||
platform item
|
||||
|> Option.exists (fun value -> value = ("linux", "amd64") || value = ("linux", "arm64")))
|
||||
|> Seq.toList
|
||||
|
||||
require
|
||||
(selected |> List.choose platform |> Set.ofList = Set.ofList [ "linux", "amd64"; "linux", "arm64" ])
|
||||
"OCI base omitted a target platform"
|
||||
|
||||
for child in selected do
|
||||
let content = manifest client token registry repository child.Digest
|
||||
save blobs child.Digest content
|
||||
use document = JsonDocument.Parse content
|
||||
let manifestRoot = document.RootElement
|
||||
let config = desc (prop "config" manifestRoot)
|
||||
blob client token registry repository blobs config
|
||||
|
||||
prop "layers" manifestRoot
|
||||
|> _.EnumerateArray()
|
||||
|> Seq.map desc
|
||||
|> Seq.iter (blob client token registry repository blobs)
|
||||
|
||||
let index = JsonObject()
|
||||
index["schemaVersion"] <- 2
|
||||
index["mediaType"] <- strProp "mediaType" root
|
||||
let manifests = JsonArray()
|
||||
|
||||
selected
|
||||
|> List.iter (fun item ->
|
||||
let value = JsonObject()
|
||||
value["mediaType"] <- item.MediaType
|
||||
value["digest"] <- item.Digest
|
||||
value["size"] <- item.Size
|
||||
item.Platform |> Option.iter (fun node -> value["platform"] <- node.DeepClone())
|
||||
manifests.Add value)
|
||||
|
||||
index["manifests"] <- manifests
|
||||
File.WriteAllText(Path.Combine(destination, "oci-layout"), "{\"imageLayoutVersion\":\"1.0.0\"}\n")
|
||||
File.WriteAllBytes(Path.Combine(destination, "index.json"), JsonSerializer.SerializeToUtf8Bytes(index, json))
|
||||
|
||||
let pull repository digest destination =
|
||||
pullFrom "quay.io" repository digest destination
|
||||
175
services/quay-custom/actions/build.fsx
Normal file
175
services/quay-custom/actions/build.fsx
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
#load "projects.fsx"
|
||||
#load "image.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open Contract
|
||||
open Image
|
||||
|
||||
let sourceRoot = "/source/repository"
|
||||
let output = "/output"
|
||||
let work = "/tmp/quay"
|
||||
let configRoot = Path.Combine(work, "config-tool")
|
||||
let web = Path.Combine(work, "endpoints/web.py")
|
||||
let go = "/toolchain/usr/lib/go/bin/go"
|
||||
|
||||
let copyTree source target =
|
||||
Directory.CreateDirectory target |> ignore
|
||||
|
||||
Directory.EnumerateDirectories(source, "*", SearchOption.AllDirectories)
|
||||
|> Seq.iter (fun path ->
|
||||
Directory.CreateDirectory(Path.Combine(target, Path.GetRelativePath(source, path)))
|
||||
|> ignore)
|
||||
|
||||
Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)
|
||||
|> Seq.iter (fun path -> File.Copy(path, Path.Combine(target, Path.GetRelativePath(source, path))))
|
||||
|
||||
copyTree "/toolchain/source/config-tool" configRoot
|
||||
Directory.CreateDirectory(Path.GetDirectoryName web) |> ignore
|
||||
File.Copy("/toolchain/source/endpoints/web.py", web)
|
||||
|
||||
let scanner = Path.Combine(work, "data/secscan_model")
|
||||
Directory.CreateDirectory scanner |> ignore
|
||||
|
||||
for name in [ "datatypes.py"; "secscan_v4_model.py" ] do
|
||||
File.Copy(Path.Combine("/toolchain/source/data/secscan_model", name), Path.Combine(scanner, name))
|
||||
|
||||
let patchPath =
|
||||
Path.Combine(sourceRoot, "services/quay-custom/patches/oauth-assign.patch")
|
||||
|
||||
requireSuccess "/toolchain/usr/bin/patch" [ "--batch"; "--forward"; "--strip=1"; $"--input={patchPath}" ] (Some work) []
|
||||
|> ignore
|
||||
|
||||
let inventoryPatch =
|
||||
Path.Combine(sourceRoot, "services/quay-custom/patches/inventory.patch")
|
||||
|
||||
requireSuccess
|
||||
"/toolchain/usr/bin/patch"
|
||||
[ "--batch"; "--forward"; "--strip=1"; $"--input={inventoryPatch}" ]
|
||||
(Some work)
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
let oidcFiles = [ "oauth/oidc.py"; "util/security/jwtutil.py" ]
|
||||
|
||||
for name in oidcFiles do
|
||||
let target = Path.Combine(work, name)
|
||||
Directory.CreateDirectory(Path.GetDirectoryName target) |> ignore
|
||||
File.Copy(Path.Combine("/toolchain/source", name), target)
|
||||
|
||||
let oidcPatch =
|
||||
Path.Combine(sourceRoot, "services/quay-custom/patches/oidc-eddsa.patch")
|
||||
|
||||
requireSuccess "/toolchain/usr/bin/patch" [ "--batch"; "--forward"; "--strip=1"; $"--input={oidcPatch}" ] (Some work) []
|
||||
|> ignore
|
||||
|
||||
let validator = Path.Combine(configRoot, "pkg/lib/shared/validators.go")
|
||||
let content = File.ReadAllText validator
|
||||
let oldValue = "} else if strings.Contains(err.Error(), \"invalid_grant\") {"
|
||||
|
||||
let newValue =
|
||||
"} else if strings.Contains(err.Error(), \"invalid_grant\") || (strings.Contains(err.Error(), \"invalid_request\") && strings.Contains(err.Error(), \"Errors.User.Code.Invalid\")) {"
|
||||
|
||||
require (content.Contains(oldValue, StringComparison.Ordinal)) "Quay OIDC validator source changed"
|
||||
File.WriteAllText(validator, content.Replace(oldValue, newValue, StringComparison.Ordinal))
|
||||
|
||||
let environment architecture =
|
||||
[ "CGO_ENABLED", "0"
|
||||
"GOCACHE", "/cache/go"
|
||||
"GOENV", "off"
|
||||
"GOOS", "linux"
|
||||
"GOARCH", architecture
|
||||
"GOFIPS140", "latest"
|
||||
"GOMODCACHE", "/toolchain/modules"
|
||||
"GOPATH", "/tmp/gopath"
|
||||
"GOPROXY", "off"
|
||||
"GOROOT", "/toolchain/usr/lib/go"
|
||||
"GOSUMDB", "sum.golang.org"
|
||||
"GOTOOLCHAIN", "local"
|
||||
"GOWORK", "off"
|
||||
"HOME", "/tmp" ]
|
||||
|
||||
let host =
|
||||
match Runtime.InteropServices.RuntimeInformation.ProcessArchitecture with
|
||||
| Runtime.InteropServices.Architecture.X64 -> "amd64"
|
||||
| Runtime.InteropServices.Architecture.Arm64 -> "arm64"
|
||||
| value -> fail $"unsupported builder architecture {value}"
|
||||
|
||||
requireSuccess
|
||||
go
|
||||
[ "test"
|
||||
"-vet=off"
|
||||
"-mod=readonly"
|
||||
"-tags=fips"
|
||||
"./commands"
|
||||
"./pkg/lib/fieldgroups/oidc" ]
|
||||
(Some configRoot)
|
||||
(environment host)
|
||||
|> ignore
|
||||
|
||||
let build architecture =
|
||||
let path = Path.Combine(output, "bin", architecture, "config-tool")
|
||||
Directory.CreateDirectory(Path.GetDirectoryName path) |> ignore
|
||||
|
||||
requireSuccess
|
||||
go
|
||||
[ "build"
|
||||
"-mod=readonly"
|
||||
"-tags=fips"
|
||||
"-trimpath"
|
||||
"-ldflags=-s -w -buildid="
|
||||
"-o"
|
||||
path
|
||||
"./cmd/config-tool" ]
|
||||
(Some configRoot)
|
||||
(environment architecture)
|
||||
|> ignore
|
||||
|
||||
File.SetUnixFileMode(
|
||||
path,
|
||||
UnixFileMode.UserRead
|
||||
||| UnixFileMode.UserWrite
|
||||
||| UnixFileMode.UserExecute
|
||||
||| UnixFileMode.GroupRead
|
||||
||| UnixFileMode.GroupExecute
|
||||
||| UnixFileMode.OtherRead
|
||||
||| UnixFileMode.OtherExecute
|
||||
)
|
||||
|
||||
architecture, path
|
||||
|
||||
let binaries = [ build "amd64"; build "arm64" ] |> Map.ofList
|
||||
|
||||
let baseFiles =
|
||||
[ "quay-registry/endpoints/web.py", web
|
||||
"quay-registry/data/secscan_model/datatypes.py", Path.Combine(scanner, "datatypes.py")
|
||||
"quay-registry/data/secscan_model/secscan_v4_model.py", Path.Combine(scanner, "secscan_v4_model.py") ]
|
||||
|
||||
let certifiRoot = "/toolchain/certifi"
|
||||
let sitePackages = "opt/app-root/lib/python3.12/site-packages"
|
||||
|
||||
let certifiFiles =
|
||||
Directory.EnumerateFiles(certifiRoot, "*", SearchOption.AllDirectories)
|
||||
|> Seq.map (fun path -> $"{sitePackages}/{Path.GetRelativePath(certifiRoot, path)}", path)
|
||||
|> Seq.toList
|
||||
|
||||
let certifiBundle = Path.Combine(certifiRoot, "certifi/cacert.pem")
|
||||
let whiteout = Path.Combine(output, "certifi-whiteout")
|
||||
File.WriteAllText(whiteout, "")
|
||||
|
||||
let trustFiles =
|
||||
[ $"{sitePackages}/.wh.certifi-2024.7.4.dist-info", whiteout
|
||||
"usr/share/pki/ca-trust-source/ca-bundle.trust.p11-kit",
|
||||
"/toolchain/ca/usr/share/pki/ca-trust-source/ca-bundle.trust.p11-kit"
|
||||
"etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", certifiBundle
|
||||
"etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt", certifiBundle ]
|
||||
|
||||
let files =
|
||||
baseFiles
|
||||
@ (oidcFiles
|
||||
|> List.map (fun name -> $"quay-registry/{name}", Path.Combine(work, name)))
|
||||
@ certifiFiles
|
||||
@ trustFiles
|
||||
|
||||
let digest = assemble "/toolchain/base" output binaries files
|
||||
printfn "build=image project=quay-custom digest=%s" digest
|
||||
174
services/quay-custom/actions/image-tests.fsx
Normal file
174
services/quay-custom/actions/image-tests.fsx
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
#r "System.Formats.Tar"
|
||||
#load "contract.fsx"
|
||||
#load "projects.fsx"
|
||||
#load "image.fsx"
|
||||
|
||||
open System
|
||||
open System.Formats.Tar
|
||||
open System.IO
|
||||
open System.IO.Compression
|
||||
open System.Security.Cryptography
|
||||
open System.Text
|
||||
open System.Text.Json
|
||||
open System.Text.Json.Nodes
|
||||
open Contract
|
||||
open Image
|
||||
|
||||
let require value message =
|
||||
if not value then
|
||||
invalidOp message
|
||||
|
||||
let temp = Path.Combine(Path.GetTempPath(), $"quay-image-{Guid.NewGuid():N}")
|
||||
let baseLayout = Path.Combine(temp, "base")
|
||||
let blobs = Path.Combine(baseLayout, "blobs/sha256")
|
||||
let output = Path.Combine(temp, "output")
|
||||
Directory.CreateDirectory blobs |> ignore
|
||||
Directory.CreateDirectory output |> ignore
|
||||
|
||||
let addBlob (content: byte array) =
|
||||
let digest = SHA256.HashData content |> Convert.ToHexString |> _.ToLowerInvariant()
|
||||
File.WriteAllBytes(Path.Combine(blobs, digest), content)
|
||||
$"sha256:{digest}", content.LongLength
|
||||
|
||||
let descriptor (mediaType: string) (digest: string) (size: int64) =
|
||||
let value = JsonObject()
|
||||
value["mediaType"] <- mediaType
|
||||
value["digest"] <- digest
|
||||
value["size"] <- size
|
||||
value
|
||||
|
||||
let platforms = JsonArray()
|
||||
|
||||
for architecture, mediaType in
|
||||
[ "amd64", "application/vnd.docker.distribution.manifest.v2+json"
|
||||
"arm64", "application/vnd.oci.image.manifest.v1+json" ] do
|
||||
let config = JsonObject()
|
||||
config["architecture"] <- architecture
|
||||
config["os"] <- "linux"
|
||||
let settings = JsonObject()
|
||||
settings["Labels"] <- JsonObject()
|
||||
config["config"] <- settings
|
||||
let rootfs = JsonObject()
|
||||
rootfs["type"] <- "layers"
|
||||
rootfs["diff_ids"] <- JsonArray()
|
||||
config["rootfs"] <- rootfs
|
||||
config["history"] <- JsonArray()
|
||||
|
||||
let configDigest, configSize =
|
||||
JsonSerializer.SerializeToUtf8Bytes(config, json) |> addBlob
|
||||
|
||||
let manifest = JsonObject()
|
||||
manifest["schemaVersion"] <- 2
|
||||
manifest["mediaType"] <- mediaType
|
||||
manifest["config"] <- descriptor "application/vnd.oci.image.config.v1+json" configDigest configSize
|
||||
manifest["layers"] <- JsonArray()
|
||||
let digest, size = JsonSerializer.SerializeToUtf8Bytes(manifest, json) |> addBlob
|
||||
let item = descriptor mediaType digest size
|
||||
let platform = JsonObject()
|
||||
platform["os"] <- "linux"
|
||||
platform["architecture"] <- architecture
|
||||
item["platform"] <- platform
|
||||
platforms.Add item
|
||||
|
||||
let index = JsonObject()
|
||||
index["schemaVersion"] <- 2
|
||||
index["mediaType"] <- "application/vnd.docker.distribution.manifest.list.v2+json"
|
||||
index["manifests"] <- platforms
|
||||
File.WriteAllBytes(Path.Combine(baseLayout, "index.json"), JsonSerializer.SerializeToUtf8Bytes(index, json))
|
||||
File.WriteAllText(Path.Combine(baseLayout, "oci-layout"), "{\"imageLayoutVersion\":\"1.0.0\"}\n")
|
||||
|
||||
let amd64 = Path.Combine(temp, "amd64")
|
||||
let arm64 = Path.Combine(temp, "arm64")
|
||||
let web = Path.Combine(temp, "web.py")
|
||||
let scanner = Path.Combine(temp, "scanner.py")
|
||||
let caBundle = Path.Combine(temp, "ca.pem")
|
||||
File.WriteAllText(amd64, "amd64")
|
||||
File.WriteAllText(arm64, "arm64")
|
||||
File.WriteAllText(web, "web")
|
||||
File.WriteAllText(scanner, "scanner")
|
||||
File.WriteAllText(caBundle, "ca")
|
||||
|
||||
let files =
|
||||
[ "quay-registry/endpoints/web.py", web
|
||||
"quay-registry/data/secscan_model/datatypes.py", scanner
|
||||
"quay-registry/data/secscan_model/secscan_v4_model.py", scanner
|
||||
"opt/app-root/lib/python3.12/site-packages/certifi/cacert.pem", caBundle
|
||||
"etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", caBundle ]
|
||||
|
||||
let digest =
|
||||
assemble baseLayout output (Map.ofList [ "amd64", amd64; "arm64", arm64 ]) files
|
||||
|
||||
require (validDigest digest) "assembled digest is invalid"
|
||||
|
||||
let layout = Path.Combine(output, "layout")
|
||||
|
||||
let resultIndex =
|
||||
JsonNode.Parse(File.ReadAllText(Path.Combine(layout, "index.json")))
|
||||
|
||||
let rootDesc = resultIndex["manifests"].AsArray()[0]
|
||||
require (rootDesc["digest"].GetValue<string>() = digest) "layout root digest changed"
|
||||
|
||||
let root =
|
||||
JsonNode.Parse(File.ReadAllText(Path.Combine(layout, "blobs/sha256", digest[7..])))
|
||||
|
||||
require (root["manifests"].AsArray().Count = 2) "platform count changed"
|
||||
|
||||
for item in root["manifests"].AsArray() do
|
||||
let platform = item["platform"].AsObject()
|
||||
let architecture = platform["architecture"].GetValue<string>()
|
||||
let manifestDigest = item["digest"].GetValue<string>()
|
||||
|
||||
let manifest =
|
||||
JsonNode.Parse(File.ReadAllText(Path.Combine(layout, "blobs/sha256", manifestDigest[7..])))
|
||||
|
||||
require (manifest["layers"].AsArray().Count = 1) $"{architecture} layer count changed"
|
||||
let configDesc = manifest["config"].AsObject()
|
||||
let configDigest = configDesc["digest"].GetValue<string>()
|
||||
|
||||
let config =
|
||||
JsonNode.Parse(File.ReadAllText(Path.Combine(layout, "blobs/sha256", configDigest[7..])))
|
||||
|
||||
let rootfs = config["rootfs"].AsObject()
|
||||
require (rootfs["diff_ids"].AsArray().Count = 1) $"{architecture} diff ID is absent"
|
||||
let settings = config["config"].AsObject()
|
||||
let labels = settings["Labels"].AsObject()
|
||||
require (labels["org.opencontainers.image.version"].GetValue<string>() = "v3.18.2") "Quay version label changed"
|
||||
require (labels["quorate.certifi"].GetValue<string>() = "2026.10.23") "certifi version label changed"
|
||||
let layers = manifest["layers"].AsArray()
|
||||
let layer = layers[0].AsObject()
|
||||
let layerDigest = layer["digest"].GetValue<string>()
|
||||
|
||||
use compressed =
|
||||
File.OpenRead(Path.Combine(layout, "blobs/sha256", layerDigest[7..]))
|
||||
|
||||
use gzip = new GZipStream(compressed, CompressionMode.Decompress)
|
||||
use reader = new TarReader(gzip)
|
||||
let found = ResizeArray<string>()
|
||||
let mutable entry = reader.GetNextEntry()
|
||||
|
||||
while not (isNull entry) do
|
||||
found.Add entry.Name
|
||||
|
||||
use content = new MemoryStream()
|
||||
entry.DataStream.CopyTo content
|
||||
let value = Encoding.UTF8.GetString(content.ToArray())
|
||||
|
||||
if entry.Name = "usr/bin/config-tool" then
|
||||
require (value = architecture) "config-tool content changed"
|
||||
require (entry.Mode.HasFlag UnixFileMode.UserExecute) "config-tool is not executable"
|
||||
elif entry.Name = "quay-registry/endpoints/web.py" then
|
||||
require (value = "web") "web.py content changed"
|
||||
require (not (entry.Mode.HasFlag UnixFileMode.UserExecute)) "web.py is executable"
|
||||
elif entry.Name.EndsWith(".pem", StringComparison.Ordinal) then
|
||||
require (value = "ca") "CA bundle content changed"
|
||||
require (entry.Mode.HasFlag UnixFileMode.GroupWrite) "CA bundle cannot accept internal roots"
|
||||
else
|
||||
require (value = "scanner") "scanner content changed"
|
||||
require (not (entry.Mode.HasFlag UnixFileMode.UserExecute)) "scanner source is executable"
|
||||
|
||||
entry <- reader.GetNextEntry()
|
||||
|
||||
require (Set.ofSeq found = Set.ofList ("usr/bin/config-tool" :: (files |> List.map fst))) "layer paths changed"
|
||||
|
||||
Directory.Delete(temp, true)
|
||||
printfn "Quay image tests passed"
|
||||
59
services/quay-custom/actions/image.fsx
Normal file
59
services/quay-custom/actions/image.fsx
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
#load "projects.fsx"
|
||||
#load "layers.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open System.Text.Json.Nodes
|
||||
open Contract
|
||||
open Projects
|
||||
|
||||
let private fileMode =
|
||||
UnixFileMode.UserRead
|
||||
||| UnixFileMode.UserWrite
|
||||
||| UnixFileMode.GroupRead
|
||||
||| UnixFileMode.OtherRead
|
||||
|
||||
let private execMode =
|
||||
fileMode
|
||||
||| UnixFileMode.UserExecute
|
||||
||| UnixFileMode.GroupExecute
|
||||
||| UnixFileMode.OtherExecute
|
||||
|
||||
let private labels (config: JsonNode) =
|
||||
let values = config["config"].AsObject()
|
||||
|
||||
let labels =
|
||||
match values["Labels"] with
|
||||
| null -> JsonObject()
|
||||
| current -> current.AsObject()
|
||||
|
||||
labels["org.opencontainers.image.source"] <- "https://github.com/quay/quay"
|
||||
labels["org.opencontainers.image.version"] <- quayVersion
|
||||
labels["org.opencontainers.image.revision"] <- quayRef
|
||||
labels["quorate.patch"] <- "Zitadel OIDC validation, OAuth-token assignment authentication, and scanner inventory"
|
||||
labels["quorate.certifi"] <- certifiVersion
|
||||
values["Labels"] <- labels
|
||||
|
||||
let assemble baseLayout output binaries files =
|
||||
let selection architecture =
|
||||
let binary =
|
||||
binaries
|
||||
|> Map.tryFind architecture
|
||||
|> Option.defaultWith (fun () -> fail $"missing {architecture} binary")
|
||||
|
||||
("usr/bin/config-tool", binary, execMode)
|
||||
:: (files
|
||||
|> List.map (fun ((path: string), source) ->
|
||||
let mode =
|
||||
// Quay startup appends internal CA roots to these bundles.
|
||||
if
|
||||
path.EndsWith("/certifi/cacert.pem", StringComparison.Ordinal)
|
||||
|| path.StartsWith("etc/pki/ca-trust/extracted/", StringComparison.Ordinal)
|
||||
then
|
||||
fileMode ||| UnixFileMode.GroupWrite
|
||||
else
|
||||
fileMode
|
||||
|
||||
path, source, mode))
|
||||
|
||||
Layers.assemble baseLayout output selection labels
|
||||
10
services/quay-custom/actions/kustomization.yaml
Normal file
10
services/quay-custom/actions/kustomization.yaml
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
configMapGenerator:
|
||||
- name: oci-base
|
||||
namespace: project-delivery
|
||||
files:
|
||||
- base.fsx
|
||||
- layers.fsx
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
167
services/quay-custom/actions/layers.fsx
Normal file
167
services/quay-custom/actions/layers.fsx
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
#load "contract.fsx"
|
||||
|
||||
open System
|
||||
open System.Formats.Tar
|
||||
open System.IO
|
||||
open System.IO.Compression
|
||||
open System.Security.Cryptography
|
||||
open System.Text.Json
|
||||
open System.Text.Json.Nodes
|
||||
open Contract
|
||||
|
||||
let private copyTree source target =
|
||||
Directory.CreateDirectory target |> ignore
|
||||
|
||||
Directory.EnumerateDirectories(source, "*", SearchOption.AllDirectories)
|
||||
|> Seq.iter (fun path ->
|
||||
Directory.CreateDirectory(Path.Combine(target, Path.GetRelativePath(source, path)))
|
||||
|> ignore)
|
||||
|
||||
Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)
|
||||
|> Seq.iter (fun path -> File.Copy(path, Path.Combine(target, Path.GetRelativePath(source, path))))
|
||||
|
||||
let private node path =
|
||||
JsonNode.Parse(File.ReadAllText path)
|
||||
|> Option.ofObj
|
||||
|> Option.defaultWith (fun () -> fail $"invalid JSON: {path}")
|
||||
|
||||
let private text (value: JsonNode) = value.GetValue<string>()
|
||||
let private blobPath root (digest: string) = Path.Combine(root, digest[7..])
|
||||
|
||||
let private blob root (content: byte array) =
|
||||
let digest = SHA256.HashData content |> Convert.ToHexString |> _.ToLowerInvariant()
|
||||
File.WriteAllBytes(Path.Combine(root, digest), content)
|
||||
$"sha256:{digest}", int64 content.Length
|
||||
|
||||
let private layer output architecture files =
|
||||
let tarPath = Path.Combine(output, $"{architecture}.tar")
|
||||
let gzipPath = tarPath + ".gz"
|
||||
|
||||
use tarOutput = File.Create tarPath
|
||||
use writer = new TarWriter(tarOutput, TarEntryFormat.Pax, true)
|
||||
|
||||
files
|
||||
|> List.sortBy (fun (path, _, _) -> path)
|
||||
|> List.iter (fun (path, source, mode) ->
|
||||
let link = FileInfo(source).LinkTarget
|
||||
|
||||
let entry =
|
||||
PaxTarEntry(
|
||||
(if isNull link then
|
||||
TarEntryType.RegularFile
|
||||
else
|
||||
TarEntryType.SymbolicLink),
|
||||
path
|
||||
)
|
||||
|
||||
entry.Gid <- 0
|
||||
entry.Uid <- 0
|
||||
entry.Mode <- mode
|
||||
entry.ModificationTime <- DateTimeOffset.UnixEpoch
|
||||
|
||||
if isNull link then
|
||||
entry.DataStream <- File.OpenRead source
|
||||
else
|
||||
entry.LinkName <- link
|
||||
|
||||
writer.WriteEntry entry
|
||||
|
||||
if not (isNull entry.DataStream) then
|
||||
entry.DataStream.Dispose())
|
||||
|
||||
writer.Dispose()
|
||||
tarOutput.Dispose()
|
||||
let diff = sha256 tarPath
|
||||
|
||||
use input = File.OpenRead tarPath
|
||||
use compressed = File.Create gzipPath
|
||||
use encoder = new GZipStream(compressed, CompressionLevel.SmallestSize, true)
|
||||
input.CopyTo encoder
|
||||
encoder.Dispose()
|
||||
compressed.Dispose()
|
||||
let digest = sha256 gzipPath
|
||||
File.Delete tarPath
|
||||
diff, digest, FileInfo(gzipPath).Length, gzipPath
|
||||
|
||||
let private addLayer layout output architecture files update (descriptor: JsonNode) =
|
||||
let blobs = Path.Combine(layout, "blobs/sha256")
|
||||
let manifest = node (blobPath blobs (text descriptor["digest"]))
|
||||
let configDesc = manifest["config"]
|
||||
let config = node (blobPath blobs (text configDesc["digest"]))
|
||||
let diff, digest, size, layerPath = layer output architecture files
|
||||
File.Move(layerPath, Path.Combine(blobs, digest), true)
|
||||
|
||||
let mediaType =
|
||||
if text descriptor["mediaType"] = "application/vnd.docker.distribution.manifest.v2+json" then
|
||||
"application/vnd.docker.image.rootfs.diff.tar.gzip"
|
||||
else
|
||||
"application/vnd.oci.image.layer.v1.tar+gzip"
|
||||
|
||||
let layerDesc = JsonObject()
|
||||
layerDesc["mediaType"] <- mediaType
|
||||
layerDesc["digest"] <- $"sha256:{digest}"
|
||||
layerDesc["size"] <- size
|
||||
manifest["layers"].AsArray().Add layerDesc
|
||||
let rootfs = config["rootfs"].AsObject()
|
||||
rootfs["diff_ids"].AsArray().Add($"sha256:{diff}")
|
||||
|
||||
match config["history"] with
|
||||
| null -> ()
|
||||
| history ->
|
||||
let item = JsonObject()
|
||||
item["created_by"] <- "quorate image delivery"
|
||||
history.AsArray().Add item
|
||||
|
||||
update config
|
||||
|
||||
let configDigest, configSize =
|
||||
JsonSerializer.SerializeToUtf8Bytes(config, json) |> blob blobs
|
||||
|
||||
configDesc["digest"] <- configDigest
|
||||
configDesc["size"] <- configSize
|
||||
|
||||
let manifestDigest, manifestSize =
|
||||
JsonSerializer.SerializeToUtf8Bytes(manifest, json) |> blob blobs
|
||||
|
||||
descriptor["digest"] <- manifestDigest
|
||||
descriptor["size"] <- manifestSize
|
||||
|
||||
let private platform (descriptor: JsonNode) =
|
||||
let platform = descriptor["platform"]
|
||||
text platform["os"], text platform["architecture"]
|
||||
|
||||
let assemble baseLayout output files update =
|
||||
let layout = Path.Combine(output, "layout")
|
||||
copyTree baseLayout layout
|
||||
let indexPath = Path.Combine(layout, "index.json")
|
||||
let index = node indexPath
|
||||
let descriptors = index["manifests"].AsArray()
|
||||
|
||||
descriptors
|
||||
|> Seq.iter (fun descriptor ->
|
||||
let os, architecture = platform descriptor
|
||||
require (os = "linux") $"unsupported OCI platform {os}/{architecture}"
|
||||
|
||||
addLayer layout output architecture (files architecture) update descriptor)
|
||||
|
||||
let root = JsonObject()
|
||||
root["schemaVersion"] <- 2
|
||||
root["mediaType"] <- text index["mediaType"]
|
||||
root["manifests"] <- descriptors.DeepClone()
|
||||
let blobs = Path.Combine(layout, "blobs/sha256")
|
||||
|
||||
let rootDigest, rootSize =
|
||||
JsonSerializer.SerializeToUtf8Bytes(root, json) |> blob blobs
|
||||
|
||||
let rootDesc = JsonObject()
|
||||
rootDesc["mediaType"] <- text root["mediaType"]
|
||||
rootDesc["digest"] <- rootDigest
|
||||
rootDesc["size"] <- rootSize
|
||||
let annotations = JsonObject()
|
||||
annotations["org.opencontainers.image.ref.name"] <- "delivery"
|
||||
rootDesc["annotations"] <- annotations
|
||||
index["mediaType"] <- "application/vnd.oci.image.index.v1+json"
|
||||
index["manifests"] <- JsonArray(rootDesc)
|
||||
File.WriteAllBytes(indexPath, JsonSerializer.SerializeToUtf8Bytes(index, json))
|
||||
File.WriteAllText(Path.Combine(output, "image-digest"), rootDigest + "\n")
|
||||
rootDigest
|
||||
81
services/quay-custom/actions/projects.fsx
Normal file
81
services/quay-custom/actions/projects.fsx
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
#load "contract.fsx"
|
||||
|
||||
open System.IO
|
||||
open Contract
|
||||
|
||||
type Binary =
|
||||
{ Module: string
|
||||
Package: string
|
||||
Destination: string }
|
||||
|
||||
type Project =
|
||||
{ Name: string
|
||||
Workflow: string
|
||||
Repository: string
|
||||
Modules: string list
|
||||
Binaries: Binary list
|
||||
Entrypoint: string list
|
||||
WorkingDir: string }
|
||||
|
||||
type ToolchainProfile =
|
||||
{ Name: string
|
||||
Packages: string list
|
||||
TestApplets: string list
|
||||
CachePaths: string list }
|
||||
|
||||
let quayVersion = "v3.18.2"
|
||||
let quayRef = "ba4ef919f228db442f74b8d7162ebc6113a82c10"
|
||||
|
||||
let quayDigest =
|
||||
"sha256:4456201ca6bfdc0bbe78ca7cfe262887df41147ddbf452dd8d51f8d330a41acb"
|
||||
|
||||
let certifiVersion = "2026.10.23"
|
||||
|
||||
let certifiUrl =
|
||||
"https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl"
|
||||
|
||||
let certifiHash = "e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1"
|
||||
|
||||
let caUrl =
|
||||
"https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/c/ca-certificates-2026.2.92_v9.0.319-90.3.el9_8.noarch.rpm"
|
||||
|
||||
let caHash = "5d29ecb5c3b2958308700381a119d623741d07e313d870dedfaf84e5c9df85cb"
|
||||
|
||||
let project =
|
||||
{ Name = "quay-custom"
|
||||
Workflow = "quay-custom-build"
|
||||
Repository = "quay/quay"
|
||||
Modules = []
|
||||
Binaries = []
|
||||
Entrypoint = []
|
||||
WorkingDir = "/" }
|
||||
|
||||
let selectProject (targets: Target array) =
|
||||
require
|
||||
(targets.Length = 1
|
||||
&& targets[0].Kind = "ZotOCI"
|
||||
&& targets[0].Repository = project.Repository)
|
||||
"invalid delivery target"
|
||||
|
||||
project
|
||||
|
||||
let validateProject sourceRoot _ =
|
||||
[ "services/quay-custom/patches/oauth-assign.patch"
|
||||
"services/quay-custom/patches/inventory.patch"
|
||||
"services/quay-custom/patches/oidc-eddsa.patch" ]
|
||||
|> List.iter (fun path -> require (File.Exists(Path.Combine(sourceRoot, path))) $"missing Quay patch: {path}")
|
||||
|
||||
let inputPaths sourceRoot _ =
|
||||
[ "services/quay-custom/patches/oauth-assign.patch"
|
||||
"services/quay-custom/patches/inventory.patch"
|
||||
"services/quay-custom/patches/oidc-eddsa.patch"
|
||||
"services/quay-custom/actions/base.fsx"
|
||||
"services/quay-custom/actions/layers.fsx"
|
||||
"services/quay-custom/actions/quay.apko.yaml"
|
||||
"services/quay-custom/actions/toolchain.fsx" ]
|
||||
|> List.filter (fun item -> File.Exists(Path.Combine(sourceRoot, item)))
|
||||
|
||||
let toolchainProfileName = "quay-custom"
|
||||
|
||||
let toolchainCachePaths (_: Project) =
|
||||
[ "base"; "modules"; "source"; "usr/bin"; "usr/lib"; "certifi"; "ca" ]
|
||||
49
services/quay-custom/actions/publish.fsx
Normal file
49
services/quay-custom/actions/publish.fsx
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
#nowarn "0760"
|
||||
|
||||
#r "/opt/fsi/packages/Oci.dll"
|
||||
#load "projects.fsx"
|
||||
|
||||
open System
|
||||
open System.IO
|
||||
open System.Text.Json
|
||||
open Contract
|
||||
open Quorate.Oci
|
||||
open Projects
|
||||
|
||||
let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
|
||||
let targets = Environment.GetEnvironmentVariable "TARGETS" |> decode<Target array>
|
||||
let project = selectProject targets
|
||||
let layout = "/handoff/layout"
|
||||
let digest = File.ReadAllText("/handoff/image-digest").Trim()
|
||||
|
||||
require (validRevision revision) "invalid source revision"
|
||||
require (validDigest digest) "invalid image digest"
|
||||
|
||||
let publish () =
|
||||
let auth = RegistryAuth.Load("/auth/.dockerconfigjson", "zot.quorate.internal")
|
||||
use root = Registry.LoadRoot "/registry-ca/ca.crt"
|
||||
use registry = Registry(Uri("https://zot.quorate.internal"), auth, root)
|
||||
|
||||
registry
|
||||
.PushAsync(project.Repository, layout, "delivery", $"delivery-{revision}", cancellationToken)
|
||||
.GetAwaiter()
|
||||
.GetResult()
|
||||
|
||||
let published = publish ()
|
||||
|
||||
require (published = digest) "published digest mismatch"
|
||||
directory "/result" |> ignore
|
||||
|
||||
File.WriteAllText(
|
||||
"/result/delivery-result",
|
||||
JsonSerializer.Serialize(
|
||||
{ SourceRevision = revision
|
||||
Targets =
|
||||
[| { Kind = "ZotOCI"
|
||||
Repository = project.Repository
|
||||
Reference = $"zot.quorate.internal/{project.Repository}@{digest}" } |] },
|
||||
json
|
||||
)
|
||||
)
|
||||
|
||||
printfn "publish=%s digest=%s" project.Repository digest
|
||||
9
services/quay-custom/actions/quay.apko.yaml
Normal file
9
services/quay-custom/actions/quay.apko.yaml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
contents:
|
||||
keyring:
|
||||
- https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
|
||||
repositories:
|
||||
- https://packages.wolfi.dev/os
|
||||
packages:
|
||||
- ca-certificates-bundle=20261020-r0
|
||||
- patch=2.8.1-r1
|
||||
- libarchive-tools=3.8.10-r0
|
||||
177
services/quay-custom/actions/toolchain.fsx
Normal file
177
services/quay-custom/actions/toolchain.fsx
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
#r "System.Formats.Tar"
|
||||
#load "projects.fsx"
|
||||
#load "base.fsx"
|
||||
|
||||
open System
|
||||
open System.Formats.Tar
|
||||
open System.IO
|
||||
open System.IO.Compression
|
||||
open System.Net.Http
|
||||
open System.Runtime.InteropServices
|
||||
open Base
|
||||
open Contract
|
||||
open Projects
|
||||
|
||||
let root = "/toolchain"
|
||||
|
||||
do
|
||||
let architecture =
|
||||
match RuntimeInformation.ProcessArchitecture with
|
||||
| Architecture.X64 -> "x86_64"
|
||||
| Architecture.Arm64 -> "aarch64"
|
||||
| value -> fail $"unsupported builder architecture {value}"
|
||||
|
||||
let archive = "/tmp/tools.tar"
|
||||
let export = "/tmp/tools"
|
||||
|
||||
requireSuccess
|
||||
"/usr/bin/apko"
|
||||
[ "build"
|
||||
"--arch"
|
||||
architecture
|
||||
"--sbom=false"
|
||||
"/orchestration/go-oci.apko.yaml"
|
||||
"quay-tools:root"
|
||||
archive ]
|
||||
None
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
Directory.CreateDirectory export |> ignore
|
||||
TarFile.ExtractToDirectory(archive, export, false)
|
||||
|
||||
let wanted (path: string) =
|
||||
path.StartsWith("usr/bin/", StringComparison.Ordinal)
|
||||
|| path.StartsWith("usr/lib/", StringComparison.Ordinal)
|
||||
|
||||
let extract layer =
|
||||
use input = File.OpenRead layer
|
||||
use gzip = new GZipStream(input, CompressionMode.Decompress)
|
||||
use reader = new TarReader(gzip)
|
||||
let mutable entry = reader.GetNextEntry()
|
||||
|
||||
while not (isNull entry) do
|
||||
let path = entry.Name.TrimStart('.', '/')
|
||||
|
||||
if wanted path then
|
||||
let target = Path.Combine(root, path)
|
||||
|
||||
match entry.EntryType with
|
||||
| TarEntryType.Directory -> Directory.CreateDirectory target |> ignore
|
||||
| TarEntryType.RegularFile ->
|
||||
Directory.CreateDirectory(Path.GetDirectoryName target) |> ignore
|
||||
use output = File.Create target
|
||||
|
||||
if entry.Length > 0L then
|
||||
entry.DataStream.CopyTo output
|
||||
|
||||
output.Dispose()
|
||||
File.SetUnixFileMode(target, entry.Mode)
|
||||
| TarEntryType.SymbolicLink ->
|
||||
Directory.CreateDirectory(Path.GetDirectoryName target) |> ignore
|
||||
File.CreateSymbolicLink(target, entry.LinkName) |> ignore
|
||||
| _ -> ()
|
||||
|
||||
entry <- reader.GetNextEntry()
|
||||
|
||||
Directory.EnumerateFiles(export, "*.tar.gz") |> Seq.iter extract
|
||||
|
||||
let sourceArchive = "/tmp/quay.tar.gz"
|
||||
|
||||
use client = new HttpClient(Timeout = TimeSpan.FromMinutes 10.0)
|
||||
|
||||
use response =
|
||||
client.GetAsync($"https://github.com/quay/quay/archive/{quayRef}.tar.gz").GetAwaiter().GetResult()
|
||||
|
||||
require response.IsSuccessStatusCode $"Quay source download failed with HTTP {int response.StatusCode}"
|
||||
|
||||
use source = response.Content.ReadAsStream()
|
||||
use target = File.Create sourceArchive
|
||||
source.CopyTo target
|
||||
target.Dispose()
|
||||
|
||||
let incoming = Path.Combine(root, "incoming")
|
||||
Directory.CreateDirectory incoming |> ignore
|
||||
extractGzip sourceArchive incoming
|
||||
|
||||
let sourceRoot =
|
||||
Directory.EnumerateDirectories incoming
|
||||
|> Seq.tryExactlyOne
|
||||
|> Option.defaultWith (fun () -> fail "Quay source archive has an invalid root")
|
||||
|
||||
Directory.Move(sourceRoot, Path.Combine(root, "source"))
|
||||
|
||||
let goArchitecture, goChecksum, loaderName =
|
||||
match RuntimeInformation.ProcessArchitecture with
|
||||
| Architecture.X64 ->
|
||||
"amd64", "65efbc80fdc1fa2cd2631cad5b481777a68e8725f34e89f50fc6ac714e379b40", "ld-linux-x86-64.so.2"
|
||||
| Architecture.Arm64 ->
|
||||
"arm64", "a06e99309dbcbdca1a8ccd2282c3ee146e55930886e26b4f1193269587165f49", "ld-linux-aarch64.so.1"
|
||||
| value -> fail $"unsupported builder architecture {value}"
|
||||
|
||||
let goArchive = "/tmp/go.tar.gz"
|
||||
|
||||
use goResponse =
|
||||
client.GetAsync($"https://dl.google.com/go/go1.27.3.linux-{goArchitecture}.tar.gz").GetAwaiter().GetResult()
|
||||
|
||||
require goResponse.IsSuccessStatusCode $"Go download failed with HTTP {int goResponse.StatusCode}"
|
||||
use goSource = goResponse.Content.ReadAsStream()
|
||||
use goTarget = File.Create goArchive
|
||||
goSource.CopyTo goTarget
|
||||
goTarget.Dispose()
|
||||
require (sha256 goArchive = goChecksum) "Go archive checksum mismatch"
|
||||
let goParent = Path.Combine(root, "usr/lib")
|
||||
Directory.CreateDirectory goParent |> ignore
|
||||
extractGzip goArchive goParent
|
||||
|
||||
let download (url: string) (path: string) checksum =
|
||||
use response = client.GetAsync(url).GetAwaiter().GetResult()
|
||||
require response.IsSuccessStatusCode $"trust input download failed with HTTP {int response.StatusCode}"
|
||||
use input = response.Content.ReadAsStream()
|
||||
use output = File.Create path
|
||||
input.CopyTo output
|
||||
output.Dispose()
|
||||
require (sha256 path = checksum) "trust input checksum mismatch"
|
||||
|
||||
let certifiArchive = "/tmp/certifi.whl"
|
||||
download certifiUrl certifiArchive certifiHash
|
||||
ZipFile.ExtractToDirectory(certifiArchive, Path.Combine(root, "certifi"))
|
||||
let caArchive = "/tmp/ca-certificates.rpm"
|
||||
download caUrl caArchive caHash
|
||||
let caRoot = directory (Path.Combine(root, "ca"))
|
||||
|
||||
requireSuccess
|
||||
(Path.Combine(root, "usr/lib", loaderName))
|
||||
[ "--library-path"
|
||||
Path.Combine(root, "usr/lib")
|
||||
Path.Combine(root, "usr/bin/bsdtar")
|
||||
"-xf"
|
||||
caArchive
|
||||
"-C"
|
||||
caRoot
|
||||
"./usr/share/pki/ca-trust-source/ca-bundle.trust.p11-kit" ]
|
||||
None
|
||||
[]
|
||||
|> ignore
|
||||
|
||||
let go = Path.Combine(root, "usr/lib/go/bin/go")
|
||||
let configRoot = Path.Combine(root, "source/config-tool")
|
||||
|
||||
let environment =
|
||||
[ "GOCACHE", "/tmp/go-cache"
|
||||
"GOENV", "off"
|
||||
"GOMODCACHE", Path.Combine(root, "modules")
|
||||
"GOPATH", "/tmp/gopath"
|
||||
"GOPROXY", "https://proxy.golang.org"
|
||||
"GOROOT", Path.Combine(root, "usr/lib/go")
|
||||
"GOSUMDB", "sum.golang.org"
|
||||
"GOTOOLCHAIN", "local"
|
||||
"GOWORK", "off"
|
||||
"HOME", "/tmp/home" ]
|
||||
|
||||
let version = requireSuccess go [ "version" ] None environment
|
||||
require (version.Trim().StartsWith("go version go1.27.3 ", StringComparison.Ordinal)) "Go toolchain version changed"
|
||||
Directory.CreateDirectory(Path.Combine(root, "modules")) |> ignore
|
||||
requireSuccess go [ "mod"; "download" ] (Some configRoot) environment |> ignore
|
||||
pull "projectquay/quay" quayDigest (Path.Combine(root, "base"))
|
||||
printfn "toolchain=primary quay=%s" quayRef
|
||||
3
services/quay-custom/k8s/kustomizeconfig.yaml
Normal file
3
services/quay-custom/k8s/kustomizeconfig.yaml
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
images:
|
||||
- path: spec/templates/container/image
|
||||
kind: WorkflowTemplate
|
||||
100
services/quay-custom/k8s/network-policy.yaml
Normal file
100
services/quay-custom/k8s/network-policy.yaml
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: tools
|
||||
namespace: project-delivery
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
delivery.quorate.dev/stage: quay-toolchain
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: kube-system
|
||||
k8s-app: kube-dns
|
||||
toPorts:
|
||||
- ports:
|
||||
- {port: "53", protocol: UDP}
|
||||
- {port: "53", protocol: TCP}
|
||||
rules:
|
||||
dns:
|
||||
- {matchName: packages.wolfi.dev}
|
||||
- {matchName: dl.google.com}
|
||||
- {matchName: github.com}
|
||||
- {matchPattern: "*.github.com"}
|
||||
- {matchName: proxy.golang.org}
|
||||
- {matchName: storage.googleapis.com}
|
||||
- {matchName: sum.golang.org}
|
||||
- {matchName: files.pythonhosted.org}
|
||||
- {matchName: cdn-ubi.redhat.com}
|
||||
- {matchName: quay.io}
|
||||
- {matchPattern: "*.quay.io"}
|
||||
- toFQDNs:
|
||||
- {matchName: packages.wolfi.dev}
|
||||
- {matchName: dl.google.com}
|
||||
- {matchName: github.com}
|
||||
- {matchPattern: "*.github.com"}
|
||||
- {matchName: proxy.golang.org}
|
||||
- {matchName: storage.googleapis.com}
|
||||
- {matchName: sum.golang.org}
|
||||
- {matchName: files.pythonhosted.org}
|
||||
- {matchName: cdn-ubi.redhat.com}
|
||||
- {matchName: quay.io}
|
||||
- {matchPattern: "*.quay.io"}
|
||||
toPorts:
|
||||
- ports:
|
||||
- {port: "443", protocol: TCP}
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: publish
|
||||
namespace: project-delivery
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
delivery.quorate.dev/stage: quay-publish
|
||||
egress:
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: kube-system
|
||||
k8s-app: kube-dns
|
||||
toPorts:
|
||||
- ports:
|
||||
- {port: "53", protocol: UDP}
|
||||
- {port: "53", protocol: TCP}
|
||||
rules:
|
||||
dns:
|
||||
- {matchName: zot.quorate.internal}
|
||||
- toCIDR: [172.28.129.182/32]
|
||||
toPorts:
|
||||
- ports:
|
||||
- {port: "443", protocol: TCP}
|
||||
- toEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: zot
|
||||
app.kubernetes.io/name: zot
|
||||
app.kubernetes.io/instance: zot
|
||||
toPorts:
|
||||
- ports:
|
||||
- {port: "5000", protocol: TCP}
|
||||
---
|
||||
apiVersion: cilium.io/v2
|
||||
kind: CiliumNetworkPolicy
|
||||
metadata:
|
||||
name: zot-in
|
||||
namespace: zot
|
||||
spec:
|
||||
endpointSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: zot
|
||||
app.kubernetes.io/instance: zot
|
||||
ingress:
|
||||
- fromEndpoints:
|
||||
- matchLabels:
|
||||
k8s:io.kubernetes.pod.namespace: project-delivery
|
||||
io.cilium.k8s.policy.serviceaccount: project-delivery-executor
|
||||
delivery.quorate.dev/stage: quay-publish
|
||||
toPorts:
|
||||
- ports:
|
||||
- {port: "5000", protocol: TCP}
|
||||
17
services/quay-custom/k8s/project.yaml
Normal file
17
services/quay-custom/k8s/project.yaml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
apiVersion: delivery.quorate.dev/v1alpha1
|
||||
kind: ProjectDelivery
|
||||
metadata:
|
||||
name: image
|
||||
namespace: project-delivery
|
||||
spec:
|
||||
source:
|
||||
repository: https://forge.quorate.internal/quorate/cluster
|
||||
continuous:
|
||||
ref: refs/heads/master
|
||||
paths:
|
||||
- services/quay-custom
|
||||
targets:
|
||||
- kind: ZotOCI
|
||||
repository: quay/quay
|
||||
workflowTemplateRef:
|
||||
name: build
|
||||
45
services/quay-custom/k8s/vso.yaml
Normal file
45
services/quay-custom/k8s/vso.yaml
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: zot-vso
|
||||
namespace: project-delivery
|
||||
automountServiceAccountToken: false
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultAuth
|
||||
metadata:
|
||||
name: zot
|
||||
namespace: project-delivery
|
||||
spec:
|
||||
method: kubernetes
|
||||
mount: kubernetes
|
||||
kubernetes:
|
||||
role: quay-custom-writer
|
||||
serviceAccount: quay-custom-zot-vso
|
||||
vaultConnectionRef: vault/vault
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: writer
|
||||
namespace: project-delivery
|
||||
spec:
|
||||
type: kv-v2
|
||||
mount: secret
|
||||
path: zot/quay-custom
|
||||
refreshAfter: 1h
|
||||
destination:
|
||||
name: quay-custom-writer
|
||||
create: true
|
||||
overwrite: true
|
||||
type: kubernetes.io/dockerconfigjson
|
||||
transformation:
|
||||
excludeRaw: true
|
||||
excludes: [.*]
|
||||
templates:
|
||||
.dockerconfigjson:
|
||||
text: |
|
||||
{{- $user := get .Secrets "username" -}}
|
||||
{{- $password := get .Secrets "password" -}}
|
||||
{"auths":{"zot.quorate.internal":{"username":"{{ $user }}","password":"{{ $password }}","auth":"{{ printf "%s:%s" $user $password | b64enc }}"}}}
|
||||
vaultAuthRef: quay-custom-zot
|
||||
50
services/quay-custom/k8s/workflow-patch.yaml
Normal file
50
services/quay-custom/k8s/workflow-patch.yaml
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
- op: replace
|
||||
path: /metadata/labels/app.kubernetes.io~1instance
|
||||
value: quay-custom
|
||||
- op: replace
|
||||
path: /spec/podMetadata/labels/delivery.quorate.dev~1project
|
||||
value: quay-custom
|
||||
- op: replace
|
||||
path: /spec/templates/5/metadata/labels/delivery.quorate.dev~1stage
|
||||
value: quay-toolchain
|
||||
- op: replace
|
||||
path: /spec/templates/7/metadata/labels/delivery.quorate.dev~1stage
|
||||
value: quay-publish
|
||||
- op: replace
|
||||
path: /spec/templates/1/volumes/0
|
||||
value: &orchestration
|
||||
name: orchestration
|
||||
projected:
|
||||
defaultMode: 0444
|
||||
sources:
|
||||
- configMap:
|
||||
name: go-oci-orch
|
||||
items:
|
||||
- {key: cache.fsx, path: cache.fsx}
|
||||
- {key: cache-gc.fsx, path: cache-gc.fsx}
|
||||
- {key: contract.fsx, path: contract.fsx}
|
||||
- {key: initialize.fsx, path: initialize.fsx}
|
||||
- {key: source.fsx, path: source.fsx}
|
||||
- configMap:
|
||||
name: quay-custom-project
|
||||
- op: replace
|
||||
path: /spec/templates/3/volumes/0
|
||||
value: *orchestration
|
||||
- op: replace
|
||||
path: /spec/templates/4/volumes/0
|
||||
value: *orchestration
|
||||
- op: replace
|
||||
path: /spec/templates/5/volumes/0
|
||||
value: *orchestration
|
||||
- op: replace
|
||||
path: /spec/templates/6/volumes/0
|
||||
value: *orchestration
|
||||
- op: replace
|
||||
path: /spec/templates/7/volumes/0
|
||||
value: *orchestration
|
||||
- op: replace
|
||||
path: /spec/templates/7/volumes/1/secret/secretName
|
||||
value: quay-custom-writer
|
||||
- op: replace
|
||||
path: /spec/templates/8/volumes/0
|
||||
value: *orchestration
|
||||
52
services/quay-custom/kustomization.yaml
Normal file
52
services/quay-custom/kustomization.yaml
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namePrefix: quay-custom-
|
||||
configurations:
|
||||
- k8s/kustomizeconfig.yaml
|
||||
images:
|
||||
- name: registry.quorate.internal/quorate/fsharp-runner
|
||||
digest: sha256:d7b904f4f0cae9fdba7c7345cdf08c91162245834a8a13658d93396272aeec83
|
||||
resources:
|
||||
- ../project-delivery/k8s/workflows/go-oci/parallel
|
||||
- k8s/network-policy.yaml
|
||||
- k8s/project.yaml
|
||||
- k8s/vso.yaml
|
||||
configMapGenerator:
|
||||
- name: project
|
||||
namespace: project-delivery
|
||||
files:
|
||||
- base.fsx=actions/base.fsx
|
||||
- build.fsx=actions/build.fsx
|
||||
- image.fsx=actions/image.fsx
|
||||
- layers.fsx=actions/layers.fsx
|
||||
- projects.fsx=actions/projects.fsx
|
||||
- publish.fsx=actions/publish.fsx
|
||||
- toolchain.fsx=actions/toolchain.fsx
|
||||
- go-oci.apko.yaml=actions/quay.apko.yaml
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
labels:
|
||||
app.kubernetes.io/name: project-delivery
|
||||
app.kubernetes.io/instance: quay-custom
|
||||
patches:
|
||||
- path: k8s/workflow-patch.yaml
|
||||
target:
|
||||
group: argoproj.io
|
||||
version: v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
name: build
|
||||
replacements:
|
||||
- source:
|
||||
group: argoproj.io
|
||||
version: v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
name: quay-custom-build
|
||||
fieldPath: metadata.name
|
||||
targets:
|
||||
- select:
|
||||
group: delivery.quorate.dev
|
||||
version: v1alpha1
|
||||
kind: ProjectDelivery
|
||||
name: quay-custom-image
|
||||
fieldPaths:
|
||||
- spec.workflowTemplateRef.name
|
||||
51
services/quay-custom/patches/inventory.patch
Normal file
51
services/quay-custom/patches/inventory.patch
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
--- a/data/secscan_model/datatypes.py
|
||||
+++ b/data/secscan_model/datatypes.py
|
||||
@@ -94,7 +94,9 @@
|
||||
Layer = namedtuple("Layer", ["Name", "NamespaceName", "ParentName", "IndexedByVersion", "Features"])
|
||||
|
||||
|
||||
-class SecurityInformation(namedtuple("SecurityInformation", ["Layer"])):
|
||||
+class SecurityInformation(
|
||||
+ namedtuple("SecurityInformation", ["Layer", "Inventory"], defaults=(None,))
|
||||
+):
|
||||
"""
|
||||
Canonical representation of security scan data for an image/manifest which is returned by the Quay API.
|
||||
"""
|
||||
@@ -146,11 +148,13 @@
|
||||
)
|
||||
for f in data_dict["Layer"].get("Features", [])
|
||||
],
|
||||
- )
|
||||
+ ),
|
||||
+ data_dict.get("Inventory"),
|
||||
)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
+ "Inventory": self.Inventory,
|
||||
"Layer": {
|
||||
"Name": self.Layer.Name,
|
||||
"ParentName": self.Layer.ParentName,
|
||||
--- a/data/secscan_model/secscan_v4_model.py
|
||||
+++ b/data/secscan_model/secscan_v4_model.py
|
||||
@@ -263,7 +263,19 @@
|
||||
# TODO(alecmerdler): Provide a way to indicate the current scan is outdated (`report.state != status.indexer_hash`)
|
||||
|
||||
return SecurityInformationLookupResult.for_data(
|
||||
- SecurityInformation(Layer(report["manifest_hash"], "", "", 4, features_for(report)))
|
||||
+ SecurityInformation(
|
||||
+ Layer(report["manifest_hash"], "", "", 4, features_for(report)),
|
||||
+ {
|
||||
+ key: report.get(key)
|
||||
+ for key in (
|
||||
+ "manifest_hash",
|
||||
+ "packages",
|
||||
+ "distributions",
|
||||
+ "environments",
|
||||
+ "repository",
|
||||
+ )
|
||||
+ },
|
||||
+ )
|
||||
)
|
||||
|
||||
def _get_manifest_iterator(
|
||||
31
services/quay-custom/patches/oauth-assign.patch
Normal file
31
services/quay-custom/patches/oauth-assign.patch
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
--- a/endpoints/web.py
|
||||
+++ b/endpoints/web.py
|
||||
@@ -873,7 +873,8 @@
|
||||
if not features.ASSIGN_OAUTH_TOKEN:
|
||||
abort(404)
|
||||
|
||||
- if not current_user.is_authenticated:
|
||||
+ assigning_user = get_authenticated_user()
|
||||
+ if not assigning_user:
|
||||
abort(401)
|
||||
|
||||
user = get_nonrobot_user(username)
|
||||
@@ -883,8 +884,7 @@
|
||||
application = get_oauth_application_for_client_id(client_id)
|
||||
if not application:
|
||||
abort(404)
|
||||
- current_db_user = current_user.db_user()
|
||||
- if not is_org_admin(current_db_user, application.organization):
|
||||
+ if not AdministerOrganizationPermission(application.organization.username).can():
|
||||
abort(403)
|
||||
|
||||
assign_token_to_user(
|
||||
@@ -899,7 +899,7 @@
|
||||
"oauth_token_assigned",
|
||||
application.organization.username,
|
||||
{
|
||||
- "assigning_user": current_db_user.username,
|
||||
+ "assigning_user": assigning_user.username,
|
||||
"assigned_user": user.username,
|
||||
"application": application.name,
|
||||
"client_id": application.client_id,
|
||||
41
services/quay-custom/patches/oidc-eddsa.patch
Normal file
41
services/quay-custom/patches/oidc-eddsa.patch
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
--- a/oauth/oidc.py
|
||||
+++ b/oauth/oidc.py
|
||||
@@ -26,7 +26,7 @@
|
||||
|
||||
OIDC_WELLKNOWN = ".well-known/openid-configuration"
|
||||
PUBLIC_KEY_CACHE_TTL = 3600 # 1 hour
|
||||
-ALLOWED_ALGORITHMS = ["RS256", "RS384"]
|
||||
+ALLOWED_ALGORITHMS = ["RS256", "RS384", "EdDSA"]
|
||||
JWT_CLOCK_SKEW_SECONDS = 30
|
||||
|
||||
|
||||
@@ -490,13 +490,12 @@
|
||||
except ValueError:
|
||||
raise PublicKeyLoadException("Public key %s not found" % kid)
|
||||
|
||||
- if key_found.kty != "RSA":
|
||||
- raise PublicKeyLoadException("No RSA form of public key %s not found" % kid)
|
||||
+ if key_found.kty not in ("RSA", "OKP"):
|
||||
+ raise PublicKeyLoadException("Unsupported public key type for %s" % kid)
|
||||
|
||||
- # Eyy, no more exporting/importing keys from PyCrypto to cryptography's format...
|
||||
- rsa_key = key_found.as_key()
|
||||
- self[kid] = rsa_key
|
||||
- return rsa_key
|
||||
+ public_key = key_found.as_key()
|
||||
+ self[kid] = public_key
|
||||
+ return public_key
|
||||
|
||||
|
||||
def _load_keys_from_url(url, verify=True):
|
||||
--- a/util/security/jwtutil.py
|
||||
+++ b/util/security/jwtutil.py
|
||||
@@ -27,7 +27,7 @@
|
||||
|
||||
# ALGORITHM_WHITELIST defines a whitelist of allowed algorithms to be used in JWTs. DO NOT ADD
|
||||
# `none` here!
|
||||
-ALGORITHM_WHITELIST = ["rs256", "hs256", "rs384"]
|
||||
+ALGORITHM_WHITELIST = ["rs256", "hs256", "rs384", "eddsa"]
|
||||
|
||||
|
||||
class _StrictJWT(PyJWT):
|
||||
Loading…
Reference in a new issue