ce7cb702218693d08ea6d83464d2457c9250027c

This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
11 changed files with 145336 additions and 0 deletions

View file

@ -0,0 +1,163 @@
#load "projects.fsx"
#load "layers.fsx"
open System
open System.Formats.Tar
open System.IO
open System.IO.Compression
open System.Runtime.InteropServices
open System.Security.Cryptography
open System.Security.Cryptography.X509Certificates
open System.Text
open System.Text.Json.Nodes
open Contract
open Projects
let loader =
match RuntimeInformation.ProcessArchitecture with
| Architecture.X64 -> "/toolchain/usr/lib/ld-linux-x86-64.so.2"
| Architecture.Arm64 -> "/toolchain/usr/lib/ld-linux-aarch64.so.1"
| value -> fail $"unsupported builder architecture {value}"
let bsdtar arguments =
requireSuccess
loader
([ "--library-path"; "/toolchain/usr/lib"; "/toolchain/usr/bin/bsdtar" ]
@ arguments)
None
[]
let work = directory "/cache/extensions"
let output = "/output"
let baseRoot = "/toolchain/base"
let blobs = Path.Combine(baseRoot, "blobs/sha256")
let node path = JsonNode.Parse(File.ReadAllText path)
let text (value: JsonNode) = value.GetValue<string>()
let blobPath (digest: string) = Path.Combine(blobs, digest[7..])
let index = node (Path.Combine(baseRoot, "index.json"))
let baseFiles (descriptor: JsonNode) =
let manifest = node (blobPath (text descriptor["digest"]))
let mutable status = ""
let mutable bundle = ""
for layer in manifest["layers"].AsArray() do
use input = File.OpenRead(blobPath (text layer["digest"]))
use gzip = new GZipStream(input, CompressionMode.Decompress)
use reader = new TarReader(gzip)
let mutable entry = reader.GetNextEntry()
while not (isNull entry) do
match entry.Name.TrimStart('.', '/') with
| "var/lib/dpkg/status" ->
use content = new StreamReader(entry.DataStream, Encoding.UTF8, true, 4096, true)
status <- content.ReadToEnd()
| "etc/ssl/certs/ca-certificates.crt" ->
use content = new StreamReader(entry.DataStream, Encoding.UTF8, true, 4096, true)
bundle <- content.ReadToEnd()
| _ -> ()
entry <- reader.GetNextEntry()
status, bundle
for descriptor in index["manifests"].AsArray() do
let architecture = text (descriptor["platform"]["architecture"])
let status, bundle = baseFiles descriptor
let caRecord =
status.Split("\n\n", StringSplitOptions.RemoveEmptyEntries)
|> Array.find (fun record -> record.StartsWith("Package: ca-certificates\n", StringComparison.Ordinal))
let version =
caRecord.Split('\n')
|> Array.find (fun line -> line.StartsWith("Version: ", StringComparison.Ordinal))
let certificates = X509Certificate2Collection()
certificates.ImportFromPem bundle
require (certificates.Count > 0) $"{architecture} base CA bundle is empty"
let affected =
Set.ofList
[ "86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b"
"d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4" ]
for certificate in certificates do
let fingerprint =
certificate.GetCertHashString(HashAlgorithmName.SHA256).ToLowerInvariant()
require
(not (Set.contains fingerprint affected))
$"{architecture} base still contains affected trust anchor {fingerprint}"
certificate.Dispose()
printfn "trust=verified architecture=%s package=%s certificates=%d" architecture version certificates.Count
let unpack archive destination =
let listing = bsdtar [ "-tf"; archive ]
let payload =
listing.Split('\n', StringSplitOptions.RemoveEmptyEntries)
|> Array.find (fun name -> name.StartsWith("data.tar", StringComparison.Ordinal))
let control = directory (Path.Combine(work, Path.GetRandomFileName()))
bsdtar [ "-xf"; archive; "-C"; control; payload ] |> ignore
bsdtar [ "-xf"; Path.Combine(control, payload); "-C"; destination ] |> ignore
Directory.Delete(control, true)
let selections =
[ "amd64"; "arm64" ]
|> List.map (fun architecture ->
let root = Path.Combine(work, architecture)
let packages = Path.Combine("/toolchain/packages", architecture)
if not (Directory.Exists root) then
directory root |> ignore
for name in [ "timescale.deb"; "loader.deb"; "vector.deb" ] do
unpack (Path.Combine(packages, name)) root
let scale = directory (Path.Combine(work, $"scale-{architecture}"))
ZipFile.ExtractToDirectory(Path.Combine(packages, "scale.zip"), scale)
let scaleDeb =
Path.Combine(scale, $"pgvectorscale-postgresql-18_{scaleVersion}-Linux_{architecture}.deb")
unpack scaleDeb root
Directory.Delete(scale, true)
let files =
[ "usr/lib/postgresql/18/lib"; "usr/share/postgresql/18/extension" ]
|> List.collect (fun path ->
Directory.EnumerateFiles(Path.Combine(root, path), "*", SearchOption.AllDirectories)
|> Seq.map (fun source -> Path.GetRelativePath(root, source), source, File.GetUnixFileMode source)
|> Seq.toList)
architecture, files)
|> Map.ofList
let labels (config: JsonNode) =
let values = config["config"].AsObject()
require (text values["User"] = "26") "PostgreSQL base runtime user changed"
let labels =
match values["Labels"] with
| null -> JsonObject()
| current -> current.AsObject()
labels["org.opencontainers.image.source"] <- "https://forge.quorate.internal/quorate/cluster"
labels["org.opencontainers.image.version"] <- $"{postgresVersion}-ts2.31.0-vectorscale{scaleVersion}"
labels["org.opencontainers.image.revision"] <- Environment.GetEnvironmentVariable "SOURCE_REVISION"
labels["quorate.timescaledb"] <- timescaleVersion
labels["quorate.pgvector"] <- vectorVersion
labels["quorate.pgvectorscale"] <- scaleVersion
values["Labels"] <- labels
let digest =
Layers.assemble baseRoot output (fun architecture -> selections[architecture]) labels
printfn "build=image project=pg-image digest=%s" digest

View file

@ -0,0 +1,91 @@
#load "contract.fsx"
open System.IO
open Contract
type Project =
{ Name: string
Workflow: string
Repository: string }
type Package =
{ Name: string
Url: string
Sha256: string }
let project =
{ Name = "pg-image"
Workflow = "pg-image-build"
Repository = "quorate/cnpg-postgresql" }
let postgresVersion = "18.6"
let baseDigest =
"sha256:39add80f5b1e573e2f6b641c66fd1a2f8dede67ca0330e870285bc1078551390"
let timescaleVersion = "2.31.0~debian13-1806"
let vectorVersion = "0.8.7-1.pgdg13+1"
let scaleVersion = "0.9.1"
let sourcePaths =
[ "services/pg-image/actions/projects.fsx"
"services/pg-image/actions/toolchain.fsx"
"services/pg-image/actions/build.fsx"
"services/pg-image/actions/tools.apko.yaml"
"services/quay-custom/actions/base.fsx"
"services/quay-custom/actions/layers.fsx" ]
let selectProject (targets: Target array) =
require
(targets.Length = 1
&& targets[0].Kind = "ZotOCI"
&& targets[0].Repository = project.Repository)
"invalid PostgreSQL delivery target"
project
let validateProject sourceRoot _ =
sourcePaths
|> List.iter (fun path ->
require (File.Exists(Path.Combine(sourceRoot, path))) $"missing PostgreSQL build input: {path}")
let inputPaths _ _ = sourcePaths
let toolchainProfileName = "pg-image"
let toolchainCachePaths _ =
[ "base"; "packages"; "usr/bin"; "usr/lib" ]
let packages architecture =
let mainHash, loaderHash, vectorHash, scaleHash =
match architecture with
| "amd64" ->
"4ad79ee22082dd14f56e3d1e6e24c375fcf83d9a50139b5fb56541b2019d0f94",
"fecaee346fb4b7f1101dd2e7fce9ba9458bf3868dc4499461e8554f679b74965",
"d9923b9ea81375604d88bfcfa167d58bfaf3de5689372c9e4504244f1671f596",
"63f244a4a4ff356d972137a3c501f9359420d1bc5fdc7e0e2bafd138feaac245"
| "arm64" ->
"3f25ea7e7d6f873729a7f671e3fcf012e4b4106a6e3b0fad49b73880b6251c9f",
"89f8423c1f22b9feb8975cad76810c74981b2071feed9aff534240fd56032275",
"87e5dfbd40adcf0fca2b08bfaa422b88fca37672a1f4fd48e64249d303f52fe3",
"6b15969f137f59787f9e84892f3cf0782fbc32b932217dd8ed07db3f985b0f96"
| value -> fail $"unsupported PostgreSQL architecture {value}"
let timescale =
"https://packagecloud.io/timescale/timescaledb/debian/pool/trixie/main/t"
[ { Name = "timescale.deb"
Url =
$"{timescale}/timescaledb-2-postgresql-18/timescaledb-2-postgresql-18_{timescaleVersion}_{architecture}.deb"
Sha256 = mainHash }
{ Name = "loader.deb"
Url =
$"{timescale}/timescaledb-2-2.31.0-postgresql-18/timescaledb-2-loader-postgresql-18_{timescaleVersion}_{architecture}.deb"
Sha256 = loaderHash }
{ Name = "vector.deb"
Url =
$"https://apt.postgresql.org/pub/repos/apt/pool/main/p/pgvector/postgresql-18-pgvector_{vectorVersion}_{architecture}.deb"
Sha256 = vectorHash }
{ Name = "scale.zip"
Url =
$"https://github.com/timescale/pgvectorscale/releases/download/{scaleVersion}/pgvectorscale-{scaleVersion}-pg18-{architecture}.zip"
Sha256 = scaleHash } ]

View file

@ -0,0 +1,49 @@
#nowarn "0760"
#r "/opt/fsi/packages/Oci.dll"
#load "projects.fsx"
open System
open System.IO
open System.Text.Json
open Contract
open Quorate.Oci
open Projects
let revision = Environment.GetEnvironmentVariable "SOURCE_REVISION"
let targets = Environment.GetEnvironmentVariable "TARGETS" |> decode<Target array>
let project = selectProject targets
let layout = "/handoff/layout"
let digest = File.ReadAllText("/handoff/image-digest").Trim()
require (validRevision revision) "invalid source revision"
require (validDigest digest) "invalid image digest"
let publish () =
let auth = RegistryAuth.Load("/auth/.dockerconfigjson", "zot.quorate.internal")
use root = Registry.LoadRoot "/registry-ca/ca.crt"
use registry = Registry(Uri("https://zot.quorate.internal"), auth, root)
registry
.PushAsync(project.Repository, layout, "delivery", $"delivery-{revision}", cancellationToken)
.GetAwaiter()
.GetResult()
let published = publish ()
require (published = digest) "published digest mismatch"
directory "/result" |> ignore
File.WriteAllText(
"/result/delivery-result",
JsonSerializer.Serialize(
{ SourceRevision = revision
Targets =
[| { Kind = "ZotOCI"
Repository = project.Repository
Reference = $"zot.quorate.internal/{project.Repository}@{digest}" } |] },
json
)
)
printfn "publish=%s digest=%s" project.Repository digest

View file

@ -0,0 +1,90 @@
#r "System.Formats.Tar"
#load "projects.fsx"
#load "base.fsx"
open System
open System.Formats.Tar
open System.IO
open System.IO.Compression
open System.Net.Http
open System.Runtime.InteropServices
open Contract
open Projects
do
let root = "/toolchain"
let architecture =
match RuntimeInformation.ProcessArchitecture with
| Architecture.X64 -> "x86_64"
| Architecture.Arm64 -> "aarch64"
| value -> fail $"unsupported builder architecture {value}"
let archive = "/tmp/tools.tar"
let export = directory "/tmp/tools"
requireSuccess
"/usr/bin/apko"
[ "build"
"--arch"
architecture
"--sbom=false"
"/orchestration/go-oci.apko.yaml"
"pg-tools:root"
archive ]
None
[]
|> ignore
TarFile.ExtractToDirectory(archive, export, false)
for layer in Directory.EnumerateFiles(export, "*.tar.gz") do
use input = File.OpenRead layer
use gzip = new GZipStream(input, CompressionMode.Decompress)
use reader = new TarReader(gzip)
let mutable entry = reader.GetNextEntry()
while not (isNull entry) do
let path = entry.Name.TrimStart('.', '/')
if
path.StartsWith("usr/bin/", StringComparison.Ordinal)
|| path.StartsWith("usr/lib/", StringComparison.Ordinal)
then
let target = Path.Combine(root, path)
match entry.EntryType with
| TarEntryType.Directory -> directory target |> ignore
| TarEntryType.RegularFile ->
directory (Path.GetDirectoryName target) |> ignore
use output = File.Create target
if entry.Length > 0L then
entry.DataStream.CopyTo output
output.Dispose()
File.SetUnixFileMode(target, entry.Mode)
| TarEntryType.SymbolicLink ->
directory (Path.GetDirectoryName target) |> ignore
File.CreateSymbolicLink(target, entry.LinkName) |> ignore
| _ -> ()
entry <- reader.GetNextEntry()
use client = new HttpClient(Timeout = TimeSpan.FromMinutes 10.0)
for target in [ "amd64"; "arm64" ] do
let packageRoot = directory (Path.Combine(root, "packages", target))
for package in packages target do
use response = client.GetAsync(package.Url).GetAwaiter().GetResult()
require response.IsSuccessStatusCode $"{package.Name} download failed with HTTP {int response.StatusCode}"
use input = response.Content.ReadAsStream()
let path = Path.Combine(packageRoot, package.Name)
use output = File.Create path
input.CopyTo output
output.Dispose()
require (sha256 path = package.Sha256) $"{target}/{package.Name} checksum mismatch"
Base.pullFrom "ghcr.io" "cloudnative-pg/postgresql" baseDigest (Path.Combine(root, "base"))
printfn "toolchain=primary project=pg-image postgres=%s" postgresVersion

View file

@ -0,0 +1,8 @@
contents:
keyring:
- https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
repositories:
- https://packages.wolfi.dev/os
packages:
- ca-certificates-bundle=20261020-r0
- libarchive-tools=3.8.10-r0

View file

@ -0,0 +1,3 @@
images:
- path: spec/templates/container/image
kind: WorkflowTemplate

View file

@ -0,0 +1,94 @@
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: tools
namespace: project-delivery
spec:
endpointSelector:
matchLabels:
delivery.quorate.dev/stage: pg-image-toolchain
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- {port: "53", protocol: UDP}
- {port: "53", protocol: TCP}
rules:
dns:
- {matchName: packages.wolfi.dev}
- {matchName: ghcr.io}
- {matchName: pkg-containers.githubusercontent.com}
- {matchName: github.com}
- {matchName: release-assets.githubusercontent.com}
- {matchName: packagecloud.io}
- {matchName: d3fo0g5hm7lbuv.cloudfront.net}
- {matchName: apt.postgresql.org}
- toFQDNs:
- {matchName: packages.wolfi.dev}
- {matchName: ghcr.io}
- {matchName: pkg-containers.githubusercontent.com}
- {matchName: github.com}
- {matchName: release-assets.githubusercontent.com}
- {matchName: packagecloud.io}
- {matchName: d3fo0g5hm7lbuv.cloudfront.net}
- {matchName: apt.postgresql.org}
toPorts:
- ports:
- {port: "443", protocol: TCP}
---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: publish
namespace: project-delivery
spec:
endpointSelector:
matchLabels:
delivery.quorate.dev/stage: pg-image-publish
egress:
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: kube-system
k8s-app: kube-dns
toPorts:
- ports:
- {port: "53", protocol: UDP}
- {port: "53", protocol: TCP}
rules:
dns:
- {matchName: zot.quorate.internal}
- toCIDR: [172.28.129.182/32]
toPorts:
- ports:
- {port: "443", protocol: TCP}
- toEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: zot
app.kubernetes.io/name: zot
app.kubernetes.io/instance: zot
toPorts:
- ports:
- {port: "5000", protocol: TCP}
---
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
name: zot-in
namespace: zot
spec:
endpointSelector:
matchLabels:
app.kubernetes.io/name: zot
app.kubernetes.io/instance: zot
ingress:
- fromEndpoints:
- matchLabels:
k8s:io.kubernetes.pod.namespace: project-delivery
io.cilium.k8s.policy.serviceaccount: project-delivery-executor
delivery.quorate.dev/stage: pg-image-publish
toPorts:
- ports:
- {port: "5000", protocol: TCP}

View file

@ -0,0 +1,19 @@
apiVersion: delivery.quorate.dev/v1alpha1
kind: ProjectDelivery
metadata:
name: image
namespace: project-delivery
spec:
source:
repository: https://forge.quorate.internal/quorate/cluster
continuous:
ref: refs/heads/master
paths:
- services/pg-image
- services/quay-custom/actions/base.fsx
- services/quay-custom/actions/layers.fsx
targets:
- kind: ZotOCI
repository: quorate/cnpg-postgresql
workflowTemplateRef:
name: build

View file

@ -0,0 +1,45 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: zot-vso
namespace: project-delivery
automountServiceAccountToken: false
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultAuth
metadata:
name: zot
namespace: project-delivery
spec:
method: kubernetes
mount: kubernetes
kubernetes:
role: pg-image-writer
serviceAccount: pg-image-zot-vso
vaultConnectionRef: vault/vault
---
apiVersion: secrets.hashicorp.com/v1beta1
kind: VaultStaticSecret
metadata:
name: writer
namespace: project-delivery
spec:
type: kv-v2
mount: secret
path: zot/pg-image
refreshAfter: 1h
destination:
name: pg-image-writer
create: true
overwrite: true
type: kubernetes.io/dockerconfigjson
transformation:
excludeRaw: true
excludes: [.*]
templates:
.dockerconfigjson:
text: |
{{- $user := get .Secrets "username" -}}
{{- $password := get .Secrets "password" -}}
{"auths":{"zot.quorate.internal":{"username":"{{ $user }}","password":"{{ $password }}","auth":"{{ printf "%s:%s" $user $password | b64enc }}"}}}
vaultAuthRef: pg-image-zot

View file

@ -0,0 +1,52 @@
- op: replace
path: /metadata/labels/app.kubernetes.io~1instance
value: pg-image
- op: replace
path: /spec/podMetadata/labels/delivery.quorate.dev~1project
value: pg-image
- op: replace
path: /spec/templates/5/metadata/labels/delivery.quorate.dev~1stage
value: pg-image-toolchain
- op: replace
path: /spec/templates/7/metadata/labels/delivery.quorate.dev~1stage
value: pg-image-publish
- op: replace
path: /spec/templates/1/volumes/0
value: &orchestration
name: orchestration
projected:
defaultMode: 0444
sources:
- configMap:
name: go-oci-orch
items:
- {key: cache.fsx, path: cache.fsx}
- {key: cache-gc.fsx, path: cache-gc.fsx}
- {key: contract.fsx, path: contract.fsx}
- {key: initialize.fsx, path: initialize.fsx}
- {key: source.fsx, path: source.fsx}
- configMap:
name: pg-image-project
- configMap:
name: pg-image-oci-base
- op: replace
path: /spec/templates/3/volumes/0
value: *orchestration
- op: replace
path: /spec/templates/4/volumes/0
value: *orchestration
- op: replace
path: /spec/templates/5/volumes/0
value: *orchestration
- op: replace
path: /spec/templates/6/volumes/0
value: *orchestration
- op: replace
path: /spec/templates/7/volumes/0
value: *orchestration
- op: replace
path: /spec/templates/7/volumes/1/secret/secretName
value: pg-image-writer
- op: replace
path: /spec/templates/8/volumes/0
value: *orchestration

View file

@ -0,0 +1,50 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namePrefix: pg-image-
configurations:
- k8s/kustomizeconfig.yaml
images:
- name: registry.quorate.internal/quorate/fsharp-runner
digest: sha256:d7b904f4f0cae9fdba7c7345cdf08c91162245834a8a13658d93396272aeec83
resources:
- ../project-delivery/k8s/workflows/go-oci/parallel
- ../quay-custom/actions
- k8s/network-policy.yaml
- k8s/project.yaml
- k8s/vso.yaml
configMapGenerator:
- name: project
namespace: project-delivery
files:
- build.fsx=actions/build.fsx
- projects.fsx=actions/projects.fsx
- publish.fsx=actions/publish.fsx
- toolchain.fsx=actions/toolchain.fsx
- go-oci.apko.yaml=actions/tools.apko.yaml
generatorOptions:
disableNameSuffixHash: true
labels:
app.kubernetes.io/name: project-delivery
app.kubernetes.io/instance: pg-image
patches:
- path: k8s/workflow-patch.yaml
target:
group: argoproj.io
version: v1alpha1
kind: WorkflowTemplate
name: build
replacements:
- source:
group: argoproj.io
version: v1alpha1
kind: WorkflowTemplate
name: pg-image-build
fieldPath: metadata.name
targets:
- select:
group: delivery.quorate.dev
version: v1alpha1
kind: ProjectDelivery
name: pg-image-image
fieldPaths:
- spec.workflowTemplateRef.name