ms2k/quorate
SHA256
Watch
0
Code Releases Activity

ce7cb702218693d08ea6d83464d2457c9250027c

Browse source
This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
10 changed files with 145336 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

708
services/mirrored-image/controller/controller.go Normal file
View file

@ -0,0 +1,708 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base32"
"encoding/json"
"errors"
"fmt"
"log/slog"
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/dynamic/dynamicinformer"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/retry"
"k8s.io/client-go/util/workqueue"
)
const (
prepareStage = "prepare"
promotionStage = "promotion"
cleanupStage = "cleanup"
scanAccepted = "Accepted"
scanPending = "Pending"
scanRejected = "Rejected"
)
var (
imageGVR = schema.GroupVersionResource{Group: "mirror.quorate.dev", Version: "v1alpha1", Resource: "mirroredimages"}
workflowGVR = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflows"}
digestRule = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
tagRule = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
retentionTagRule = regexp.MustCompile(`^sha256-[0-9a-f]{64}$`)
)
type controllerConfig struct {
namespace string
pollInterval time.Duration
prepareTemplate string
promotionTemplate string
cleanupTemplate string
}
type controller struct {
client dynamic.Interface
registry registryReader
config controllerConfig
}
type registryReader interface {
scan(context.Context, string, string, string) (string, error)
exists(context.Context, string, string) (bool, error)
physicallyPresent(context.Context, string, string) (bool, error)
}
type retryRef struct {
digest int
stage string
successor bool
}
type imageKey struct {
namespace string
name string
}
func (c *controller) run(ctx context.Context) error {
factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(c.client, 0, c.config.namespace, nil)
informer := factory.ForResource(imageGVR).Informer()
queue := workqueue.NewTypedRateLimitingQueueWithConfig(
workqueue.DefaultTypedControllerRateLimiter[imageKey](),
workqueue.TypedRateLimitingQueueConfig[imageKey]{Name: "mirrored-image"},
)
defer queue.ShutDown()
enqueue := func(object any) {
image, ok := object.(*unstructured.Unstructured)
if ok {
queue.Add(imageKey{namespace: image.GetNamespace(), name: image.GetName()})
}
}
_, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: enqueue,
UpdateFunc: func(_, newObject any) { enqueue(newObject) },
})
if err != nil {
return fmt.Errorf("register MirroredImage informer: %w", err)
}
factory.Start(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
if err := ctx.Err(); err != nil {
return err
}
return errors.New("MirroredImage informer did not synchronize")
}
go func() {
<-ctx.Done()
queue.ShutDown()
}()
for c.processNext(ctx, queue) {
}
return ctx.Err()
}
func (c *controller) processNext(ctx context.Context, queue workqueue.TypedRateLimitingInterface[imageKey]) bool {
key, shutdown := queue.Get()
if shutdown {
return false
}
defer queue.Done(key)
found, err := c.reconcileKey(ctx, key.namespace, key.name)
if err != nil {
slog.Error("reconcile MirroredImage", "namespace", key.namespace, "name", key.name, "error", err)
c.recordError(ctx, key.namespace, key.name, err)
queue.AddRateLimited(key)
return true
}
queue.Forget(key)
if found {
queue.AddAfter(key, c.config.pollInterval)
}
return true
}
func (c *controller) reconcileKey(ctx context.Context, namespace, name string) (bool, error) {
found := false
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
object, err := c.client.Resource(imageGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
found = true
return c.reconcile(ctx, object)
})
return found, err
}
func (c *controller) reconcile(ctx context.Context, object *unstructured.Unstructured) error {
image, err := decodeImage(object)
if err != nil {
return err
}
if image.Metadata.DeletionTimestamp != nil {
return nil
}
if err := validateImage(&image); err != nil {
return err
}
retried, err := c.reconcileRetry(ctx, &image)
if err != nil {
return err
}
if !retried {
if err := c.reconcileDesired(ctx, &image); err != nil {
return err
}
}
image.Status.ObservedGeneration = image.Metadata.Generation
return c.updateStatus(ctx, object, &image)
}
func (c *controller) reconcileRetry(ctx context.Context, image *mirroredImage) (bool, error) {
request := image.Metadata.Annotations[retryAnnotation]
if request == "" || request == image.Status.LastHandledRetry {
return false, nil
}
target, found := retryTarget(image, request)
if !found {
return false, fmt.Errorf("retry Workflow %q is not a recorded attempt", request)
}
attempt := attemptAt(image, target)
if target.successor {
if otherAttemptActive(image, attempt.WorkflowName) {
return false, nil
}
status := &image.Status.Digests[target.digest]
if err := c.advanceAttempt(ctx, image, status, target.stage, attempt); err != nil {
return false, err
}
image.Status.LastHandledRetry = request
image.Status.Message = "retry " + target.stage + ": " + phaseOrPending(attempt.Phase)
return true, nil
}
if otherAttemptActive(image, request) || !terminalPhase(attempt.Phase) {
return false, nil
}
status := &image.Status.Digests[target.digest]
successor := c.newAttempt(image, status, target.stage, request)
switch target.stage {
case prepareStage:
status.Prepare = successor
status.Scan = ""
status.Promotion = nil
case promotionStage:
status.Promotion = successor
case cleanupStage:
status.Cleanup = successor
default:
return false, fmt.Errorf("unknown retry stage %q", target.stage)
}
if target.stage != cleanupStage && status.Cleanup != nil {
status.Cleanup = c.newAttempt(image, status, cleanupStage, "")
}
image.Status.Message = "retry " + target.stage + ": Pending"
return true, nil
}
func (c *controller) reconcileDesired(ctx context.Context, image *mirroredImage) error {
desired := make(map[string]digestSpec, len(image.Spec.Digests))
failedMessage := ""
for _, item := range image.Spec.Digests {
desired[item.Digest] = item
index := digestIndex(image.Status.Digests, item.Digest)
if index < 0 {
image.Status.Digests = append(image.Status.Digests, digestStatus{Digest: item.Digest, Tag: item.Tag})
}
}
for index := 0; index < len(image.Status.Digests); index++ {
status := &image.Status.Digests[index]
item, wanted := desired[status.Digest]
retiring := status.Cleanup != nil || !wanted
if retiring {
active := activeAttempt(status)
if active != nil {
stage := activeStage(status)
if err := c.advanceAttempt(ctx, image, status, stage, active); err != nil {
return err
}
image.Status.Message = "awaiting " + stage + " before cleanup: " + phaseOrPending(active.Phase)
return nil
}
switch {
case status.Cleanup == nil:
status.Cleanup = c.newAttempt(image, status, cleanupStage, "")
image.Status.Message = "cleanup: Pending"
return nil
case !terminalPhase(status.Cleanup.Phase):
if err := c.advanceAttempt(ctx, image, status, cleanupStage, status.Cleanup); err != nil {
return err
}
image.Status.Message = "cleanup: " + phaseOrPending(status.Cleanup.Phase)
return nil
case status.Cleanup.Phase != "Succeeded":
if failedMessage == "" {
failedMessage = "cleanup: " + status.Cleanup.Phase
}
continue
default:
complete, err := c.cleanupComplete(ctx, image.Metadata.Name, status.Digest)
if err != nil {
return err
}
if !complete {
image.Status.Message = "cleanup: awaiting repository GC"
return nil
}
if wanted {
*status = digestStatus{Digest: item.Digest, Tag: item.Tag}
image.Status.Message = "cleanup completed before remirroring"
return nil
}
image.Status.Digests = append(image.Status.Digests[:index], image.Status.Digests[index+1:]...)
image.Status.Message = "cleanup completed"
return nil
}
}
switch {
case status.Cleanup != nil:
return fmt.Errorf("digest %q has cleanup state while desired", status.Digest)
case status.Prepare == nil:
status.Prepare = c.newAttempt(image, status, prepareStage, "")
image.Status.Message = "prepare: Pending"
return nil
case !terminalPhase(status.Prepare.Phase):
if err := c.advanceAttempt(ctx, image, status, prepareStage, status.Prepare); err != nil {
return err
}
image.Status.Message = "prepare: " + phaseOrPending(status.Prepare.Phase)
return nil
case status.Prepare.Phase != "Succeeded":
if failedMessage == "" {
failedMessage = "prepare: " + status.Prepare.Phase
}
continue
case status.Scan == scanRejected:
if failedMessage == "" {
failedMessage = "scan: " + status.Scan
}
continue
case status.Scan != scanAccepted:
phase, err := c.registry.scan(ctx, "candidates/"+image.Metadata.Name, status.Digest, image.Spec.Policy.RejectAtOrAbove)
if err != nil {
return err
}
status.Scan = phase
if phase == scanRejected {
if failedMessage == "" {
failedMessage = "scan: " + phase
}
continue
}
image.Status.Message = "scan: " + phase
return nil
case status.Promotion == nil:
status.Promotion = c.newAttempt(image, status, promotionStage, "")
image.Status.Message = "promotion: Pending"
return nil
case !terminalPhase(status.Promotion.Phase):
if err := c.advanceAttempt(ctx, image, status, promotionStage, status.Promotion); err != nil {
return err
}
image.Status.Message = "promotion: " + phaseOrPending(status.Promotion.Phase)
return nil
case status.Promotion.Phase != "Succeeded":
if failedMessage == "" {
failedMessage = "promotion: " + status.Promotion.Phase
}
default:
present, err := c.registry.exists(ctx, "mirror/"+image.Metadata.Name, status.Digest)
if err != nil {
return err
}
if !present {
image.Status.Message = "promotion: awaiting mirror"
return nil
}
}
}
if failedMessage == "" {
image.Status.Message = "desired digests reconciled"
} else {
image.Status.Message = failedMessage
}
return nil
}
func (c *controller) newAttempt(image *mirroredImage, status *digestStatus, stage, retryOf string) *attemptStatus {
attempt := &attemptStatus{RetryOf: retryOf, Phase: "Pending", Generation: image.Metadata.Generation}
lineage := []string{retryOf}
if stage == promotionStage && status.Prepare != nil {
lineage = append(lineage, status.Prepare.WorkflowName)
}
if stage == cleanupStage {
for _, copy := range []*attemptStatus{status.Prepare, status.Promotion} {
if copy != nil {
lineage = append(lineage, copy.WorkflowName)
}
}
}
attempt.WorkflowName = attemptName(image, status.Digest, stage, attempt.Generation, lineage...)
return attempt
}
func (c *controller) advanceAttempt(
ctx context.Context,
image *mirroredImage,
status *digestStatus,
stage string,
attempt *attemptStatus,
) error {
if attempt.WorkflowName == "" {
return errors.New("workflow attempt has no name")
}
template, parameters, err := c.workflowInput(image, status, stage)
if err != nil {
return err
}
if err := c.ensureWorkflow(ctx, image, stage, attempt.WorkflowName, template, parameters); err != nil {
return err
}
phase, err := c.workflowPhase(ctx, image.Metadata.Namespace, attempt.WorkflowName)
if err != nil {
return err
}
attempt.Phase = phaseOrPending(phase)
return nil
}
func (c *controller) cleanupComplete(ctx context.Context, name, digest string) (bool, error) {
candidate, err := c.registry.physicallyPresent(ctx, "candidates/"+name, digest)
if err != nil {
return false, err
}
mirror, err := c.registry.physicallyPresent(ctx, "mirror/"+name, digest)
if err != nil {
return false, err
}
return !candidate && !mirror, nil
}
func (c *controller) workflowInput(
image *mirroredImage,
status *digestStatus,
stage string,
) (string, map[string]string, error) {
parameters := map[string]string{
"digest": status.Digest,
}
switch stage {
case prepareStage:
parameters["tag"] = status.Tag
parameters["source-repository"] = image.Spec.Upstream
parameters["target-repository"] = "candidates/" + image.Metadata.Name
return c.config.prepareTemplate, parameters, nil
case promotionStage:
parameters["tag"] = status.Tag
parameters["source-repository"] = "candidates/" + image.Metadata.Name
parameters["target-repository"] = "mirror/" + image.Metadata.Name
return c.config.promotionTemplate, parameters, nil
case cleanupStage:
parameters["candidate-repository"] = "candidates/" + image.Metadata.Name
parameters["mirror-repository"] = "mirror/" + image.Metadata.Name
return c.config.cleanupTemplate, parameters, nil
default:
return "", nil, fmt.Errorf("unknown Workflow stage %q", stage)
}
}
func (c *controller) ensureWorkflow(
ctx context.Context,
image *mirroredImage,
stage, name, template string,
parameters map[string]string,
) error {
if stage == promotionStage {
_, err := c.client.Resource(workflowGVR).Namespace(image.Metadata.Namespace).Get(ctx, name, metav1.GetOptions{})
if err == nil {
return nil
}
if !apierrors.IsNotFound(err) {
return err
}
phase, err := c.registry.scan(ctx, "candidates/"+image.Metadata.Name, parameters["digest"], image.Spec.Policy.RejectAtOrAbove)
if err != nil {
return err
}
if phase != scanAccepted {
return fmt.Errorf("promotion scan: %s", phase)
}
}
names := make([]string, 0, len(parameters))
for name := range parameters {
names = append(names, name)
}
sort.Strings(names)
arguments := make([]any, 0, len(names))
for _, name := range names {
arguments = append(arguments, map[string]any{"name": name, "value": parameters[name]})
}
workflow := &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "argoproj.io/v1alpha1",
"kind": "Workflow",
"metadata": map[string]any{
"name": name,
"namespace": image.Metadata.Namespace,
"labels": map[string]any{
"app.kubernetes.io/name": "mirrored-image",
"app.kubernetes.io/instance": image.Metadata.Name,
"mirror.quorate.dev/purpose": stage,
},
"ownerReferences": []any{map[string]any{
"apiVersion": "mirror.quorate.dev/v1alpha1",
"kind": "MirroredImage",
"name": image.Metadata.Name,
"uid": image.Metadata.UID,
"controller": true,
"blockOwnerDeletion": false,
}},
},
"spec": map[string]any{
"workflowTemplateRef": map[string]any{"name": template},
"arguments": map[string]any{"parameters": arguments},
"synchronization": map[string]any{
"mutexes": []any{map[string]any{"name": workflowName("lock", image.Metadata.UID)}},
},
},
}}
_, err := c.client.Resource(workflowGVR).Namespace(image.Metadata.Namespace).Create(ctx, workflow, metav1.CreateOptions{})
if apierrors.IsAlreadyExists(err) {
return nil
}
return err
}
func (c *controller) workflowPhase(ctx context.Context, namespace, name string) (string, error) {
object, err := c.client.Resource(workflowGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return "", err
}
phase, _, _ := unstructured.NestedString(object.Object, "status", "phase")
return phase, nil
}
func (c *controller) updateStatus(ctx context.Context, object *unstructured.Unstructured, image *mirroredImage) error {
status, err := toMap(image.Status)
if err != nil {
return err
}
current, _, err := unstructured.NestedMap(object.Object, "status")
if err != nil {
return err
}
if reflect.DeepEqual(current, status) {
return nil
}
updated := object.DeepCopy()
updated.Object["status"] = status
_, err = c.client.Resource(imageGVR).Namespace(updated.GetNamespace()).UpdateStatus(ctx, updated, metav1.UpdateOptions{})
if apierrors.IsNotFound(err) {
return nil
}
return err
}
func (c *controller) recordError(ctx context.Context, namespace, name string, reconcileError error) {
object, err := c.client.Resource(imageGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return
}
image, err := decodeImage(object)
if err != nil {
return
}
image.Status.Message = reconcileError.Error()
image.Status.ObservedGeneration = image.Metadata.Generation
_ = c.updateStatus(ctx, object, &image)
}
func decodeImage(object *unstructured.Unstructured) (mirroredImage, error) {
content, err := json.Marshal(object.Object)
if err != nil {
return mirroredImage{}, err
}
var image mirroredImage
if err := json.Unmarshal(content, &image); err != nil {
return mirroredImage{}, err
}
return image, nil
}
func validateImage(image *mirroredImage) error {
if image.Spec.Upstream == "" {
return errors.New("upstream repository is empty")
}
if image.Spec.Policy.RejectAtOrAbove == "" {
return errors.New("severity threshold is empty")
}
seen := make(map[string]struct{}, len(image.Spec.Digests))
for index, item := range image.Spec.Digests {
if !digestRule.MatchString(item.Digest) {
return fmt.Errorf("invalid digest at index %d", index)
}
if item.Tag != "" && !tagRule.MatchString(item.Tag) {
return fmt.Errorf("invalid tag at index %d", index)
}
if retentionTagRule.MatchString(item.Tag) && item.Tag != "sha256-"+strings.TrimPrefix(item.Digest, "sha256:") {
return fmt.Errorf("tag at index %d is owned by another digest", index)
}
if _, found := seen[item.Digest]; found {
return fmt.Errorf("duplicate digest at index %d", index)
}
seen[item.Digest] = struct{}{}
}
return nil
}
func retryTarget(image *mirroredImage, workflow string) (retryRef, bool) {
for index := range image.Status.Digests {
status := &image.Status.Digests[index]
if status.Prepare != nil {
if status.Prepare.WorkflowName == workflow {
return retryRef{digest: index, stage: prepareStage}, true
}
if status.Prepare.RetryOf == workflow {
return retryRef{digest: index, stage: prepareStage, successor: true}, true
}
}
if status.Promotion != nil {
if status.Promotion.WorkflowName == workflow {
return retryRef{digest: index, stage: promotionStage}, true
}
if status.Promotion.RetryOf == workflow {
return retryRef{digest: index, stage: promotionStage, successor: true}, true
}
}
if status.Cleanup != nil {
if status.Cleanup.WorkflowName == workflow {
return retryRef{digest: index, stage: cleanupStage}, true
}
if status.Cleanup.RetryOf == workflow {
return retryRef{digest: index, stage: cleanupStage, successor: true}, true
}
}
}
return retryRef{}, false
}
func attemptAt(image *mirroredImage, target retryRef) *attemptStatus {
status := &image.Status.Digests[target.digest]
if target.stage == prepareStage {
return status.Prepare
}
if target.stage == promotionStage {
return status.Promotion
}
return status.Cleanup
}
func otherAttemptActive(image *mirroredImage, excluded string) bool {
for index := range image.Status.Digests {
status := &image.Status.Digests[index]
for _, attempt := range []*attemptStatus{status.Prepare, status.Promotion, status.Cleanup} {
if attempt == status.Cleanup && attempt != nil && attempt.Phase == "Pending" && activeAttempt(status) != nil {
continue
}
if attempt != nil && attempt.WorkflowName != excluded && !terminalPhase(attempt.Phase) {
return true
}
}
}
return false
}
func activeAttempt(status *digestStatus) *attemptStatus {
for _, attempt := range []*attemptStatus{status.Prepare, status.Promotion} {
if attempt != nil && !terminalPhase(attempt.Phase) {
return attempt
}
}
return nil
}
func activeStage(status *digestStatus) string {
if status.Prepare != nil && !terminalPhase(status.Prepare.Phase) {
return prepareStage
}
return promotionStage
}
func digestIndex(statuses []digestStatus, digest string) int {
for index := range statuses {
if statuses[index].Digest == digest {
return index
}
}
return -1
}
func toMap(value any) (map[string]any, error) {
content, err := json.Marshal(value)
if err != nil {
return nil, err
}
var result map[string]any
err = json.Unmarshal(content, &result)
return result, err
}
func workflowName(purpose string, fields ...string) string {
digest := sha256.Sum256([]byte("v1\x00" + purpose + "\x00" + strings.Join(fields, "\x00")))
encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(digest[:15])
return "mi-" + purpose + "-" + strings.ToLower(encoded)
}
func attemptName(image *mirroredImage, digest, stage string, generation int64, lineage ...string) string {
fields := []string{image.Metadata.UID, digest}
lineageFound := false
for _, value := range lineage {
if value != "" {
fields = append(fields, value)
lineageFound = true
}
}
if !lineageFound {
fields = append(fields, strconv.FormatInt(generation, 10))
}
return workflowName(stage, fields...)
}
func terminalPhase(phase string) bool {
return phase == "Succeeded" || phase == "Failed" || phase == "Error"
}
func phaseOrPending(phase string) string {
if phase == "" {
return "Pending"
}
return phase
}

577
services/mirrored-image/controller/controller_test.go Normal file
View file

@ -0,0 +1,577 @@
package main
import (
"context"
"errors"
"fmt"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic/fake"
)
const testDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
func TestRetentionTagOwnership(t *testing.T) {
t.Parallel()
for _, test := range []struct {
tag string
valid bool
}{
{"", true},
{"stable", true},
{"sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", true},
{"sha256-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", false},
{"sha256-release", true},
} {
t.Run(test.tag, func(t *testing.T) {
image := testImage()
image.Spec.Digests = []digestSpec{{Digest: testDigest, Tag: test.tag}}
err := validateImage(&image)
if (err == nil) != test.valid {
t.Fatalf("RetentionTags.AliasAllowed: valid=%t err=%v", test.valid, err)
}
})
}
}
type fakeRegistry struct {
scanPhase string
present map[string]bool
stored map[string]bool
}
func (f *fakeRegistry) scan(context.Context, string, string, string) (string, error) {
return f.scanPhase, nil
}
func (f *fakeRegistry) exists(_ context.Context, repository, digest string) (bool, error) {
return f.present[repository+"@"+digest], nil
}
func (f *fakeRegistry) physicallyPresent(_ context.Context, repository, digest string) (bool, error) {
if f.stored != nil {
return f.stored[repository+"@"+digest], nil
}
return f.present[repository+"@"+digest], nil
}
func TestCleanupObservedMeansGone(t *testing.T) {
t.Parallel()
for _, repository := range []string{"candidates/pause", "mirror/pause"} {
t.Run(repository, func(t *testing.T) {
registry := &fakeRegistry{stored: map[string]bool{repository + "@" + testDigest: true}}
instance := testController(newClient(), registry)
image := testImage()
image.Spec.Digests = nil
image.Status.Digests = []digestStatus{{Digest: testDigest, Cleanup: &attemptStatus{WorkflowName: "cleanup", Phase: "Succeeded"}}}
reconcileDesired(context.Background(), t, instance, &image)
if len(image.Status.Digests) != 1 {
t.Fatal("CleanupObservedMeansGone: physical manifest was forgotten")
}
delete(registry.stored, repository+"@"+testDigest)
reconcileDesired(context.Background(), t, instance, &image)
if len(image.Status.Digests) != 0 {
t.Fatal("physically absent digest was retained")
}
})
}
}
func TestCopyRetryRearmsCleanup(t *testing.T) {
t.Parallel()
for _, stage := range []string{prepareStage, promotionStage} {
for _, readd := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/readd=%t", stage, readd), func(t *testing.T) {
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{scanPhase: scanAccepted, stored: map[string]bool{}}
instance := testController(client, registry)
image := testImage()
image.Spec.Digests = nil
image.Status.Digests = []digestStatus{{
Digest: testDigest, Tag: "stable", Scan: scanAccepted,
Prepare: &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
Promotion: &attemptStatus{WorkflowName: "promotion", Phase: "Succeeded"},
}}
status := &image.Status.Digests[0]
status.Cleanup = instance.newAttempt(&image, status, cleanupStage, "")
status.Cleanup.Phase = "Succeeded"
oldCleanup := status.Cleanup.WorkflowName
if _, err := client.Resource(workflowGVR).Namespace("mirrored-image").Create(ctx, workflowObject(oldCleanup, "Succeeded"), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
image.Metadata.Annotations = map[string]string{retryAnnotation: stage}
for pass := 0; pass < 2; pass++ {
handled, err := instance.reconcileRetry(ctx, &image)
if err != nil || !handled {
t.Fatalf("retry pass %d: handled=%t err=%v", pass, handled, err)
}
if pass == 0 {
if image.Status.LastHandledRetry != "" {
t.Fatal("RetryAcknowledgedAfterCreation")
}
if readd {
image.Spec.Digests = []digestSpec{{Digest: testDigest, Tag: "edited"}}
}
}
}
if image.Status.LastHandledRetry != stage {
t.Fatal("retry was not acknowledged")
}
copy := status.Prepare
if stage == promotionStage {
copy = status.Promotion
}
setPhase(ctx, t, client, copy.WorkflowName, "Succeeded")
registry.stored["candidates/pause@"+testDigest] = true
reconcileDesired(ctx, t, instance, &image)
reconcileDesired(ctx, t, instance, &image)
if status.Cleanup == nil || status.Cleanup.WorkflowName == oldCleanup || terminalPhase(status.Cleanup.Phase) {
t.Fatal("copy retry reused completed cleanup")
}
fresh := status.Cleanup.WorkflowName
if _, err := client.Resource(workflowGVR).Namespace("mirrored-image").Get(ctx, fresh, metav1.GetOptions{}); err != nil {
t.Fatal(err)
}
if status.Tag != "stable" {
t.Fatal("RetryPreservesTagIdentity")
}
setPhase(ctx, t, client, fresh, "Succeeded")
reconcileDesired(ctx, t, instance, &image)
reconcileDesired(ctx, t, instance, &image)
if len(image.Status.Digests) != 1 {
t.Fatal("CleanupObservedMeansGone")
}
delete(registry.stored, "candidates/pause@"+testDigest)
reconcileDesired(ctx, t, instance, &image)
if readd {
if status.Cleanup != nil || status.Prepare != nil || status.Tag != "edited" {
t.Fatal("re-addition did not wait for fresh cleanup")
}
} else if len(image.Status.Digests) != 0 {
t.Fatal("retirement did not complete")
}
})
}
}
}
func TestDigestLifecycle(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{scanPhase: scanAccepted, present: map[string]bool{}}
instance := testController(client, registry)
image := testImage()
reconcileDesired(ctx, t, instance, &image)
status := &image.Status.Digests[0]
if status.Prepare == nil || status.Prepare.WorkflowName == "" {
t.Fatal("prepare workflow was not recorded")
}
reconcileDesired(ctx, t, instance, &image)
setPhase(ctx, t, client, status.Prepare.WorkflowName, "Succeeded")
reconcileDesired(ctx, t, instance, &image)
reconcileDesired(ctx, t, instance, &image)
if image.Status.Digests[0].Scan != scanAccepted {
t.Fatal("candidate scan was not accepted")
}
reconcileDesired(ctx, t, instance, &image)
status = &image.Status.Digests[0]
if status.Promotion == nil || status.Promotion.WorkflowName == "" {
t.Fatal("promotion workflow was not recorded")
}
reconcileDesired(ctx, t, instance, &image)
setPhase(ctx, t, client, status.Promotion.WorkflowName, "Succeeded")
registry.present["mirror/pause@"+testDigest] = true
reconcileDesired(ctx, t, instance, &image)
image.Metadata.Generation++
image.Spec.Digests = nil
reconcileDesired(ctx, t, instance, &image)
status = &image.Status.Digests[0]
if status.Cleanup == nil || status.Cleanup.WorkflowName == "" {
t.Fatal("cleanup workflow was not recorded")
}
reconcileDesired(ctx, t, instance, &image)
setPhase(ctx, t, client, status.Cleanup.WorkflowName, "Succeeded")
delete(registry.present, "mirror/pause@"+testDigest)
reconcileDesired(ctx, t, instance, &image)
reconcileDesired(ctx, t, instance, &image)
if len(image.Status.Digests) != 0 {
t.Fatalf("cleanup retained %d digest statuses", len(image.Status.Digests))
}
}
func TestAttemptIsRecordedFirst(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
instance := testController(client, &fakeRegistry{present: map[string]bool{}})
image := testImage()
reconcileDesired(ctx, t, instance, &image)
if image.Status.Digests[0].Prepare == nil {
t.Fatal("prepare attempt was not recorded")
}
workflows, err := client.Resource(workflowGVR).Namespace("mirrored-image").List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
if len(workflows.Items) != 0 {
t.Fatal("workflow existed before its status record")
}
reconcileDesired(ctx, t, instance, &image)
workflows, err = client.Resource(workflowGVR).Namespace("mirrored-image").List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
if len(workflows.Items) != 1 || workflows.Items[0].GetName() != image.Status.Digests[0].Prepare.WorkflowName {
t.Fatal("recorded prepare workflow was not created")
}
}
func TestCleanupWaitsForAbsence(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{present: map[string]bool{"candidates/pause@" + testDigest: true}}
instance := testController(client, registry)
image := testImage()
image.Spec.Digests = nil
image.Status.Digests = []digestStatus{{
Digest: testDigest,
Cleanup: &attemptStatus{WorkflowName: "cleanup", Phase: "Succeeded"},
}}
reconcileDesired(ctx, t, instance, &image)
if len(image.Status.Digests) != 1 || image.Status.Message != "cleanup: awaiting repository GC" {
t.Fatal("cleanup completed while the candidate remained present")
}
delete(registry.present, "candidates/pause@"+testDigest)
reconcileDesired(ctx, t, instance, &image)
if len(image.Status.Digests) != 0 {
t.Fatal("cleanup state remained after both repositories were absent")
}
}
func TestReaddWaitsForCleanup(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{scanPhase: scanAccepted, present: map[string]bool{}}
instance := testController(client, registry)
image := testImage()
oldPrepare := attemptName(&image, testDigest, prepareStage, image.Metadata.Generation, "")
image.Metadata.Generation = 3
image.Status.Digests = []digestStatus{{
Digest: testDigest,
Tag: "stable",
Prepare: &attemptStatus{WorkflowName: oldPrepare, Phase: "Succeeded", Generation: 1},
Scan: scanAccepted,
Promotion: &attemptStatus{WorkflowName: "old-promotion", Phase: "Succeeded"},
Cleanup: &attemptStatus{WorkflowName: "old-cleanup", Phase: "Running"},
}}
workflow := workflowObject("old-cleanup", "Running")
if _, err := client.Resource(workflowGVR).Namespace("mirrored-image").Create(ctx, workflow, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
reconcileDesired(ctx, t, instance, &image)
if image.Status.Digests[0].Prepare.WorkflowName != oldPrepare {
t.Fatal("prepare restarted while cleanup was active")
}
setPhase(ctx, t, client, "old-cleanup", "Succeeded")
reconcileDesired(ctx, t, instance, &image)
reconcileDesired(ctx, t, instance, &image)
status := image.Status.Digests[0]
if status.Cleanup != nil || status.Prepare != nil {
t.Fatal("completed cleanup did not reset digest state")
}
reconcileDesired(ctx, t, instance, &image)
if image.Status.Digests[0].Prepare == nil {
t.Fatal("prepare did not restart after cleanup")
}
if image.Status.Digests[0].Prepare.WorkflowName == oldPrepare {
t.Fatal("prepare reused the workflow from an earlier generation")
}
}
func TestRetryCreatesSuccessor(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{scanPhase: scanAccepted, present: map[string]bool{}}
instance := testController(client, registry)
image := testImage()
image.Status.Digests = []digestStatus{{
Digest: testDigest,
Tag: "stable",
Prepare: &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"},
}}
image.Metadata.Annotations = map[string]string{retryAnnotation: "failed-prepare"}
retried, err := instance.reconcileRetry(ctx, &image)
if err != nil {
t.Fatal(err)
}
if !retried {
t.Fatal("terminal retry was not handled")
}
attempt := image.Status.Digests[0].Prepare
if attempt.RetryOf != "failed-prepare" || attempt.WorkflowName == "failed-prepare" {
t.Fatalf("invalid successor: %#v", attempt)
}
if image.Status.LastHandledRetry != "" {
t.Fatal("retry was acknowledged before the successor existed")
}
retried, err = instance.reconcileRetry(ctx, &image)
if err != nil || !retried {
t.Fatalf("create retry successor: retried=%v error=%v", retried, err)
}
if image.Status.LastHandledRetry != "failed-prepare" {
t.Fatal("retry was not acknowledged after the successor existed")
}
setPhase(ctx, t, client, attempt.WorkflowName, "Succeeded")
reconcileDesired(ctx, t, instance, &image)
reconcileDesired(ctx, t, instance, &image)
reconcileDesired(ctx, t, instance, &image)
promotion := image.Status.Digests[0].Promotion
if promotion == nil || promotion.WorkflowName == "" {
t.Fatal("promotion was not created after the retry")
}
oldName := attemptName(&image, testDigest, promotionStage, image.Metadata.Generation)
if promotion.WorkflowName == oldName {
t.Fatal("promotion reused the predecessor prepare lineage")
}
}
func TestRetryNameIgnoresGeneration(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
instance := testController(client, &fakeRegistry{present: map[string]bool{}})
image := testImage()
image.Status.Digests = []digestStatus{{
Digest: testDigest,
Prepare: &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"},
}}
image.Metadata.Annotations = map[string]string{retryAnnotation: "failed-prepare"}
if _, err := instance.reconcileRetry(ctx, &image); err != nil {
t.Fatal(err)
}
first := image.Status.Digests[0].Prepare.WorkflowName
image.Metadata.Generation++
image.Status.Digests[0].Prepare = &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"}
image.Status.LastHandledRetry = ""
if _, err := instance.reconcileRetry(ctx, &image); err != nil {
t.Fatal(err)
}
if second := image.Status.Digests[0].Prepare.WorkflowName; second != first {
t.Fatalf("retry generation changed successor from %q to %q", first, second)
}
}
func TestRunStopsWhenIdle(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())
client := newClient()
instance := testController(client, &fakeRegistry{present: map[string]bool{}})
done := make(chan error, 1)
go func() {
done <- instance.run(ctx)
}()
time.Sleep(20 * time.Millisecond)
cancel()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("run returned %v", err)
}
case <-time.After(time.Second):
t.Fatal("run did not stop")
}
}
func TestRejectedCandidateIsNotPromoted(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{scanPhase: scanRejected, present: map[string]bool{}}
instance := testController(client, registry)
image := testImage()
image.Status.Digests = []digestStatus{{
Digest: testDigest,
Tag: "stable",
Prepare: &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
}}
reconcileDesired(ctx, t, instance, &image)
status := image.Status.Digests[0]
if status.Scan != scanRejected || status.Promotion != nil {
t.Fatal("rejected candidate was promoted")
}
if image.Status.Message != "scan: Rejected" {
t.Fatalf("unexpected status message %q", image.Status.Message)
}
}
func TestTaglessDeclarationIsValid(t *testing.T) {
t.Parallel()
image := testImage()
image.Spec.Digests[0].Tag = ""
if err := validateImage(&image); err != nil {
t.Fatal(err)
}
}
func TestTagEditPreservesState(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{present: map[string]bool{"mirror/pause@" + testDigest: true}}
instance := testController(client, registry)
image := testImage()
image.Spec.Digests[0].Tag = "new"
image.Status.Digests = []digestStatus{{
Digest: testDigest,
Tag: "stable",
Prepare: &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
Scan: scanAccepted,
Promotion: &attemptStatus{WorkflowName: "promotion", Phase: "Succeeded"},
}}
reconcileDesired(ctx, t, instance, &image)
status := image.Status.Digests[0]
if status.Tag != "stable" || status.Cleanup != nil {
t.Fatalf("tag edit changed managed state: %#v", status)
}
workflows, err := client.Resource(workflowGVR).Namespace("mirrored-image").List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
if len(workflows.Items) != 0 {
t.Fatal("tag edit created a workflow")
}
}
func TestFailureMessage(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{present: map[string]bool{}}
instance := testController(client, registry)
image := testImage()
image.Status.Digests = []digestStatus{{
Digest: testDigest,
Tag: "stable",
Prepare: &attemptStatus{WorkflowName: "failed-prepare", Phase: "Failed"},
}}
reconcileDesired(ctx, t, instance, &image)
if image.Status.Message != "prepare: Failed" {
t.Fatalf("unexpected status message %q", image.Status.Message)
}
}
func TestPromotionRescan(t *testing.T) {
for _, phase := range []string{scanPending, scanRejected, scanAccepted} {
t.Run(phase, func(t *testing.T) {
ctx := context.Background()
client := newClient()
registry := &fakeRegistry{scanPhase: phase}
instance := testController(client, registry)
image := testImage()
image.Status.Digests = []digestStatus{{
Digest: testDigest, Scan: scanAccepted,
Prepare: &attemptStatus{WorkflowName: "prepare", Phase: "Succeeded"},
Promotion: &attemptStatus{WorkflowName: "promotion", Phase: "Pending"},
}}
err := instance.reconcileDesired(ctx, &image)
if (err == nil) != (phase == scanAccepted) {
t.Fatalf("scan %s: %v", phase, err)
}
workflows, err := client.Resource(workflowGVR).Namespace(image.Metadata.Namespace).List(ctx, metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
if (len(workflows.Items) == 1) != (phase == scanAccepted) {
t.Fatalf("scan %s created %d workflows", phase, len(workflows.Items))
}
if phase == scanAccepted {
registry.scanPhase = scanPending
if err := instance.reconcileDesired(ctx, &image); err != nil {
t.Fatalf("existing workflow observation requires a scan: %v", err)
}
}
})
}
}
func testController(client *fake.FakeDynamicClient, registry registryReader) *controller {
return &controller{client: client, registry: registry, config: controllerConfig{
prepareTemplate: "image-prepare",
promotionTemplate: "image-promote",
cleanupTemplate: "image-cleanup",
}}
}
func newClient(objects ...runtime.Object) *fake.FakeDynamicClient {
return fake.NewSimpleDynamicClientWithCustomListKinds(
runtime.NewScheme(),
map[schema.GroupVersionResource]string{
imageGVR: "MirroredImageList",
workflowGVR: "WorkflowList",
},
objects...,
)
}
func testImage() mirroredImage {
return mirroredImage{
Metadata: objectMetadata{Name: "pause", Namespace: "mirrored-image", UID: "image-uid", Generation: 1},
Spec: mirroredImageSpec{
Upstream: "registry.k8s.io/pause",
Policy: scanPolicy{RejectAtOrAbove: "High"},
Digests: []digestSpec{{Digest: testDigest, Tag: "stable"}},
},
}
}
func reconcileDesired(ctx context.Context, t *testing.T, instance *controller, image *mirroredImage) {
t.Helper()
if err := instance.reconcileDesired(ctx, image); err != nil {
t.Fatal(err)
}
}
func setPhase(ctx context.Context, t *testing.T, client *fake.FakeDynamicClient, name, phase string) {
t.Helper()
resource := client.Resource(workflowGVR).Namespace("mirrored-image")
object, err := resource.Get(ctx, name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if err := unstructured.SetNestedField(object.Object, phase, "status", "phase"); err != nil {
t.Fatal(err)
}
if _, err := resource.Update(ctx, object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
}
func workflowObject(name, phase string) *unstructured.Unstructured {
return &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "argoproj.io/v1alpha1",
"kind": "Workflow",
"metadata": map[string]any{"name": name, "namespace": "mirrored-image"},
"status": map[string]any{"phase": phase},
}}
}

57
services/mirrored-image/controller/go.mod Normal file
View file

@ -0,0 +1,57 @@
module quorate.dev/services/mirrored-image-controller
go 1.27.1
require (
github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
k8s.io/apimachinery v0.37.0
k8s.io/client-go v0.37.0
)
require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emicklei/go-restful/v3 v3.13.0 // indirect
github.com/fxamacker/cbor/v2 v2.9.1 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-openapi/jsonpointer v1.0.0 // indirect
github.com/go-openapi/jsonreference v1.0.0 // indirect
github.com/go-openapi/swag v0.27.1 // indirect
github.com/go-openapi/swag/cmdutils v0.27.1 // indirect
github.com/go-openapi/swag/conv v0.27.1 // indirect
github.com/go-openapi/swag/fileutils v0.27.1 // indirect
github.com/go-openapi/swag/jsonutils v0.27.1 // indirect
github.com/go-openapi/swag/loading v0.27.1 // indirect
github.com/go-openapi/swag/mangling v0.27.1 // indirect
github.com/go-openapi/swag/netutils v0.27.1 // indirect
github.com/go-openapi/swag/pools v0.27.1 // indirect
github.com/go-openapi/swag/stringutils v0.27.1 // indirect
github.com/go-openapi/swag/typeutils v0.27.1 // indirect
github.com/go-openapi/swag/yamlutils v0.27.1 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/term v0.45.0 // indirect
golang.org/x/text v0.40.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/api v0.37.0 // indirect
k8s.io/klog/v2 v2.140.0 // indirect
k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect
k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)

133
services/mirrored-image/controller/go.sum Normal file
View file

@ -0,0 +1,133 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s=
github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y=
github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY=
github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0=
github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg=
github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE=
github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE=
github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM=
github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU=
github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs=
github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM=
github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8=
github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU=
github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8=
github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY=
github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY=
github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE=
github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA=
github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w=
github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU=
github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ=
github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E=
github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE=
github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8=
github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM=
github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc=
github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ=
github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA=
github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo=
github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0=
github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo=
github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f h1:GvCU5GXhHq+7LeOzx/haG7HSIZokl3/0GkoUFzsRJjg=
github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f/go.mod h1:q59u9px8b7UTj0nIjEjvmTWekazka6xIt6Uogz5Dm+8=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4=
k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk=
k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0=
k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE=
k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ=
k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A=
k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I=
k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs=
k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q=
sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

96
services/mirrored-image/controller/inventory.go Normal file
View file

@ -0,0 +1,96 @@
package main
import (
"fmt"
apk "github.com/knqyf263/go-apk-version"
)
type scanInventory struct {
Hash string `json:"manifest_hash"`
Packages map[string]*scanPackage `json:"packages"`
Distributions map[string]*scanDistribution `json:"distributions"`
Environments map[string][]*scanEnvironment `json:"environments"`
}
type scanPackage struct {
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Kind string `json:"kind"`
Source *scanPackage `json:"source"`
}
type scanDistribution struct {
ID string `json:"id"`
DID string `json:"did"`
Name string `json:"name"`
Version string `json:"version"`
PrettyName string `json:"pretty_name"`
}
type scanEnvironment struct {
PackageDB string `json:"package_db"`
IntroducedIn string `json:"introduced_in"`
DistributionID string `json:"distribution_id"`
RepositoryIDs []string `json:"repository_ids"`
}
type featureKey struct {
Name string
Version string
Layer string
}
func checkInventory(data *securityData, digest string, layers map[string]bool) error {
if data == nil || data.Inventory == nil || data.Layer == nil || data.Layer.Features == nil {
return fmt.Errorf("scan inventory is unavailable")
}
inv := data.Inventory
if inv.Hash != digest || data.Layer.Name != digest {
return fmt.Errorf("scan identity does not match candidate")
}
if len(inv.Packages) == 0 || len(inv.Distributions) == 0 || len(inv.Environments) != len(inv.Packages) {
return fmt.Errorf("scan inventory is incomplete")
}
for id, distro := range inv.Distributions {
if distro == nil || distro.ID != id || distro.DID != "alpine" || distro.Version != "3.23" || distro.Name != "Alpine Linux" || distro.PrettyName != "Alpine Linux v3.23" {
return fmt.Errorf("distribution coverage is not qualified")
}
}
expected := map[featureKey]int{}
for id, pkg := range inv.Packages {
if pkg == nil || pkg.ID != id || pkg.Name == "" || pkg.Version == "" || pkg.Kind != "binary" {
return fmt.Errorf("package identity is incomplete")
}
if _, err := apk.NewVersion(pkg.Version); err != nil {
return fmt.Errorf("package version is not matchable")
}
if pkg.Source == nil || pkg.Source.Name == "" || pkg.Source.Kind != "source" {
return fmt.Errorf("source package identity is missing")
}
envs := inv.Environments[id]
if len(envs) == 0 {
return fmt.Errorf("package environment is missing")
}
for _, env := range envs {
if env == nil || env.PackageDB != "lib/apk/db/installed" || !layers[env.IntroducedIn] || inv.Distributions[env.DistributionID] == nil || len(env.RepositoryIDs) != 0 {
return fmt.Errorf("package environment is not qualified")
}
}
expected[featureKey{pkg.Name, pkg.Version, envs[0].IntroducedIn}]++
}
for _, feature := range *data.Layer.Features {
key := featureKey{feature.Name, feature.Version, feature.AddedBy}
if expected[key] == 0 {
return fmt.Errorf("scan feature does not match inventory")
}
expected[key]--
}
for _, count := range expected {
if count != 0 {
return fmt.Errorf("scan features omit indexed packages")
}
}
return nil
}

55
services/mirrored-image/controller/inventory_test.go Normal file
View file

@ -0,0 +1,55 @@
package main
import "testing"
func inventoryFixture() *securityData {
return &securityData{
Layer: &securityLayer{Name: testDigest, Features: &[]securityFeature{{Name: "busybox", Version: "1.37.0-r30", AddedBy: testDigest}}},
Inventory: &scanInventory{Hash: testDigest,
Packages: map[string]*scanPackage{"p": {ID: "p", Name: "busybox", Version: "1.37.0-r30", Kind: "binary", Source: &scanPackage{Name: "busybox", Kind: "source"}}},
Distributions: map[string]*scanDistribution{"d": {ID: "d", DID: "alpine", Version: "3.23", Name: "Alpine Linux", PrettyName: "Alpine Linux v3.23"}},
Environments: map[string][]*scanEnvironment{"p": {{PackageDB: "lib/apk/db/installed", IntroducedIn: testDigest, DistributionID: "d"}}},
},
}
}
func TestInventory(t *testing.T) {
for _, test := range []struct {
name string
mutate func(*securityData)
}{
{"qualified", func(*securityData) {}},
{"missing", func(d *securityData) { d.Inventory = nil }},
{"identity", func(d *securityData) { d.Inventory.Hash = childDigest }},
{"layer identity", func(d *securityData) { d.Layer.Name = childDigest }},
{"empty", func(d *securityData) { d.Inventory.Packages = nil }},
{"no distribution", func(d *securityData) { d.Inventory.Distributions = nil }},
{"Amazon Linux", func(d *securityData) { d.Inventory.Distributions["d"].DID = "amzn" }},
{"RHEL", func(d *securityData) { d.Inventory.Distributions["d"].DID = "rhel" }},
{"unknown", func(d *securityData) { d.Inventory.Distributions["d"].DID = "unknown" }},
{"unqualified release", func(d *securityData) { d.Inventory.Distributions["d"].Version = "3.2" }},
{"missing package", func(d *securityData) { d.Inventory.Packages["p"] = nil }},
{"invalid version", func(d *securityData) { d.Inventory.Packages["p"].Version = "invalid" }},
{"missing source", func(d *securityData) { d.Inventory.Packages["p"].Source = nil }},
{"empty source", func(d *securityData) { d.Inventory.Packages["p"].Source.Name = "" }},
{"foreign package", func(d *securityData) { d.Inventory.Packages["p"].Kind = "python" }},
{"missing environment", func(d *securityData) { d.Inventory.Environments["p"] = nil }},
{"foreign layer", func(d *securityData) { d.Inventory.Environments["p"][0].IntroducedIn = childDigest }},
{"foreign database", func(d *securityData) { d.Inventory.Environments["p"][0].PackageDB = "var/lib/rpm" }},
{"second environment", func(d *securityData) {
d.Inventory.Environments["p"] = append(d.Inventory.Environments["p"], &scanEnvironment{PackageDB: "var/lib/rpm"})
}},
{"missing features", func(d *securityData) { *d.Layer.Features = nil }},
{"wrong feature", func(d *securityData) { (*d.Layer.Features)[0].Version = "1-r1" }},
{"duplicate feature", func(d *securityData) { *d.Layer.Features = append(*d.Layer.Features, (*d.Layer.Features)[0]) }},
} {
t.Run(test.name, func(t *testing.T) {
data := inventoryFixture()
test.mutate(data)
err := checkInventory(data, testDigest, map[string]bool{testDigest: true})
if (err == nil) != (test.name == "qualified") {
t.Fatalf("unexpected inventory verdict: %v", err)
}
})
}
}

87
services/mirrored-image/controller/main.go Normal file
View file

@ -0,0 +1,87 @@
package main
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"time"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Args[1:]); err != nil && !errors.Is(err, context.Canceled) {
slog.Error("image mirror", "error", err)
os.Exit(1)
}
}
func run(ctx context.Context, arguments []string) error {
if len(arguments) != 1 {
return errors.New("expected one command: run")
}
if arguments[0] != "run" {
return fmt.Errorf("unknown command %q", arguments[0])
}
clientConfig, err := rest.InClusterConfig()
if err != nil {
return fmt.Errorf("load Kubernetes configuration: %w", err)
}
client, err := dynamic.NewForConfig(clientConfig)
if err != nil {
return fmt.Errorf("create Kubernetes client: %w", err)
}
config, err := loadConfig()
if err != nil {
return err
}
registry, err := loadRegistry()
if err != nil {
return err
}
return (&controller{client: client, registry: registry, config: config}).run(ctx)
}
func loadConfig() (controllerConfig, error) {
namespace := os.Getenv("POD_NAMESPACE")
if namespace == "" {
return controllerConfig{}, errors.New("POD_NAMESPACE is required")
}
pollInterval, err := durationEnv("POLL_INTERVAL", 20*time.Second)
if err != nil {
return controllerConfig{}, err
}
return controllerConfig{
namespace: namespace,
pollInterval: pollInterval,
prepareTemplate: valueOrDefault("PREPARE_TEMPLATE", "image-prepare"),
promotionTemplate: valueOrDefault("PROMOTION_TEMPLATE", "image-promote"),
cleanupTemplate: valueOrDefault("CLEANUP_TEMPLATE", "image-cleanup"),
}, nil
}
func durationEnv(name string, fallback time.Duration) (time.Duration, error) {
value := os.Getenv(name)
if value == "" {
return fallback, nil
}
duration, err := time.ParseDuration(value)
if err != nil || duration <= 0 {
return 0, fmt.Errorf("%s must be a positive duration", name)
}
return duration, nil
}
func valueOrDefault(name, fallback string) string {
if value := os.Getenv(name); value != "" {
return value
}
return fallback
}

400
services/mirrored-image/controller/registry.go Normal file
View file

@ -0,0 +1,400 @@
package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
)
const oauthUser = "$oauthtoken"
var challengeRule = regexp.MustCompile(`([A-Za-z]+)="([^"]*)"`)
type quayReader struct {
api *url.URL
registry string
client *http.Client
candidates string
mirror string
}
type manifestDoc struct {
Manifests json.RawMessage `json:"manifests"`
Layers []descriptor `json:"layers"`
}
type descriptor struct {
Digest string `json:"digest"`
}
type securityDoc struct {
Status string `json:"status"`
Data *securityData `json:"data"`
}
type securityData struct {
Layer *securityLayer `json:"Layer"`
Inventory *scanInventory `json:"Inventory"`
}
type securityLayer struct {
Name string `json:"Name"`
Features *[]securityFeature `json:"Features"`
}
type securityFeature struct {
Name string `json:"Name"`
Version string `json:"Version"`
AddedBy string `json:"AddedBy"`
Vulnerabilities []vulnerability `json:"Vulnerabilities"`
}
type vulnerability struct {
Severity string `json:"Severity"`
}
func loadRegistry() (*quayReader, error) {
registry := os.Getenv("REGISTRY")
if registry == "" {
return nil, errors.New("REGISTRY is required")
}
apiValue := os.Getenv("QUAY_API")
if apiValue == "" {
return nil, errors.New("QUAY_API is required")
}
api, err := url.Parse(apiValue)
if err != nil || api.Host == "" || (api.Scheme != "http" && api.Scheme != "https") {
return nil, errors.New("QUAY_API must be an HTTP or HTTPS origin")
}
transport := http.DefaultTransport.(*http.Transport).Clone()
if api.Scheme == "https" {
roots, err := x509.SystemCertPool()
if err != nil {
return nil, fmt.Errorf("load system roots: %w", err)
}
certificate, err := os.ReadFile(valueOrDefault("REGISTRY_CA", "/registry-ca/ca.crt"))
if err != nil {
return nil, fmt.Errorf("read registry CA: %w", err)
}
if !roots.AppendCertsFromPEM(certificate) {
return nil, errors.New("registry CA contains no certificate")
}
transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: roots}
}
candidates, err := readSecret(valueOrDefault("CANDIDATES_TOKEN", "/candidates-reader/token"))
if err != nil {
return nil, err
}
mirror, err := readSecret(valueOrDefault("MIRROR_TOKEN", "/mirror-reader/token"))
if err != nil {
return nil, err
}
return &quayReader{
api: api,
registry: registry,
client: &http.Client{Timeout: 20 * time.Second, Transport: transport},
candidates: candidates,
mirror: mirror,
}, nil
}
func readSecret(path string) (string, error) {
content, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read %s: %w", path, err)
}
value := strings.TrimSpace(string(content))
if value == "" {
return "", fmt.Errorf("secret %s is empty", path)
}
return value, nil
}
func (q *quayReader) scan(ctx context.Context, repository, digest, threshold string) (string, error) {
content, found, err := q.manifest(ctx, repository, digest)
if err != nil {
return "", err
}
if !found {
return scanPending, nil
}
if fmt.Sprintf("sha256:%x", sha256.Sum256(content)) != digest {
return "", errors.New("candidate digest does not match manifest")
}
digests, err := scanDigests(content, digest)
if err != nil {
return "", err
}
limit, err := severityRank(threshold)
if err != nil {
return "", err
}
for _, child := range digests {
result, found, err := q.security(ctx, repository, child)
if err != nil {
return "", err
}
if !found || result.Status == "queued" || result.Status == "scanning" || result.Status == "unsupported" {
return scanPending, nil
}
if result.Status != "scanned" {
return "", fmt.Errorf("clair returned unusable status %q for %s", result.Status, child)
}
if result.Data == nil || result.Data.Layer == nil || result.Data.Layer.Features == nil {
return "", fmt.Errorf("clair scan data is unavailable for %s", child)
}
manifest := content
if child != digest {
manifest, found, err = q.manifest(ctx, repository, child)
if err != nil || !found {
return scanPending, err
}
}
if fmt.Sprintf("sha256:%x", sha256.Sum256(manifest)) != child {
return "", errors.New("child digest does not match manifest")
}
var document manifestDoc
if err := json.Unmarshal(manifest, &document); err != nil {
return "", errors.New("candidate manifest is invalid")
}
if len(document.Manifests) != 0 || len(document.Layers) == 0 {
return scanRejected, nil
}
layers := map[string]bool{}
for _, layer := range document.Layers {
if !digestRule.MatchString(layer.Digest) {
return "", errors.New("candidate layer digest is invalid")
}
layers[layer.Digest] = true
}
if err := checkInventory(result.Data, child, layers); err != nil {
return scanRejected, nil
}
for _, feature := range *result.Data.Layer.Features {
for _, finding := range feature.Vulnerabilities {
rank, err := severityRank(finding.Severity)
if err != nil {
return "", err
}
if rank == 0 || rank >= limit {
return scanRejected, nil
}
}
}
}
return scanAccepted, nil
}
func scanDigests(content []byte, digest string) ([]string, error) {
var manifest manifestDoc
if err := json.Unmarshal(content, &manifest); err != nil {
return nil, fmt.Errorf("decode candidate manifest: %w", err)
}
if len(manifest.Manifests) == 0 {
return []string{digest}, nil
}
var children []descriptor
if err := json.Unmarshal(manifest.Manifests, &children); err != nil {
return nil, fmt.Errorf("decode candidate index: %w", err)
}
if len(children) == 0 {
return nil, errors.New("candidate index contains no manifests")
}
digests := make([]string, 0, len(children))
for _, child := range children {
if !digestRule.MatchString(child.Digest) {
return nil, fmt.Errorf("candidate index contains invalid digest %q", child.Digest)
}
digests = append(digests, child.Digest)
}
return digests, nil
}
func severityRank(value string) (int, error) {
switch strings.ToLower(value) {
case "unknown":
return 0, nil
case "negligible":
return 1, nil
case "low":
return 2, nil
case "medium":
return 3, nil
case "high":
return 4, nil
case "critical":
return 5, nil
default:
return 0, fmt.Errorf("unknown Clair severity %q", value)
}
}
func (q *quayReader) exists(ctx context.Context, repository, digest string) (bool, error) {
_, found, err := q.manifest(ctx, repository, digest)
return found, err
}
func (q *quayReader) physicallyPresent(ctx context.Context, repository, digest string) (bool, error) {
response, err := q.apiGet(ctx, repository, digest, "/security?vulnerabilities=false")
if err != nil {
return false, err
}
defer func() { _ = response.Body.Close() }()
switch response.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusNotFound:
return false, nil
default:
return false, fmt.Errorf("quay physical manifest lookup failed with HTTP %d", response.StatusCode)
}
}
func (q *quayReader) security(ctx context.Context, repository, digest string) (securityDoc, bool, error) {
response, err := q.apiGet(ctx, repository, digest, "/security?vulnerabilities=true")
if err != nil {
return securityDoc{}, false, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode == http.StatusNotFound {
return securityDoc{}, false, nil
}
if response.StatusCode != http.StatusOK {
return securityDoc{}, false, fmt.Errorf("quay security lookup failed with HTTP %d", response.StatusCode)
}
var result securityDoc
if err := json.NewDecoder(io.LimitReader(response.Body, 8<<20)).Decode(&result); err != nil {
return securityDoc{}, false, fmt.Errorf("decode Quay security result: %w", err)
}
return result, true, nil
}
func (q *quayReader) apiGet(ctx context.Context, repository, digest, suffix string) (*http.Response, error) {
endpoint := *q.api
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/api/v1/repository/" + repository + "/manifest/" + digest
endpoint.RawQuery = ""
if queryIndex := strings.IndexByte(suffix, '?'); queryIndex >= 0 {
endpoint.Path += suffix[:queryIndex]
endpoint.RawQuery = suffix[queryIndex+1:]
} else {
endpoint.Path += suffix
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
if err != nil {
return nil, err
}
request.Header.Set("Authorization", "Bearer "+q.token(repository))
return q.client.Do(request)
}
func (q *quayReader) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) {
endpoint := fmt.Sprintf("%s://%s/v2/%s/manifests/%s", q.api.Scheme, q.registry, repository, digest)
send := func(authorization string) (*http.Response, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return nil, err
}
request.Header.Set("Accept", strings.Join([]string{
"application/vnd.oci.image.index.v1+json",
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.docker.distribution.manifest.list.v2+json",
"application/vnd.docker.distribution.manifest.v2+json",
}, ", "))
if authorization != "" {
request.Header.Set("Authorization", authorization)
}
return q.client.Do(request)
}
response, err := send("")
if err != nil {
return nil, false, err
}
if response.StatusCode == http.StatusUnauthorized {
challenge := response.Header.Get("WWW-Authenticate")
if err := response.Body.Close(); err != nil {
return nil, false, fmt.Errorf("close registry challenge: %w", err)
}
token, err := q.bearerToken(ctx, challenge, repository)
if err != nil {
return nil, false, err
}
response, err = send("Bearer " + token)
if err != nil {
return nil, false, err
}
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode == http.StatusNotFound {
return nil, false, nil
}
if response.StatusCode != http.StatusOK {
return nil, false, fmt.Errorf("registry manifest lookup failed with HTTP %d", response.StatusCode)
}
content, err := io.ReadAll(io.LimitReader(response.Body, 8<<20))
if err != nil {
return nil, false, fmt.Errorf("read candidate manifest: %w", err)
}
return content, true, nil
}
func (q *quayReader) bearerToken(ctx context.Context, challenge, repository string) (string, error) {
values := make(map[string]string)
for _, match := range challengeRule.FindAllStringSubmatch(challenge, -1) {
values[strings.ToLower(match[1])] = match[2]
}
realm, err := url.Parse(values["realm"])
if err != nil || realm.Host == "" {
return "", errors.New("registry challenge omitted a valid realm")
}
query := realm.Query()
query.Set("service", values["service"])
query.Set("scope", values["scope"])
query.Set("account", oauthUser)
realm.RawQuery = query.Encode()
request, err := http.NewRequestWithContext(ctx, http.MethodGet, realm.String(), nil)
if err != nil {
return "", err
}
request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(oauthUser+":"+q.token(repository))))
response, err := q.client.Do(request)
if err != nil {
return "", err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return "", fmt.Errorf("registry authentication failed with HTTP %d", response.StatusCode)
}
var result struct {
Token string `json:"token"`
AccessToken string `json:"access_token"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil {
return "", fmt.Errorf("decode registry token: %w", err)
}
if result.Token != "" {
return result.Token, nil
}
if result.AccessToken != "" {
return result.AccessToken, nil
}
return "", errors.New("registry authentication returned no token")
}
func (q *quayReader) token(repository string) string {
if strings.HasPrefix(repository, "candidates/") {
return q.candidates
}
return q.mirror
}

163
services/mirrored-image/controller/registry_test.go Normal file
View file

@ -0,0 +1,163 @@
package main
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
const childDigest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
func TestPhysicalPresence(t *testing.T) {
t.Parallel()
for _, status := range []int{200, 404, 401, 403, 429, 500, 502} {
t.Run(fmt.Sprint(status), func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/api/v1/repository/candidates/pause/manifest/"+testDigest+"/security" || request.URL.Query().Get("vulnerabilities") != "false" {
t.Errorf("unexpected lookup: %s", request.URL)
}
if request.Header.Get("Authorization") != "Bearer reader-token" {
t.Error("lookup did not use reader authority")
}
response.WriteHeader(status)
_, _ = fmt.Fprint(response, `{"status":"queued"}`)
}))
defer server.Close()
api, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
reader := &quayReader{api: api, client: server.Client(), candidates: "reader-token"}
present, err := reader.physicallyPresent(context.Background(), "candidates/pause", testDigest)
if present != (status == 200) || (err != nil) != (status != 200 && status != 404) {
t.Fatalf("HTTP %d: present=%t err=%v", status, present, err)
}
})
}
}
func TestQuayScanAndExistence(t *testing.T) {
t.Parallel()
child := fmt.Sprintf(`{"schemaVersion":2,"layers":[{"digest":%q}]}`, testDigest)
childDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(child)))
index := fmt.Sprintf(`{"manifests":[{"digest":%q}]}`, childDigest)
testDigest := fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(index)))
severity := "Low"
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
switch {
case request.URL.Path == "/v2/candidates/pause/manifests/"+childDigest:
_, _ = fmt.Fprint(response, child)
case request.URL.Path == "/v2/auth":
username, password, found := request.BasicAuth()
if !found || username != oauthUser || (password != "candidate-token" && password != "mirror-token") {
http.Error(response, "unauthorized", http.StatusUnauthorized)
return
}
response.Header().Set("Content-Type", "application/json")
if _, err := fmt.Fprintf(response, `{"token":%q}`, password+"-registry"); err != nil {
t.Errorf("write token response: %v", err)
}
case request.URL.Path == "/v2/candidates/pause/manifests/"+testDigest || request.URL.Path == "/v2/mirror/pause/manifests/"+testDigest:
expected := "Bearer candidate-token-registry"
if strings.Contains(request.URL.Path, "/mirror/") {
expected = "Bearer mirror-token-registry"
}
if request.Header.Get("Authorization") != expected {
response.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/v2/auth",service="fixture",scope="repository:test:pull"`)
response.WriteHeader(http.StatusUnauthorized)
return
}
if strings.Contains(request.URL.Path, "/mirror/") {
response.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json")
if _, err := fmt.Fprint(response, `{"schemaVersion":2}`); err != nil {
t.Errorf("write manifest response: %v", err)
}
return
}
response.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json")
if _, err := fmt.Fprintf(response, `{"manifests":[{"digest":%q}]}`, childDigest); err != nil {
t.Errorf("write index response: %v", err)
}
case strings.HasSuffix(request.URL.Path, "/security"):
if request.Header.Get("Authorization") != "Bearer candidate-token" || request.URL.Query().Get("vulnerabilities") != "true" {
http.Error(response, "unauthorized", http.StatusUnauthorized)
return
}
data := inventoryFixture()
data.Inventory.Hash = childDigest
data.Layer.Name = childDigest
(*data.Layer.Features)[0].Vulnerabilities = []vulnerability{{Severity: severity}}
if err := json.NewEncoder(response).Encode(securityDoc{Status: "scanned", Data: data}); err != nil {
t.Errorf("write scan response: %v", err)
}
default:
http.NotFound(response, request)
}
}))
defer server.Close()
api, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
reader := &quayReader{
api: api,
registry: server.Listener.Addr().String(),
client: server.Client(),
candidates: "candidate-token",
mirror: "mirror-token",
}
phase, err := reader.scan(context.Background(), "candidates/pause", testDigest, "High")
if err != nil {
t.Fatal(err)
}
if phase != scanAccepted {
t.Fatalf("unexpected accepted phase %q", phase)
}
severity = "Critical"
phase, err = reader.scan(context.Background(), "candidates/pause", testDigest, "High")
if err != nil {
t.Fatal(err)
}
if phase != scanRejected {
t.Fatalf("unexpected rejected phase %q", phase)
}
severity = "Unknown"
phase, err = reader.scan(context.Background(), "candidates/pause", testDigest, "Critical")
if err != nil || phase != scanRejected {
t.Fatalf("unknown severity: %s %v", phase, err)
}
present, err := reader.exists(context.Background(), "mirror/pause", testDigest)
if err != nil {
t.Fatal(err)
}
if !present {
t.Fatal("promoted manifest was not observed")
}
present, err = reader.exists(context.Background(), "candidates/missing", testDigest)
if err != nil {
t.Fatal(err)
}
if present {
t.Fatal("missing manifest was observed")
}
}
func TestSingleManifestScanDigest(t *testing.T) {
t.Parallel()
digests, err := scanDigests([]byte(`{"schemaVersion":2}`), testDigest)
if err != nil {
t.Fatal(err)
}
if len(digests) != 1 || digests[0] != testDigest {
t.Fatalf("unexpected scan digests %#v", digests)
}
}

61
services/mirrored-image/controller/types.go Normal file
View file

@ -0,0 +1,61 @@
package main
import "time"
const (
retryAnnotation = "mirror.quorate.dev/retry"
purposeLabel = "mirror.quorate.dev/purpose"
)
type mirroredImage struct {
Metadata objectMetadata `json:"metadata"`
Spec mirroredImageSpec `json:"spec"`
Status mirroredImageStatus `json:"status,omitempty"`
}
type objectMetadata struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
UID string `json:"uid"`
Generation int64 `json:"generation"`
Annotations map[string]string `json:"annotations,omitempty"`
DeletionTimestamp *time.Time `json:"deletionTimestamp,omitempty"`
}
type mirroredImageSpec struct {
Upstream string `json:"upstream"`
Policy scanPolicy `json:"policy"`
Digests []digestSpec `json:"digests"`
}
type scanPolicy struct {
RejectAtOrAbove string `json:"rejectAtOrAbove"`
}
type digestSpec struct {
Digest string `json:"digest"`
Tag string `json:"tag,omitempty"`
}
type mirroredImageStatus struct {
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
Message string `json:"message,omitempty"`
LastHandledRetry string `json:"lastHandledRetry,omitempty"`
Digests []digestStatus `json:"digests,omitempty"`
}
type digestStatus struct {
Digest string `json:"digest"`
Tag string `json:"tag,omitempty"`
Prepare *attemptStatus `json:"prepare,omitempty"`
Scan string `json:"scan,omitempty"`
Promotion *attemptStatus `json:"promotion,omitempty"`
Cleanup *attemptStatus `json:"cleanup,omitempty"`
}
type attemptStatus struct {
WorkflowName string `json:"workflowName"`
RetryOf string `json:"retryOf,omitempty"`
Phase string `json:"phase"`
Generation int64 `json:"generation"`
}