package main import ( "context" "encoding/json" "errors" "reflect" "sync" "testing" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" k8stesting "k8s.io/client-go/testing" ) func TestContinuousCreatesBuildWithOpaqueTargets(t *testing.T) { revision := "0123456789abcdef0123456789abcdef01234567" refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}} now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC) object := continuousDelivery() object.Object["spec"].(map[string]any)["source"].(map[string]any)["repository"] = "https://forge.quorate.internal/quorate/cluster" targets := []any{map[string]any{ "kind": "Git", "repository": "https://git.quorate.dev/ms2k/quorate", "ref": "refs/heads/master", "options": map[string]any{"force": true}, }} object.Object["spec"].(map[string]any)["targets"] = targets instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get( ctx, workflowName("build", string(object.GetUID()), "continuous", revision), metav1.GetOptions{}, ) if err != nil { t.Fatal(err) } if workflowHasParameter(t, build, "input-plan") { t.Fatal("controller supplied input-plan") } if mutex := workflowMutex(t, build); mutex != "nuthatch-delivery" { t.Fatalf("Workflow mutex = %q", mutex) } var passedTargets []any if err := json.Unmarshal([]byte(workflowParameter(t, build, "targets")), &passedTargets); err != nil { t.Fatal(err) } if !reflect.DeepEqual(passedTargets, targets) { t.Fatalf("targets = %#v", passedTargets) } } func TestDuplicateTargetsAreRejected(t *testing.T) { object := continuousDelivery() target := map[string]any{"kind": "QuayOCI", "repository": "quorate/nuthatch"} object.Object["spec"].(map[string]any)["targets"] = []any{target, target} instance := testController(t, object, &mutableRefs{}, new(time.Time)) err := instance.reconcile(context.Background(), object) if err == nil || err.Error() != "duplicate target at index 1" { t.Fatalf("reconcile error = %v", err) } } func TestWorkflowMutexIsScopedByTemplateReference(t *testing.T) { now := time.Date(2026, time.August, 25, 0, 0, 0, 0, time.UTC) object := continuousDelivery() instance := testController(t, object, &mutableRefs{}, &now) delivery, err := decodeProjectDelivery(object) if err != nil { t.Fatal(err) } ctx := context.Background() for _, item := range []struct { name string template string }{ {name: "first", template: "shared-delivery"}, {name: "second", template: "shared-delivery"}, {name: "third", template: "other-delivery"}, } { if err := instance.ensureWorkflow(ctx, &delivery, item.name, item.template, "build", map[string]string{}, nil); err != nil { t.Fatal(err) } workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, item.name, metav1.GetOptions{}) if err != nil { t.Fatal(err) } if mutex := workflowMutex(t, workflow); mutex != item.template { t.Fatalf("Workflow %q mutex = %q", item.name, mutex) } } } func TestPointCreatesBuildWithoutReadingRefs(t *testing.T) { revision := "0123456789abcdef0123456789abcdef01234567" refs := &mutableRefs{values: map[string]string{}} now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC) object := pointDelivery(revision) instance := testController(t, object, refs, &now) instance.reconcileAll(context.Background()) if refs.calls != 0 { t.Fatalf("Forgejo ref reads = %d", refs.calls) } if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } } type mutableRefs struct { values map[string]string identities map[string]string calls int pathCalls int } func (source *mutableRefs) PathIdentity(_ context.Context, _ string, revision string, _ []string) (string, error) { source.pathCalls++ if value, found := source.identities[revision]; found { return value, nil } return revision, nil } func (source *mutableRefs) Refs(context.Context, string) (map[string]string, error) { source.calls++ result := make(map[string]string, len(source.values)) for name, revision := range source.values { result[name] = revision } return result, nil } func TestPointDeliveryConsumesRevisionsWithoutReadingRefs(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" refs := &mutableRefs{values: map[string]string{}} now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC) object := pointDelivery(first) instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) if refs.calls != 0 { t.Fatalf("Forgejo ref reads = %d", refs.calls) } firstBuild := workflowName("build", string(object.GetUID()), "", first) build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, firstBuild, metav1.GetOptions{}) if err != nil { t.Fatal(err) } if value := workflowParameter(t, build, "source-ref"); value != "" { t.Fatalf("source-ref = %q", value) } if value := workflowParameter(t, build, "source-revision"); value != first { t.Fatalf("source-revision = %q", value) } delivery := getDelivery(t, instance, "point") if err := unstructured.SetNestedStringSlice(delivery.Object, []string{second}, "spec", "source", "point", "revisions"); err != nil { t.Fatal(err) } if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(ctx, delivery, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } setNamedWorkflowStatus(t, instance, firstBuild, "Succeeded", map[string]string{ "delivery-result": `{"digest":"sha256:first"}`, }) instance.reconcileAll(ctx) instance.reconcileAll(ctx) secondBuild := workflowName("build", string(object.GetUID()), "", second) setNamedWorkflowStatus(t, instance, secondBuild, "Failed", nil) instance.reconcileAll(ctx) if refs.calls != 0 { t.Fatalf("Forgejo ref reads = %d", refs.calls) } if countWorkflows(t, instance, "build") != 2 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } delivery = getDelivery(t, instance, "point") points, found, err := unstructured.NestedSlice(delivery.Object, "status", "points") if err != nil || !found { t.Fatalf("read point status: found=%t error=%v", found, err) } if len(points) != 2 { t.Fatalf("point status count = %d", len(points)) } firstStatus := points[0].(map[string]any) secondStatus := points[1].(map[string]any) if firstStatus["revision"] != first || firstStatus["phase"] != "Succeeded" { t.Fatalf("first point status = %#v", firstStatus) } if secondStatus["revision"] != second || secondStatus["phase"] != "Failed" { t.Fatalf("second point status = %#v", secondStatus) } } func TestPointDeliveryCreatesOneBuildWithTwoReconcilers(t *testing.T) { revision := "0123456789abcdef0123456789abcdef01234567" refs := &mutableRefs{values: map[string]string{}} now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC) object := pointDelivery(revision) instance := testController(t, object, refs, &now) peer := *instance firstObject := getDelivery(t, instance, "point") secondObject := firstObject.DeepCopy() errors := make(chan error, 2) var reconcilers sync.WaitGroup reconcilers.Add(2) for index, reconciler := range []*controller{instance, &peer} { object := []*unstructured.Unstructured{firstObject, secondObject}[index] go func() { defer reconcilers.Done() errors <- reconciler.reconcile(context.Background(), object) }() } reconcilers.Wait() close(errors) for err := range errors { if err != nil { t.Fatal(err) } } if refs.calls != 0 { t.Fatalf("Forgejo ref reads = %d", refs.calls) } if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } delivery := getDelivery(t, instance, "point") points, found, err := unstructured.NestedSlice(delivery.Object, "status", "points") if err != nil || !found || len(points) != 1 { t.Fatalf("point status: count=%d found=%t error=%v", len(points), found, err) } } func TestContinuousDeliveryDebouncesAndDoesNotRetrigger(t *testing.T) { revision := "0123456789abcdef0123456789abcdef01234567" refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}} now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC) instance := testController(t, continuousDelivery(), refs, &now) ctx := context.Background() instance.reconcileAll(ctx) instance.reconcileAll(ctx) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } setWorkflowStatus(t, instance, "build", "Succeeded", map[string]string{"delivery-result": `{"digest":"sha256:abc"}`}) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } delivery := getDelivery(t, instance, "continuous") phase, _, _ := unstructured.NestedString(delivery.Object, "status", "continuous", "attempt", "phase") if phase != "Succeeded" { t.Fatalf("attempt phase = %q", phase) } } func TestContinuousDeliveryUsesConfiguredDebounce(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" refs := &mutableRefs{values: map[string]string{"refs/heads/main": first}} now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC) object := continuousDelivery() if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil { t.Fatal(err) } instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) instance.reconcileAll(ctx) instance.reconcileAll(ctx) now = now.Add(119 * time.Second) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 0 { t.Fatal("build Workflow created before configured debounce elapsed") } refs.values["refs/heads/main"] = second instance.reconcileAll(ctx) now = now.Add(time.Second) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 0 { t.Fatal("build Workflow created without restarting configured debounce") } now = now.Add(119 * time.Second) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } } func TestPathFilteredDebounceIgnoresUnrelatedCommits(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" refs := &mutableRefs{ values: map[string]string{"refs/heads/main": first}, identities: map[string]string{first: "sha256:selected", second: "sha256:selected"}, } now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC) object := continuousDelivery() if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil { t.Fatal(err) } if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil { t.Fatal(err) } instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) now = now.Add(119 * time.Second) refs.values["refs/heads/main"] = second instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 0 { t.Fatal("build created before filtered debounce elapsed") } now = now.Add(time.Second) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get( ctx, workflowName("build", string(object.GetUID()), "continuous", refs.identities[second]), metav1.GetOptions{}, ) if err != nil { t.Fatal(err) } if workflowParameter(t, build, "source-revision") != second { t.Fatal("build did not advance to the latest unrelated commit") } } func TestPathFilteredDebounceRestartsForSelectedChanges(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" refs := &mutableRefs{ values: map[string]string{"refs/heads/main": first}, identities: map[string]string{first: "sha256:first", second: "sha256:second"}, } now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC) object := continuousDelivery() if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil { t.Fatal(err) } if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil { t.Fatal(err) } instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) now = now.Add(119 * time.Second) refs.values["refs/heads/main"] = second instance.reconcileAll(ctx) now = now.Add(119 * time.Second) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 0 { t.Fatal("build created before restarted filtered debounce elapsed") } now = now.Add(time.Second) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } } func TestPathFilteredDeliveryUsesSuccessfulBaseline(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" third := "fedcba9876543210fedcba9876543210fedcba98" refs := &mutableRefs{ values: map[string]string{"refs/heads/main": first}, identities: map[string]string{ first: "sha256:first", second: "sha256:first", third: "sha256:third", }, } now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC) object := continuousDelivery() if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil { t.Fatal(err) } if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil { t.Fatal(err) } instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) setWorkflowStatus(t, instance, "build", "Succeeded", nil) instance.reconcileAll(ctx) refs.values["refs/heads/main"] = second instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatal("unrelated commit created another build") } delivery := getDelivery(t, instance, "continuous") delivered, found, err := unstructured.NestedMap(delivery.Object, "status", "continuous", "delivered") if err != nil || !found || delivered["revision"] != first || delivered["identity"] != "sha256:first" { t.Fatalf("delivered baseline = %#v, found=%t, error=%v", delivered, found, err) } refs.values["refs/heads/main"] = third instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 2 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } } func TestPathFilteredFailureDoesNotAdvanceBaseline(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" third := "fedcba9876543210fedcba9876543210fedcba98" fourth := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" refs := &mutableRefs{ values: map[string]string{"refs/heads/main": first}, identities: map[string]string{ first: "sha256:first", second: "sha256:second", third: "sha256:second", fourth: "sha256:fourth", }, } now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC) object := continuousDelivery() if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil { t.Fatal(err) } if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil { t.Fatal(err) } instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) setWorkflowStatus(t, instance, "build", "Succeeded", nil) instance.reconcileAll(ctx) refs.values["refs/heads/main"] = second instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) setNamedWorkflowStatus( t, instance, workflowName("build", string(object.GetUID()), "continuous", refs.identities[second]), "Failed", nil, ) instance.reconcileAll(ctx) refs.values["refs/heads/main"] = third instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 2 { t.Fatal("same failed filtered identity was retried") } delivery := getDelivery(t, instance, "continuous") delivered, _, _ := unstructured.NestedMap(delivery.Object, "status", "continuous", "delivered") if delivered["identity"] != "sha256:first" { t.Fatalf("delivered baseline = %#v", delivered) } refs.values["refs/heads/main"] = fourth instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 3 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } } func TestPathFilteredRevertCancelsCandidate(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" third := "fedcba9876543210fedcba9876543210fedcba98" refs := &mutableRefs{ values: map[string]string{"refs/heads/main": first}, identities: map[string]string{ first: "sha256:first", second: "sha256:second", third: "sha256:first", }, } now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC) object := continuousDelivery() if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil { t.Fatal(err) } if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil { t.Fatal(err) } instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) setWorkflowStatus(t, instance, "build", "Succeeded", nil) instance.reconcileAll(ctx) refs.values["refs/heads/main"] = second instance.reconcileAll(ctx) refs.values["refs/heads/main"] = third now = now.Add(time.Minute) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatal("reverted filtered candidate created a build") } delivery := getDelivery(t, instance, "continuous") candidate, _, _ := unstructured.NestedString(delivery.Object, "status", "continuous", "candidateIdentity") if candidate != "" { t.Fatalf("candidate identity = %q", candidate) } } func TestContinuousDeliveryCoalescesUpdatesWhileBuildIsActive(t *testing.T) { first := "0123456789abcdef0123456789abcdef01234567" second := "89abcdef0123456789abcdef0123456789abcdef" third := "fedcba9876543210fedcba9876543210fedcba98" refs := &mutableRefs{values: map[string]string{"refs/heads/main": first}} now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC) object := continuousDelivery() instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) instance.reconcileAll(ctx) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) refs.values["refs/heads/main"] = second instance.reconcileAll(ctx) refs.values["refs/heads/main"] = third instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build count while first build active = %d", countWorkflows(t, instance, "build")) } setWorkflowStatus(t, instance, "build", "Succeeded", nil) instance.reconcileAll(ctx) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 2 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get( ctx, workflowName("build", string(object.GetUID()), "continuous", second), metav1.GetOptions{}, ); !apierrors.IsNotFound(err) { t.Fatal("intermediate revision received a build workflow") } } func TestDeletionCreatesNoWorkflow(t *testing.T) { refs := &mutableRefs{values: map[string]string{}} now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC) object := continuousDelivery() deletion := metav1.NewTime(now) object.SetDeletionTimestamp(&deletion) instance := testController(t, object, refs, &now) instance.reconcileAll(context.Background()) if countWorkflows(t, instance, "build") != 0 { t.Fatal("deletion created a build Workflow") } } func TestContinuousRetryAfterWorkflowGC(t *testing.T) { revision := "0123456789abcdef0123456789abcdef01234567" refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}} now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC) object := continuousDelivery() instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) now = now.Add(time.Minute) instance.reconcileAll(ctx) first := workflowName("build", string(object.GetUID()), "continuous", revision) setNamedWorkflowStatus(t, instance, first, "Failed", nil) instance.reconcileAll(ctx) if err := instance.client.Resource(workflowGVR).Namespace("delivery").Delete(ctx, first, metav1.DeleteOptions{}); err != nil { t.Fatal(err) } setRetry(t, instance, "continuous", first) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } delivery := getDelivery(t, instance, "continuous") handled, _, _ := unstructured.NestedString(delivery.Object, "status", "lastHandledRetry") if handled != first { t.Fatalf("last handled retry = %q", handled) } successor := workflowName("build", string(object.GetUID()), "continuous", revision, first) workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, successor, metav1.GetOptions{}) if err != nil { t.Fatal(err) } if workflowHasParameter(t, workflow, "input-plan") { t.Fatal("retry supplied input-plan") } instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatal("acknowledged retry created another successor") } } func TestPointRetryReplaysSucceededAttempt(t *testing.T) { revision := "0123456789abcdef0123456789abcdef01234567" refs := &mutableRefs{values: map[string]string{}} now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC) object := pointDelivery(revision) instance := testController(t, object, refs, &now) ctx := context.Background() instance.reconcileAll(ctx) first := workflowName("build", string(object.GetUID()), "", revision) setNamedWorkflowStatus(t, instance, first, "Running", nil) instance.reconcileAll(ctx) setRetry(t, instance, "point", first) instance.reconcileAll(ctx) if countWorkflows(t, instance, "build") != 1 { t.Fatal("active attempt received a retry") } delivery := getDelivery(t, instance, "point") handled, _, _ := unstructured.NestedString(delivery.Object, "status", "lastHandledRetry") if handled != "" { t.Fatalf("active retry was acknowledged as %q", handled) } setNamedWorkflowStatus(t, instance, first, "Succeeded", nil) instance.reconcileAll(ctx) instance.reconcileAll(ctx) delivery = getDelivery(t, instance, "point") points, _, _ := unstructured.NestedSlice(delivery.Object, "status", "points") if len(points) != 2 { t.Fatalf("point status count = %d", len(points)) } successor := points[1].(map[string]any) if successor["revision"] != revision { t.Fatalf("retry status = %#v", successor) } if countWorkflows(t, instance, "build") != 2 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } } func TestRetryCreatesOneSuccessorWithTwoReconcilers(t *testing.T) { revision := "0123456789abcdef0123456789abcdef01234567" refs := &mutableRefs{values: map[string]string{}} now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC) object := pointDelivery(revision) instance := testController(t, object, refs, &now) peer := *instance ctx := context.Background() instance.reconcileAll(ctx) first := workflowName("build", string(object.GetUID()), "", revision) setNamedWorkflowStatus(t, instance, first, "Failed", nil) instance.reconcileAll(ctx) setRetry(t, instance, "point", first) errors := make(chan error, 2) var reconcilers sync.WaitGroup for _, reconciler := range []*controller{instance, &peer} { reconcilers.Add(1) go func() { defer reconcilers.Done() errors <- reconciler.reconcileKey(ctx, "delivery", "point") }() } reconcilers.Wait() close(errors) for err := range errors { if err != nil { t.Fatal(err) } } if countWorkflows(t, instance, "build") != 2 { t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build")) } } func TestReconcileRetriesResourceConflict(t *testing.T) { refs := &mutableRefs{values: map[string]string{}} now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC) instance := testController(t, continuousDelivery(), refs, &now) conflicts := 0 instance.client.(*dynamicfake.FakeDynamicClient).PrependReactor( "update", "projectdeliveries", func(k8stesting.Action) (bool, runtime.Object, error) { if conflicts > 0 { return false, nil, nil } conflicts++ return true, nil, apierrors.NewConflict( schema.GroupResource{Group: projectDeliveryGVR.Group, Resource: projectDeliveryGVR.Resource}, "continuous", errors.New("concurrent update"), ) }, ) instance.reconcileAll(context.Background()) if conflicts != 1 { t.Fatalf("conflicts = %d", conflicts) } if countWorkflows(t, instance, "build") != 0 { t.Fatal("candidate created a build too early") } } func testController(t *testing.T, delivery *unstructured.Unstructured, refs sourceReader, now *time.Time) *controller { t.Helper() objects := []runtime.Object{delivery} for _, namespace := range []string{"delivery", "other"} { for _, name := range []string{"nuthatch-delivery", "shared-delivery", "other-delivery", "qual-tests", "other-template", "other-qual"} { objects = append(objects, testTemplate(namespace, name)) } } client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds( runtime.NewScheme(), map[schema.GroupVersionResource]string{ projectDeliveryGVR: "ProjectDeliveryList", workflowGVR: "WorkflowList", templateGVR: "WorkflowTemplateList", configMapGVR: "ConfigMapList", }, objects..., ) return &controller{ client: client, source: refs, now: func() time.Time { return *now }, config: controllerConfig{ pollInterval: time.Second, bucketInterval: time.Minute, requestTimeout: time.Second, }, } } func testTemplate(namespace, name string) *unstructured.Unstructured { return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "argoproj.io/v1alpha1", "kind": "WorkflowTemplate", "metadata": map[string]any{ "name": name, "namespace": namespace, "generation": int64(1), "labels": map[string]any{"app.kubernetes.io/name": "project-delivery", "app.kubernetes.io/instance": "synthetic"}, }, "spec": map[string]any{"entrypoint": "build", "templates": []any{map[string]any{ "name": "build", "container": map[string]any{"image": "example.invalid/builder:v1"}, }}}, }} } func continuousDelivery() *unstructured.Unstructured { return deliveryObject("continuous", map[string]any{"continuous": map[string]any{"ref": "refs/heads/main"}}) } func pointDelivery(revisions ...string) *unstructured.Unstructured { values := make([]any, len(revisions)) for index, revision := range revisions { values[index] = revision } return deliveryObject("point", map[string]any{"point": map[string]any{"revisions": values}}) } func deliveryObject(name string, sourceMode map[string]any) *unstructured.Unstructured { source := map[string]any{"repository": "https://git.quorate.dev/quorate/nuthatch"} for key, value := range sourceMode { source[key] = value } return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "delivery.quorate.dev/v1alpha1", "kind": "ProjectDelivery", "metadata": map[string]any{ "name": name, "namespace": "delivery", "uid": "11111111-2222-3333-4444-555555555555", "generation": int64(1), }, "spec": map[string]any{ "source": source, "targets": []any{ map[string]any{"kind": "QuayOCI", "repository": "quorate/nuthatch"}, }, "workflowTemplateRef": map[string]any{"name": "nuthatch-delivery"}, }, }} } func setWorkflowStatus(t *testing.T, instance *controller, purpose, phase string, outputs map[string]string) { t.Helper() items, err := instance.client.Resource(workflowGVR).Namespace("delivery").List(context.Background(), metav1.ListOptions{}) if err != nil { t.Fatal(err) } for index := range items.Items { item := &items.Items[index] if item.GetLabels()[purposeAnnotation] != purpose { continue } status := map[string]any{"phase": phase} if len(outputs) != 0 { parameters := make([]any, 0, len(outputs)) for name, value := range outputs { parameters = append(parameters, map[string]any{"name": name, "value": value}) } status["nodes"] = map[string]any{ item.GetName(): map[string]any{"outputs": map[string]any{"parameters": parameters}}, } } item.Object["status"] = status if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").UpdateStatus(context.Background(), item, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } return } t.Fatalf("workflow purpose %q not found", purpose) } func setNamedWorkflowStatus(t *testing.T, instance *controller, name, phase string, outputs map[string]string) { t.Helper() item, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(context.Background(), name, metav1.GetOptions{}) if err != nil { t.Fatal(err) } status := map[string]any{"phase": phase} if len(outputs) != 0 { parameters := make([]any, 0, len(outputs)) for outputName, value := range outputs { parameters = append(parameters, map[string]any{"name": outputName, "value": value}) } status["nodes"] = map[string]any{ item.GetName(): map[string]any{"outputs": map[string]any{"parameters": parameters}}, } } item.Object["status"] = status if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").UpdateStatus(context.Background(), item, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } } func countWorkflows(t *testing.T, instance *controller, purpose string) int { t.Helper() items, err := instance.client.Resource(workflowGVR).Namespace("delivery").List(context.Background(), metav1.ListOptions{}) if err != nil { t.Fatal(err) } count := 0 for _, item := range items.Items { if item.GetLabels()[purposeAnnotation] == purpose { count++ } } return count } func getDelivery(t *testing.T, instance *controller, name string) *unstructured.Unstructured { t.Helper() object, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Get(context.Background(), name, metav1.GetOptions{}) if err != nil { t.Fatal(err) } return object } func setRetry(t *testing.T, instance *controller, name, workflow string) { t.Helper() object := getDelivery(t, instance, name) annotations := object.GetAnnotations() if annotations == nil { annotations = map[string]string{} } annotations[retryAnnotation] = workflow object.SetAnnotations(annotations) if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } } func workflowHasParameter(t *testing.T, workflow *unstructured.Unstructured, name string) bool { t.Helper() parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") if err != nil || !found { t.Fatalf("read Workflow parameters: found=%t error=%v", found, err) } for _, item := range parameters { parameter := item.(map[string]any) if parameter["name"] == name { return true } } return false } func workflowParameter(t *testing.T, workflow *unstructured.Unstructured, name string) string { t.Helper() parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters") if err != nil || !found { t.Fatalf("read Workflow parameters: found=%t error=%v", found, err) } for _, item := range parameters { parameter := item.(map[string]any) if parameter["name"] == name { value, _ := parameter["value"].(string) return value } } t.Fatalf("Workflow parameter %q not found", name) return "" } func workflowMutex(t *testing.T, workflow *unstructured.Unstructured) string { t.Helper() mutexes, found, err := unstructured.NestedSlice(workflow.Object, "spec", "synchronization", "mutexes") if err != nil || !found || len(mutexes) != 1 { t.Fatalf("read Workflow mutex: count=%d found=%t error=%v", len(mutexes), found, err) } mutex, ok := mutexes[0].(map[string]any) if !ok { t.Fatalf("Workflow mutex = %#v", mutexes[0]) } name, _ := mutex["name"].(string) return name }