ce7cb702218693d08ea6d83464d2457c9250027c

services/project-delivery/controller/controller_test.go

@@ -0,0 +1,992 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	dynamicfake "k8s.io/client-go/dynamic/fake"
+	k8stesting "k8s.io/client-go/testing"
+)
+
+func TestContinuousCreatesBuildWithOpaqueTargets(t *testing.T) {
+	revision := "0123456789abcdef0123456789abcdef01234567"
+	refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}}
+	now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	object.Object["spec"].(map[string]any)["source"].(map[string]any)["repository"] = "https://forge.quorate.internal/quorate/cluster"
+	targets := []any{map[string]any{
+		"kind":       "Git",
+		"repository": "https://git.quorate.dev/ms2k/quorate",
+		"ref":        "refs/heads/master",
+		"options":    map[string]any{"force": true},
+	}}
+	object.Object["spec"].(map[string]any)["targets"] = targets
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(
+		ctx,
+		workflowName("build", string(object.GetUID()), "continuous", revision),
+		metav1.GetOptions{},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if workflowHasParameter(t, build, "input-plan") {
+		t.Fatal("controller supplied input-plan")
+	}
+	if mutex := workflowMutex(t, build); mutex != "nuthatch-delivery" {
+		t.Fatalf("Workflow mutex = %q", mutex)
+	}
+	var passedTargets []any
+	if err := json.Unmarshal([]byte(workflowParameter(t, build, "targets")), &passedTargets); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(passedTargets, targets) {
+		t.Fatalf("targets = %#v", passedTargets)
+	}
+}
+
+func TestDuplicateTargetsAreRejected(t *testing.T) {
+	object := continuousDelivery()
+	target := map[string]any{"kind": "QuayOCI", "repository": "quorate/nuthatch"}
+	object.Object["spec"].(map[string]any)["targets"] = []any{target, target}
+	instance := testController(t, object, &mutableRefs{}, new(time.Time))
+
+	err := instance.reconcile(context.Background(), object)
+	if err == nil || err.Error() != "duplicate target at index 1" {
+		t.Fatalf("reconcile error = %v", err)
+	}
+}
+
+func TestWorkflowMutexIsScopedByTemplateReference(t *testing.T) {
+	now := time.Date(2026, time.August, 25, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	instance := testController(t, object, &mutableRefs{}, &now)
+	delivery, err := decodeProjectDelivery(object)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := context.Background()
+	for _, item := range []struct {
+		name     string
+		template string
+	}{
+		{name: "first", template: "shared-delivery"},
+		{name: "second", template: "shared-delivery"},
+		{name: "third", template: "other-delivery"},
+	} {
+		if err := instance.ensureWorkflow(ctx, &delivery, item.name, item.template, "build", map[string]string{}, nil); err != nil {
+			t.Fatal(err)
+		}
+		workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, item.name, metav1.GetOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if mutex := workflowMutex(t, workflow); mutex != item.template {
+			t.Fatalf("Workflow %q mutex = %q", item.name, mutex)
+		}
+	}
+}
+
+func TestPointCreatesBuildWithoutReadingRefs(t *testing.T) {
+	revision := "0123456789abcdef0123456789abcdef01234567"
+	refs := &mutableRefs{values: map[string]string{}}
+	now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
+	object := pointDelivery(revision)
+	instance := testController(t, object, refs, &now)
+
+	instance.reconcileAll(context.Background())
+
+	if refs.calls != 0 {
+		t.Fatalf("Forgejo ref reads = %d", refs.calls)
+	}
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+}
+
+type mutableRefs struct {
+	values     map[string]string
+	identities map[string]string
+	calls      int
+	pathCalls  int
+}
+
+func (source *mutableRefs) PathIdentity(_ context.Context, _ string, revision string, _ []string) (string, error) {
+	source.pathCalls++
+	if value, found := source.identities[revision]; found {
+		return value, nil
+	}
+	return revision, nil
+}
+
+func (source *mutableRefs) Refs(context.Context, string) (map[string]string, error) {
+	source.calls++
+	result := make(map[string]string, len(source.values))
+	for name, revision := range source.values {
+		result[name] = revision
+	}
+	return result, nil
+}
+
+func TestPointDeliveryConsumesRevisionsWithoutReadingRefs(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	refs := &mutableRefs{values: map[string]string{}}
+	now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
+	object := pointDelivery(first)
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	if refs.calls != 0 {
+		t.Fatalf("Forgejo ref reads = %d", refs.calls)
+	}
+	firstBuild := workflowName("build", string(object.GetUID()), "", first)
+	build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, firstBuild, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if value := workflowParameter(t, build, "source-ref"); value != "" {
+		t.Fatalf("source-ref = %q", value)
+	}
+	if value := workflowParameter(t, build, "source-revision"); value != first {
+		t.Fatalf("source-revision = %q", value)
+	}
+
+	delivery := getDelivery(t, instance, "point")
+	if err := unstructured.SetNestedStringSlice(delivery.Object, []string{second}, "spec", "source", "point", "revisions"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(ctx, delivery, metav1.UpdateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	setNamedWorkflowStatus(t, instance, firstBuild, "Succeeded", map[string]string{
+		"delivery-result": `{"digest":"sha256:first"}`,
+	})
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+
+	secondBuild := workflowName("build", string(object.GetUID()), "", second)
+	setNamedWorkflowStatus(t, instance, secondBuild, "Failed", nil)
+	instance.reconcileAll(ctx)
+
+	if refs.calls != 0 {
+		t.Fatalf("Forgejo ref reads = %d", refs.calls)
+	}
+	if countWorkflows(t, instance, "build") != 2 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	delivery = getDelivery(t, instance, "point")
+	points, found, err := unstructured.NestedSlice(delivery.Object, "status", "points")
+	if err != nil || !found {
+		t.Fatalf("read point status: found=%t error=%v", found, err)
+	}
+	if len(points) != 2 {
+		t.Fatalf("point status count = %d", len(points))
+	}
+	firstStatus := points[0].(map[string]any)
+	secondStatus := points[1].(map[string]any)
+	if firstStatus["revision"] != first || firstStatus["phase"] != "Succeeded" {
+		t.Fatalf("first point status = %#v", firstStatus)
+	}
+	if secondStatus["revision"] != second || secondStatus["phase"] != "Failed" {
+		t.Fatalf("second point status = %#v", secondStatus)
+	}
+}
+
+func TestPointDeliveryCreatesOneBuildWithTwoReconcilers(t *testing.T) {
+	revision := "0123456789abcdef0123456789abcdef01234567"
+	refs := &mutableRefs{values: map[string]string{}}
+	now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
+	object := pointDelivery(revision)
+	instance := testController(t, object, refs, &now)
+	peer := *instance
+	firstObject := getDelivery(t, instance, "point")
+	secondObject := firstObject.DeepCopy()
+
+	errors := make(chan error, 2)
+	var reconcilers sync.WaitGroup
+	reconcilers.Add(2)
+	for index, reconciler := range []*controller{instance, &peer} {
+		object := []*unstructured.Unstructured{firstObject, secondObject}[index]
+		go func() {
+			defer reconcilers.Done()
+			errors <- reconciler.reconcile(context.Background(), object)
+		}()
+	}
+	reconcilers.Wait()
+	close(errors)
+	for err := range errors {
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if refs.calls != 0 {
+		t.Fatalf("Forgejo ref reads = %d", refs.calls)
+	}
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	delivery := getDelivery(t, instance, "point")
+	points, found, err := unstructured.NestedSlice(delivery.Object, "status", "points")
+	if err != nil || !found || len(points) != 1 {
+		t.Fatalf("point status: count=%d found=%t error=%v", len(points), found, err)
+	}
+}
+
+func TestContinuousDeliveryDebouncesAndDoesNotRetrigger(t *testing.T) {
+	revision := "0123456789abcdef0123456789abcdef01234567"
+	refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}}
+	now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
+	instance := testController(t, continuousDelivery(), refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	setWorkflowStatus(t, instance, "build", "Succeeded", map[string]string{"delivery-result": `{"digest":"sha256:abc"}`})
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	delivery := getDelivery(t, instance, "continuous")
+	phase, _, _ := unstructured.NestedString(delivery.Object, "status", "continuous", "attempt", "phase")
+	if phase != "Succeeded" {
+		t.Fatalf("attempt phase = %q", phase)
+	}
+}
+
+func TestContinuousDeliveryUsesConfiguredDebounce(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	refs := &mutableRefs{values: map[string]string{"refs/heads/main": first}}
+	now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil {
+		t.Fatal(err)
+	}
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+	now = now.Add(119 * time.Second)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 0 {
+		t.Fatal("build Workflow created before configured debounce elapsed")
+	}
+
+	refs.values["refs/heads/main"] = second
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Second)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 0 {
+		t.Fatal("build Workflow created without restarting configured debounce")
+	}
+
+	now = now.Add(119 * time.Second)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+}
+
+func TestPathFilteredDebounceIgnoresUnrelatedCommits(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	refs := &mutableRefs{
+		values:     map[string]string{"refs/heads/main": first},
+		identities: map[string]string{first: "sha256:selected", second: "sha256:selected"},
+	}
+	now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil {
+		t.Fatal(err)
+	}
+	if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
+		t.Fatal(err)
+	}
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	now = now.Add(119 * time.Second)
+	refs.values["refs/heads/main"] = second
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 0 {
+		t.Fatal("build created before filtered debounce elapsed")
+	}
+	now = now.Add(time.Second)
+	instance.reconcileAll(ctx)
+
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(
+		ctx,
+		workflowName("build", string(object.GetUID()), "continuous", refs.identities[second]),
+		metav1.GetOptions{},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if workflowParameter(t, build, "source-revision") != second {
+		t.Fatal("build did not advance to the latest unrelated commit")
+	}
+}
+
+func TestPathFilteredDebounceRestartsForSelectedChanges(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	refs := &mutableRefs{
+		values:     map[string]string{"refs/heads/main": first},
+		identities: map[string]string{first: "sha256:first", second: "sha256:second"},
+	}
+	now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil {
+		t.Fatal(err)
+	}
+	if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
+		t.Fatal(err)
+	}
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	now = now.Add(119 * time.Second)
+	refs.values["refs/heads/main"] = second
+	instance.reconcileAll(ctx)
+	now = now.Add(119 * time.Second)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 0 {
+		t.Fatal("build created before restarted filtered debounce elapsed")
+	}
+	now = now.Add(time.Second)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+}
+
+func TestPathFilteredDeliveryUsesSuccessfulBaseline(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	third := "fedcba9876543210fedcba9876543210fedcba98"
+	refs := &mutableRefs{
+		values: map[string]string{"refs/heads/main": first},
+		identities: map[string]string{
+			first:  "sha256:first",
+			second: "sha256:first",
+			third:  "sha256:third",
+		},
+	}
+	now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil {
+		t.Fatal(err)
+	}
+	if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
+		t.Fatal(err)
+	}
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	setWorkflowStatus(t, instance, "build", "Succeeded", nil)
+	instance.reconcileAll(ctx)
+
+	refs.values["refs/heads/main"] = second
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatal("unrelated commit created another build")
+	}
+	delivery := getDelivery(t, instance, "continuous")
+	delivered, found, err := unstructured.NestedMap(delivery.Object, "status", "continuous", "delivered")
+	if err != nil || !found || delivered["revision"] != first || delivered["identity"] != "sha256:first" {
+		t.Fatalf("delivered baseline = %#v, found=%t, error=%v", delivered, found, err)
+	}
+
+	refs.values["refs/heads/main"] = third
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 2 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+}
+
+func TestPathFilteredFailureDoesNotAdvanceBaseline(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	third := "fedcba9876543210fedcba9876543210fedcba98"
+	fourth := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	refs := &mutableRefs{
+		values: map[string]string{"refs/heads/main": first},
+		identities: map[string]string{
+			first:  "sha256:first",
+			second: "sha256:second",
+			third:  "sha256:second",
+			fourth: "sha256:fourth",
+		},
+	}
+	now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil {
+		t.Fatal(err)
+	}
+	if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
+		t.Fatal(err)
+	}
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	setWorkflowStatus(t, instance, "build", "Succeeded", nil)
+	instance.reconcileAll(ctx)
+
+	refs.values["refs/heads/main"] = second
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	setNamedWorkflowStatus(
+		t,
+		instance,
+		workflowName("build", string(object.GetUID()), "continuous", refs.identities[second]),
+		"Failed",
+		nil,
+	)
+	instance.reconcileAll(ctx)
+
+	refs.values["refs/heads/main"] = third
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 2 {
+		t.Fatal("same failed filtered identity was retried")
+	}
+	delivery := getDelivery(t, instance, "continuous")
+	delivered, _, _ := unstructured.NestedMap(delivery.Object, "status", "continuous", "delivered")
+	if delivered["identity"] != "sha256:first" {
+		t.Fatalf("delivered baseline = %#v", delivered)
+	}
+
+	refs.values["refs/heads/main"] = fourth
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 3 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+}
+
+func TestPathFilteredRevertCancelsCandidate(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	third := "fedcba9876543210fedcba9876543210fedcba98"
+	refs := &mutableRefs{
+		values: map[string]string{"refs/heads/main": first},
+		identities: map[string]string{
+			first:  "sha256:first",
+			second: "sha256:second",
+			third:  "sha256:first",
+		},
+	}
+	now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil {
+		t.Fatal(err)
+	}
+	if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
+		t.Fatal(err)
+	}
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	setWorkflowStatus(t, instance, "build", "Succeeded", nil)
+	instance.reconcileAll(ctx)
+
+	refs.values["refs/heads/main"] = second
+	instance.reconcileAll(ctx)
+	refs.values["refs/heads/main"] = third
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatal("reverted filtered candidate created a build")
+	}
+	delivery := getDelivery(t, instance, "continuous")
+	candidate, _, _ := unstructured.NestedString(delivery.Object, "status", "continuous", "candidateIdentity")
+	if candidate != "" {
+		t.Fatalf("candidate identity = %q", candidate)
+	}
+}
+
+func TestContinuousDeliveryCoalescesUpdatesWhileBuildIsActive(t *testing.T) {
+	first := "0123456789abcdef0123456789abcdef01234567"
+	second := "89abcdef0123456789abcdef0123456789abcdef"
+	third := "fedcba9876543210fedcba9876543210fedcba98"
+	refs := &mutableRefs{values: map[string]string{"refs/heads/main": first}}
+	now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+
+	refs.values["refs/heads/main"] = second
+	instance.reconcileAll(ctx)
+	refs.values["refs/heads/main"] = third
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build count while first build active = %d", countWorkflows(t, instance, "build"))
+	}
+
+	setWorkflowStatus(t, instance, "build", "Succeeded", nil)
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+
+	if countWorkflows(t, instance, "build") != 2 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(
+		ctx,
+		workflowName("build", string(object.GetUID()), "continuous", second),
+		metav1.GetOptions{},
+	); !apierrors.IsNotFound(err) {
+		t.Fatal("intermediate revision received a build workflow")
+	}
+}
+
+func TestDeletionCreatesNoWorkflow(t *testing.T) {
+	refs := &mutableRefs{values: map[string]string{}}
+	now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	deletion := metav1.NewTime(now)
+	object.SetDeletionTimestamp(&deletion)
+	instance := testController(t, object, refs, &now)
+
+	instance.reconcileAll(context.Background())
+	if countWorkflows(t, instance, "build") != 0 {
+		t.Fatal("deletion created a build Workflow")
+	}
+}
+
+func TestContinuousRetryAfterWorkflowGC(t *testing.T) {
+	revision := "0123456789abcdef0123456789abcdef01234567"
+	refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}}
+	now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC)
+	object := continuousDelivery()
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	now = now.Add(time.Minute)
+	instance.reconcileAll(ctx)
+	first := workflowName("build", string(object.GetUID()), "continuous", revision)
+	setNamedWorkflowStatus(t, instance, first, "Failed", nil)
+	instance.reconcileAll(ctx)
+	if err := instance.client.Resource(workflowGVR).Namespace("delivery").Delete(ctx, first, metav1.DeleteOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	setRetry(t, instance, "continuous", first)
+	instance.reconcileAll(ctx)
+
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+	delivery := getDelivery(t, instance, "continuous")
+	handled, _, _ := unstructured.NestedString(delivery.Object, "status", "lastHandledRetry")
+	if handled != first {
+		t.Fatalf("last handled retry = %q", handled)
+	}
+	successor := workflowName("build", string(object.GetUID()), "continuous", revision, first)
+	workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, successor, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if workflowHasParameter(t, workflow, "input-plan") {
+		t.Fatal("retry supplied input-plan")
+	}
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatal("acknowledged retry created another successor")
+	}
+}
+
+func TestPointRetryReplaysSucceededAttempt(t *testing.T) {
+	revision := "0123456789abcdef0123456789abcdef01234567"
+	refs := &mutableRefs{values: map[string]string{}}
+	now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC)
+	object := pointDelivery(revision)
+	instance := testController(t, object, refs, &now)
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	first := workflowName("build", string(object.GetUID()), "", revision)
+	setNamedWorkflowStatus(t, instance, first, "Running", nil)
+	instance.reconcileAll(ctx)
+	setRetry(t, instance, "point", first)
+	instance.reconcileAll(ctx)
+	if countWorkflows(t, instance, "build") != 1 {
+		t.Fatal("active attempt received a retry")
+	}
+	delivery := getDelivery(t, instance, "point")
+	handled, _, _ := unstructured.NestedString(delivery.Object, "status", "lastHandledRetry")
+	if handled != "" {
+		t.Fatalf("active retry was acknowledged as %q", handled)
+	}
+	setNamedWorkflowStatus(t, instance, first, "Succeeded", nil)
+	instance.reconcileAll(ctx)
+	instance.reconcileAll(ctx)
+
+	delivery = getDelivery(t, instance, "point")
+	points, _, _ := unstructured.NestedSlice(delivery.Object, "status", "points")
+	if len(points) != 2 {
+		t.Fatalf("point status count = %d", len(points))
+	}
+	successor := points[1].(map[string]any)
+	if successor["revision"] != revision {
+		t.Fatalf("retry status = %#v", successor)
+	}
+	if countWorkflows(t, instance, "build") != 2 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+}
+
+func TestRetryCreatesOneSuccessorWithTwoReconcilers(t *testing.T) {
+	revision := "0123456789abcdef0123456789abcdef01234567"
+	refs := &mutableRefs{values: map[string]string{}}
+	now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC)
+	object := pointDelivery(revision)
+	instance := testController(t, object, refs, &now)
+	peer := *instance
+	ctx := context.Background()
+
+	instance.reconcileAll(ctx)
+	first := workflowName("build", string(object.GetUID()), "", revision)
+	setNamedWorkflowStatus(t, instance, first, "Failed", nil)
+	instance.reconcileAll(ctx)
+	setRetry(t, instance, "point", first)
+
+	errors := make(chan error, 2)
+	var reconcilers sync.WaitGroup
+	for _, reconciler := range []*controller{instance, &peer} {
+		reconcilers.Add(1)
+		go func() {
+			defer reconcilers.Done()
+			errors <- reconciler.reconcileKey(ctx, "delivery", "point")
+		}()
+	}
+	reconcilers.Wait()
+	close(errors)
+	for err := range errors {
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	if countWorkflows(t, instance, "build") != 2 {
+		t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
+	}
+}
+
+func TestReconcileRetriesResourceConflict(t *testing.T) {
+	refs := &mutableRefs{values: map[string]string{}}
+	now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
+	instance := testController(t, continuousDelivery(), refs, &now)
+	conflicts := 0
+	instance.client.(*dynamicfake.FakeDynamicClient).PrependReactor(
+		"update",
+		"projectdeliveries",
+		func(k8stesting.Action) (bool, runtime.Object, error) {
+			if conflicts > 0 {
+				return false, nil, nil
+			}
+			conflicts++
+			return true, nil, apierrors.NewConflict(
+				schema.GroupResource{Group: projectDeliveryGVR.Group, Resource: projectDeliveryGVR.Resource},
+				"continuous",
+				errors.New("concurrent update"),
+			)
+		},
+	)
+
+	instance.reconcileAll(context.Background())
+
+	if conflicts != 1 {
+		t.Fatalf("conflicts = %d", conflicts)
+	}
+	if countWorkflows(t, instance, "build") != 0 {
+		t.Fatal("candidate created a build too early")
+	}
+}
+
+func testController(t *testing.T, delivery *unstructured.Unstructured, refs sourceReader, now *time.Time) *controller {
+	t.Helper()
+	objects := []runtime.Object{delivery}
+	for _, namespace := range []string{"delivery", "other"} {
+		for _, name := range []string{"nuthatch-delivery", "shared-delivery", "other-delivery", "qual-tests", "other-template", "other-qual"} {
+			objects = append(objects, testTemplate(namespace, name))
+		}
+	}
+	client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(
+		runtime.NewScheme(),
+		map[schema.GroupVersionResource]string{
+			projectDeliveryGVR: "ProjectDeliveryList",
+			workflowGVR:        "WorkflowList",
+			templateGVR:        "WorkflowTemplateList",
+			configMapGVR:       "ConfigMapList",
+		},
+		objects...,
+	)
+	return &controller{
+		client: client,
+		source: refs,
+		now:    func() time.Time { return *now },
+		config: controllerConfig{
+			pollInterval:   time.Second,
+			bucketInterval: time.Minute,
+			requestTimeout: time.Second,
+		},
+	}
+}
+
+func testTemplate(namespace, name string) *unstructured.Unstructured {
+	return &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": "argoproj.io/v1alpha1", "kind": "WorkflowTemplate",
+		"metadata": map[string]any{
+			"name": name, "namespace": namespace, "generation": int64(1),
+			"labels": map[string]any{"app.kubernetes.io/name": "project-delivery", "app.kubernetes.io/instance": "synthetic"},
+		},
+		"spec": map[string]any{"entrypoint": "build", "templates": []any{map[string]any{
+			"name": "build", "container": map[string]any{"image": "example.invalid/builder:v1"},
+		}}},
+	}}
+}
+
+func continuousDelivery() *unstructured.Unstructured {
+	return deliveryObject("continuous", map[string]any{"continuous": map[string]any{"ref": "refs/heads/main"}})
+}
+
+func pointDelivery(revisions ...string) *unstructured.Unstructured {
+	values := make([]any, len(revisions))
+	for index, revision := range revisions {
+		values[index] = revision
+	}
+	return deliveryObject("point", map[string]any{"point": map[string]any{"revisions": values}})
+}
+
+func deliveryObject(name string, sourceMode map[string]any) *unstructured.Unstructured {
+	source := map[string]any{"repository": "https://git.quorate.dev/quorate/nuthatch"}
+	for key, value := range sourceMode {
+		source[key] = value
+	}
+	return &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": "delivery.quorate.dev/v1alpha1",
+		"kind":       "ProjectDelivery",
+		"metadata": map[string]any{
+			"name":       name,
+			"namespace":  "delivery",
+			"uid":        "11111111-2222-3333-4444-555555555555",
+			"generation": int64(1),
+		},
+		"spec": map[string]any{
+			"source": source,
+			"targets": []any{
+				map[string]any{"kind": "QuayOCI", "repository": "quorate/nuthatch"},
+			},
+			"workflowTemplateRef": map[string]any{"name": "nuthatch-delivery"},
+		},
+	}}
+}
+
+func setWorkflowStatus(t *testing.T, instance *controller, purpose, phase string, outputs map[string]string) {
+	t.Helper()
+	items, err := instance.client.Resource(workflowGVR).Namespace("delivery").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for index := range items.Items {
+		item := &items.Items[index]
+		if item.GetLabels()[purposeAnnotation] != purpose {
+			continue
+		}
+		status := map[string]any{"phase": phase}
+		if len(outputs) != 0 {
+			parameters := make([]any, 0, len(outputs))
+			for name, value := range outputs {
+				parameters = append(parameters, map[string]any{"name": name, "value": value})
+			}
+			status["nodes"] = map[string]any{
+				item.GetName(): map[string]any{"outputs": map[string]any{"parameters": parameters}},
+			}
+		}
+		item.Object["status"] = status
+		if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").UpdateStatus(context.Background(), item, metav1.UpdateOptions{}); err != nil {
+			t.Fatal(err)
+		}
+		return
+	}
+	t.Fatalf("workflow purpose %q not found", purpose)
+}
+
+func setNamedWorkflowStatus(t *testing.T, instance *controller, name, phase string, outputs map[string]string) {
+	t.Helper()
+	item, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(context.Background(), name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	status := map[string]any{"phase": phase}
+	if len(outputs) != 0 {
+		parameters := make([]any, 0, len(outputs))
+		for outputName, value := range outputs {
+			parameters = append(parameters, map[string]any{"name": outputName, "value": value})
+		}
+		status["nodes"] = map[string]any{
+			item.GetName(): map[string]any{"outputs": map[string]any{"parameters": parameters}},
+		}
+	}
+	item.Object["status"] = status
+	if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").UpdateStatus(context.Background(), item, metav1.UpdateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func countWorkflows(t *testing.T, instance *controller, purpose string) int {
+	t.Helper()
+	items, err := instance.client.Resource(workflowGVR).Namespace("delivery").List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	count := 0
+	for _, item := range items.Items {
+		if item.GetLabels()[purposeAnnotation] == purpose {
+			count++
+		}
+	}
+	return count
+}
+
+func getDelivery(t *testing.T, instance *controller, name string) *unstructured.Unstructured {
+	t.Helper()
+	object, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Get(context.Background(), name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return object
+}
+
+func setRetry(t *testing.T, instance *controller, name, workflow string) {
+	t.Helper()
+	object := getDelivery(t, instance, name)
+	annotations := object.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[retryAnnotation] = workflow
+	object.SetAnnotations(annotations)
+	if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func workflowHasParameter(t *testing.T, workflow *unstructured.Unstructured, name string) bool {
+	t.Helper()
+	parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters")
+	if err != nil || !found {
+		t.Fatalf("read Workflow parameters: found=%t error=%v", found, err)
+	}
+	for _, item := range parameters {
+		parameter := item.(map[string]any)
+		if parameter["name"] == name {
+			return true
+		}
+	}
+	return false
+}
+
+func workflowParameter(t *testing.T, workflow *unstructured.Unstructured, name string) string {
+	t.Helper()
+	parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters")
+	if err != nil || !found {
+		t.Fatalf("read Workflow parameters: found=%t error=%v", found, err)
+	}
+	for _, item := range parameters {
+		parameter := item.(map[string]any)
+		if parameter["name"] == name {
+			value, _ := parameter["value"].(string)
+			return value
+		}
+	}
+	t.Fatalf("Workflow parameter %q not found", name)
+	return ""
+}
+
+func workflowMutex(t *testing.T, workflow *unstructured.Unstructured) string {
+	t.Helper()
+	mutexes, found, err := unstructured.NestedSlice(workflow.Object, "spec", "synchronization", "mutexes")
+	if err != nil || !found || len(mutexes) != 1 {
+		t.Fatalf("read Workflow mutex: count=%d found=%t error=%v", len(mutexes), found, err)
+	}
+	mutex, ok := mutexes[0].(map[string]any)
+	if !ok {
+		t.Fatalf("Workflow mutex = %#v", mutexes[0])
+	}
+	name, _ := mutex["name"].(string)
+	return name
+}