ms2k/quorate
SHA256
Watch
0
Code Releases Activity

ce7cb702218693d08ea6d83464d2457c9250027c

Browse source
This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
23 changed files with 145336 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

367
services/project-delivery/controller/atomic_test.go Normal file
View file

@ -0,0 +1,367 @@
package main
import (
"context"
"errors"
"reflect"
"slices"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
)
type admitClient struct {
dynamic.Interface
before func(*unstructured.Unstructured)
afterList func()
}
type admitScope struct {
dynamic.NamespaceableResourceInterface
before func(*unstructured.Unstructured)
afterList func()
}
type admitResource struct {
dynamic.ResourceInterface
before func(*unstructured.Unstructured)
afterList func()
}
func (c admitClient) Resource(gvr schema.GroupVersionResource) dynamic.NamespaceableResourceInterface {
resource := c.Interface.Resource(gvr)
if gvr == workflowGVR {
return admitScope{NamespaceableResourceInterface: resource, before: c.before, afterList: c.afterList}
}
return resource
}
func (r admitScope) Namespace(namespace string) dynamic.ResourceInterface {
return admitResource{ResourceInterface: r.NamespaceableResourceInterface.Namespace(namespace), before: r.before, afterList: r.afterList}
}
func (r admitResource) Create(ctx context.Context, object *unstructured.Unstructured, options metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) {
if r.before != nil {
r.before(object)
}
return r.ResourceInterface.Create(ctx, object, options, subresources...)
}
func TestLateAdmission(t *testing.T) {
ctx := context.Background()
instance, template := bundleFixture(t)
peer := *instance
previous, err := peer.readyTemplate(ctx, "delivery", template.GetName())
if err != nil {
t.Fatal(err)
}
old, err := peer.client.Resource(templateGVR).Namespace("delivery").Get(ctx, previous, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
published := ""
instance.client = admitClient{Interface: peer.client, before: func(_ *unstructured.Unstructured) {
changeImage(t, &peer, template.GetName(), "example.invalid/builder:v2")
published, err = peer.readyTemplate(ctx, "delivery", template.GetName())
if err != nil {
t.Fatal(err)
}
}}
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
if err := instance.ensureWorkflow(ctx, &delivery, "late", template.GetName(), "build", nil, nil); err != nil {
t.Fatal(err)
}
captured := boundTemplate(t, instance, "late")
if !reflect.DeepEqual(old.Object["spec"], captured.Object["spec"]) {
t.Fatal("captured content changed")
}
if captured.GetName() == published {
t.Fatal("the schedule did not overlap two published revisions")
}
}
func (r admitResource) List(ctx context.Context, options metav1.ListOptions) (*unstructured.UnstructuredList, error) {
items, err := r.ResourceInterface.List(ctx, options)
if err == nil && r.afterList != nil {
r.afterList()
}
return items, err
}
func TestPublishOverlap(t *testing.T) {
ctx := context.Background()
instance, template := bundleFixture(t)
peer := *instance
base := peer.client
previous, err := peer.readyTemplate(ctx, "delivery", template.GetName())
if err != nil {
t.Fatal(err)
}
published := ""
instance.client = admitClient{Interface: base, before: func(workflow *unstructured.Unstructured) {
changeImage(t, &peer, template.GetName(), "example.invalid/builder:v2")
peer.client = admitClient{Interface: base, afterList: func() {
if _, err := base.Resource(workflowGVR).Namespace("delivery").Create(ctx, workflow.DeepCopy(), metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
}}
published, err = peer.readyTemplate(ctx, "delivery", template.GetName())
if err != nil {
t.Fatal(err)
}
}}
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
if err := instance.ensureWorkflow(ctx, &delivery, "late", template.GetName(), "build", nil, nil); err != nil {
t.Fatal(err)
}
captured := boundTemplate(t, instance, "late")
if captured.GetName() != previous {
t.Fatal("execution lost its captured revision")
}
if published == previous {
t.Fatal("the schedule did not publish during an in-flight admission")
}
}
func TestQuietSameBundle(t *testing.T) {
ctx := context.Background()
instance, template := bundleFixture(t)
now := instance.now()
instance.now = func() time.Time { return now }
instance.config.templateQuiet = time.Minute
if err := unstructured.SetNestedField(template.Object, "nodeSelector: {}", "spec", "podSpecPatch"); err != nil {
t.Fatal(err)
}
if _, err := instance.client.Resource(templateGVR).Namespace("delivery").Update(ctx, template, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
if _, err := instance.readyTemplate(ctx, "delivery", template.GetName()); !errors.Is(err, templatePending) {
t.Fatalf("initial observation = %v", err)
}
now = now.Add(time.Minute)
previous, err := instance.readyTemplate(ctx, "delivery", template.GetName())
if err != nil {
t.Fatal(err)
}
template, err = instance.client.Resource(templateGVR).Namespace("delivery").Get(ctx, template.GetName(), metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
template.SetGeneration(template.GetGeneration() + 1)
if err := unstructured.SetNestedField(template.Object, "{nodeSelector: {}}\n", "spec", "podSpecPatch"); err != nil {
t.Fatal(err)
}
if _, err := instance.client.Resource(templateGVR).Namespace("delivery").Update(ctx, template, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
bundle, err := instance.captureBundle(ctx, template)
if err != nil {
t.Fatal(err)
}
if bundle.name != previous {
t.Fatal("equivalent patches changed captured revision")
}
published, err := instance.readyTemplate(ctx, "delivery", template.GetName())
if err != nil || published != previous {
t.Fatalf("unchanged revision restarted quiet interval: published=%q error=%v", published, err)
}
}
func TestPointCheckpoint(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
revisions := []string{"0123456789abcdef0123456789abcdef01234567", "1123456789abcdef0123456789abcdef01234567"}
object := pointDelivery(revisions...)
instance := testController(t, object, &mutableRefs{}, &now)
delivery, err := decodeProjectDelivery(object)
if err != nil {
t.Fatal(err)
}
for _, revision := range revisions {
attempt := &attemptStatus{Revision: revision}
if err := instance.ensureWorkflow(ctx, &delivery, attemptName(&delivery, attempt), delivery.Spec.WorkflowTemplateRef.Name, "build", map[string]string{"source-ref": "", "source-revision": revision}, nil); err != nil {
t.Fatal(err)
}
}
if err := instance.reconcileKey(ctx, "delivery", "point"); err != nil {
t.Fatal(err)
}
stored, err := decodeProjectDelivery(getDelivery(t, instance, "point"))
if err != nil {
t.Fatal(err)
}
if len(stored.Status.Points) != 1 {
t.Fatalf("Point checkpoint count = %d", len(stored.Status.Points))
}
first := stored.Status.Points[0].WorkflowName
for _, revision := range revisions {
name := attemptName(&delivery, &attemptStatus{Revision: revision})
workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
held := slices.Contains(workflow.GetFinalizers(), recordKey)
if held != (name != first) {
t.Fatalf("checkpoint protection for %s = %t", name, held)
}
}
setNamedWorkflowStatus(t, instance, first, "Succeeded", nil)
for range 2 {
if err := instance.reconcileKey(ctx, "delivery", "point"); err != nil {
t.Fatal(err)
}
}
stored, err = decodeProjectDelivery(getDelivery(t, instance, "point"))
if err != nil {
t.Fatal(err)
}
if len(stored.Status.Points) != 2 {
t.Fatalf("recovered checkpoint count = %d", len(stored.Status.Points))
}
}
func TestRetryOldMode(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
revision := "0123456789abcdef0123456789abcdef01234567"
instance := testController(t, continuousDelivery(), &mutableRefs{values: map[string]string{"refs/heads/main": revision}}, &now)
for range 3 {
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
}
object := getDelivery(t, instance, "continuous")
delivery, err := decodeProjectDelivery(object)
if err != nil {
t.Fatal(err)
}
first := delivery.Status.Continuous.Attempt.WorkflowName
setNamedWorkflowStatus(t, instance, first, "Failed", nil)
instance.reconcileAll(ctx)
object = getDelivery(t, instance, "continuous")
unstructured.RemoveNestedField(object.Object, "spec", "source", "continuous")
if err := unstructured.SetNestedMap(object.Object, map[string]any{"revisions": []any{revision}}, "spec", "source", "point"); err != nil {
t.Fatal(err)
}
annotations := object.GetAnnotations()
if annotations == nil {
annotations = make(map[string]string)
}
annotations[retryAnnotation] = first
object.SetAnnotations(annotations)
if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(ctx, object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
if err := instance.reconcileKey(ctx, "delivery", "continuous"); err != nil {
t.Fatal(err)
}
stored, err := decodeProjectDelivery(getDelivery(t, instance, "continuous"))
if err != nil {
t.Fatal(err)
}
retry := stored.Status.Continuous.Attempt
if stored.Spec.Source.Point == nil || retry.RetryOf != first || retry.Ref != "refs/heads/main" || retry.WorkflowName == first {
t.Fatalf("old-mode retry did not preserve captured selection: %+v", retry)
}
t.Log("retry accepted in Continuous mode while the declared source mode is Point")
}
func TestTreeOverlap(t *testing.T) {
ctx := context.Background()
instance, _ := bundleFixture(t)
peer := *instance
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
first := &attemptStatus{Ref: "refs/heads/main", Revision: "0123456789abcdef0123456789abcdef01234567", SourceIdentity: "sha256:tree"}
second := &attemptStatus{Ref: first.Ref, Revision: "1123456789abcdef0123456789abcdef01234567", SourceIdentity: first.SourceIdentity}
instance.client = admitClient{Interface: peer.client, before: func(_ *unstructured.Unstructured) {
if err := peer.advanceAttempt(ctx, &delivery, second); err != nil {
t.Fatal(err)
}
}}
if err := instance.advanceAttempt(ctx, &delivery, first); err != nil {
t.Fatal(err)
}
if count := countWorkflows(t, instance, "build"); count != 1 {
t.Fatalf("one source identity produced %d accepted workflows from overlapping SHA observations", count)
}
if first.WorkflowName != second.WorkflowName {
t.Fatal("overlapping observations did not converge on one captured execution")
}
if first.Revision != first.Input["source-revision"] {
t.Fatalf("accepted Workflow SHA %q differs from recorded attempt SHA %q", first.Input["source-revision"], first.Revision)
}
}
func TestRefOverlap(t *testing.T) {
ctx := context.Background()
instance, _ := bundleFixture(t)
peer := *instance
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
first := &attemptStatus{Ref: "refs/heads/main", Revision: "0123456789abcdef0123456789abcdef01234567", SourceIdentity: "0123456789abcdef0123456789abcdef01234567"}
second := &attemptStatus{Ref: "refs/heads/release", Revision: first.Revision, SourceIdentity: first.SourceIdentity}
instance.client = admitClient{Interface: peer.client, before: func(_ *unstructured.Unstructured) {
if err := peer.advanceAttempt(ctx, &delivery, second); err != nil {
t.Fatal(err)
}
}}
if err := instance.advanceAttempt(ctx, &delivery, first); err != nil {
t.Fatal(err)
}
if count := countWorkflows(t, instance, "build"); count != 1 {
t.Fatalf("one trigger identity produced %d workflows after a ref change", count)
}
if first.Ref != first.Input["source-ref"] || first.WorkflowName != second.WorkflowName {
t.Fatal("captured ref did not follow the accepted execution")
}
}
func TestDetachedPublish(t *testing.T) {
ctx := context.Background()
instance, template := bundleFixture(t)
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
if err := instance.ensureWorkflow(ctx, &delivery, "old", template.GetName(), "build", nil, nil); err != nil {
t.Fatal(err)
}
changeImage(t, instance, template.GetName(), "example.invalid/builder:v2")
if _, err := instance.readyTemplate(ctx, "delivery", template.GetName()); !errors.Is(err, templatePending) {
t.Fatalf("pending update = %v", err)
}
object := getDelivery(t, instance, "continuous")
if err := unstructured.SetNestedField(object.Object, "other-delivery", "spec", "workflowTemplateRef", "name"); err != nil {
t.Fatal(err)
}
if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(ctx, object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
setNamedWorkflowStatus(t, instance, "old", "Succeeded", nil)
for range 2 {
if err := instance.reconcileTemplates(ctx); err != nil {
t.Fatal(err)
}
}
stored, err := instance.client.Resource(templateGVR).Namespace("delivery").Get(ctx, template.GetName(), metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if stored.GetAnnotations()[publishedKey] != stored.GetAnnotations()[candidateKey] {
t.Fatal("unreferenced template retained its update but never published it after drain")
}
}

164
services/project-delivery/controller/bundle_test.go Normal file
View file

@ -0,0 +1,164 @@
package main
import (
"context"
"encoding/json"
"reflect"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/yaml"
)
func TestCapturedSpec(t *testing.T) {
instance, template := bundleFixture(t)
ctx := context.Background()
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
if err := instance.ensureWorkflow(ctx, &delivery, "first", template.GetName(), "build", nil, nil); err != nil {
t.Fatal(err)
}
bound := boundTemplate(t, instance, "first")
stored, _, _ := unstructured.NestedMap(bound.Object, "spec")
if err := unstructured.SetNestedSlice(template.Object, []any{map[string]any{"name": "build", "container": map[string]any{"image": "example.invalid/builder:v2"}}}, "spec", "templates"); err != nil {
t.Fatal(err)
}
gvr := schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflowtemplates"}
if _, err := instance.client.Resource(gvr).Namespace("delivery").Update(ctx, template, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
live, _, _ := unstructured.NestedMap(boundTemplate(t, instance, "first").Object, "spec")
if !reflect.DeepEqual(stored, live) {
t.Fatal("captured Workflow spec changed through its live template reference")
}
}
func TestCapturedActions(t *testing.T) {
instance, template := bundleFixture(t)
ctx := context.Background()
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
if err := instance.ensureWorkflow(ctx, &delivery, "first", template.GetName(), "build", nil, nil); err != nil {
t.Fatal(err)
}
gvr := schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}
config, err := instance.client.Resource(gvr).Namespace("delivery").Get(ctx, "actions", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
config.Object["data"] = map[string]any{"build.fsx": "printfn \"new action\""}
if _, err := instance.client.Resource(gvr).Namespace("delivery").Update(ctx, config, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
volumes, _, _ := unstructured.NestedSlice(boundTemplate(t, instance, "first").Object, "spec", "volumes")
name := volumes[0].(map[string]any)["configMap"].(map[string]any)["name"].(string)
captured, err := instance.client.Resource(gvr).Namespace("delivery").Get(ctx, name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
value, _, _ := unstructured.NestedString(captured.Object, "data", "build.fsx")
if value != "printfn \"old action\"" {
t.Fatalf("a later Pod resolves changed action content: %q", value)
}
}
func bundleFixture(t *testing.T) (*controller, *unstructured.Unstructured) {
t.Helper()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
instance := testController(t, continuousDelivery(), &mutableRefs{}, &now)
ctx := context.Background()
config := &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "v1", "kind": "ConfigMap",
"metadata": map[string]any{"name": "actions", "namespace": "delivery", "labels": map[string]any{"app.kubernetes.io/name": "project-delivery"}},
"data": map[string]any{"build.fsx": "printfn \"old action\""},
}}
if _, err := instance.client.Resource(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}).Namespace("delivery").Create(ctx, config, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
template := &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "argoproj.io/v1alpha1", "kind": "WorkflowTemplate",
"metadata": map[string]any{"name": "nuthatch-delivery", "namespace": "delivery"},
"spec": map[string]any{
"entrypoint": "build",
"templates": []any{map[string]any{"name": "build", "container": map[string]any{"image": "example.invalid/builder:v1"}}},
"volumes": []any{map[string]any{"name": "actions", "configMap": map[string]any{"name": "actions"}}},
},
}}
gvr := schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflowtemplates"}
if _, err := instance.client.Resource(gvr).Namespace("delivery").Update(ctx, template, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
return instance, template
}
func boundTemplate(t *testing.T, instance *controller, name string) *unstructured.Unstructured {
t.Helper()
ctx := context.Background()
workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
ref, _, _ := unstructured.NestedString(workflow.Object, "spec", "workflowTemplateRef", "name")
gvr := schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflowtemplates"}
template, err := instance.client.Resource(gvr).Namespace("delivery").Get(ctx, ref, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
return template
}
func TestCapturedPatch(t *testing.T) {
instance, template := bundleFixture(t)
ctx := context.Background()
unstructured.RemoveNestedField(template.Object, "spec", "volumes")
if err := unstructured.SetNestedSlice(template.Object, []any{map[string]any{
"name": "build", "container": map[string]any{"image": "example.invalid/builder:v1"},
"podSpecPatch": "volumes:\n- name: actions\n configMap:\n name: actions\n",
}}, "spec", "templates"); err != nil {
t.Fatal(err)
}
if _, err := instance.client.Resource(templateGVR).Namespace("delivery").Update(ctx, template, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
if err := instance.ensureWorkflow(ctx, &delivery, "first", template.GetName(), "build", nil, nil); err != nil {
t.Fatal(err)
}
config, err := instance.client.Resource(configMapGVR).Namespace("delivery").Get(ctx, "actions", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
config.Object["data"] = map[string]any{"build.fsx": "printfn \"new action\""}
if _, err := instance.client.Resource(configMapGVR).Namespace("delivery").Update(ctx, config, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
templates, _, _ := unstructured.NestedSlice(boundTemplate(t, instance, "first").Object, "spec", "templates")
patch := templates[0].(map[string]any)["podSpecPatch"].(string)
data, err := yaml.YAMLToJSON([]byte(patch))
if err != nil {
t.Fatal(err)
}
var body map[string]any
if err := json.Unmarshal(data, &body); err != nil {
t.Fatal(err)
}
name := body["volumes"].([]any)[0].(map[string]any)["configMap"].(map[string]any)["name"].(string)
captured, err := instance.client.Resource(configMapGVR).Namespace("delivery").Get(ctx, name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
value, _, _ := unstructured.NestedString(captured.Object, "data", "build.fsx")
if value != "printfn \"old action\"" {
t.Fatalf("nested podSpecPatch resolves changed actions: %q", value)
}
}

70
services/project-delivery/controller/checkpoint_test.go Normal file
View file

@ -0,0 +1,70 @@
package main
import (
"context"
"errors"
"testing"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
dynamicfake "k8s.io/client-go/dynamic/fake"
k8stesting "k8s.io/client-go/testing"
)
func TestCaptureCollected(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
revision := "0123456789abcdef0123456789abcdef01234567"
instance := testController(t, pointDelivery(revision), &mutableRefs{values: map[string]string{"refs/heads/main": revision}}, &now)
client := instance.client.(*dynamicfake.FakeDynamicClient)
changed := false
client.PrependReactor("update", "projectdeliveries", func(action k8stesting.Action) (bool, runtime.Object, error) {
if changed || action.GetSubresource() != "status" {
return false, nil, nil
}
changed = true
raw, err := client.Tracker().Get(projectDeliveryGVR, "delivery", "point")
if err != nil {
t.Fatal(err)
}
object := raw.(*unstructured.Unstructured).DeepCopy()
name := workflowName("build", string(object.GetUID()), "", revision)
accepted, err := client.Tracker().Get(workflowGVR, "delivery", name)
if err != nil {
t.Fatal(err)
}
workflow := accepted.(*unstructured.Unstructured).DeepCopy()
workflow.Object["status"] = map[string]any{"phase": "Succeeded"}
if len(workflow.GetFinalizers()) == 0 {
if err := client.Tracker().Delete(workflowGVR, "delivery", name); err != nil {
t.Fatal(err)
}
} else {
workflow.SetDeletionTimestamp(&metav1.Time{Time: now})
if err := client.Tracker().Update(workflowGVR, workflow, "delivery"); err != nil {
t.Fatal(err)
}
}
unstructured.RemoveNestedField(object.Object, "spec", "source", "point")
if err := unstructured.SetNestedMap(object.Object, map[string]any{"ref": "refs/heads/main"}, "spec", "source", "continuous"); err != nil {
t.Fatal(err)
}
if err := client.Tracker().Update(projectDeliveryGVR, object, "delivery"); err != nil {
t.Fatal(err)
}
return true, nil, apierrors.NewConflict(projectDeliveryGVR.GroupResource(), "point", errors.New("synthetic source update"))
})
if err := instance.reconcileKey(ctx, "delivery", "point"); err != nil {
t.Fatal(err)
}
points, _, err := unstructured.NestedSlice(getDelivery(t, instance, "point").Object, "status", "points")
if err != nil {
t.Fatal(err)
}
if len(points) != 1 {
t.Fatalf("accepted execution was collected before its status checkpoint: %d records", len(points))
}
}

37
services/project-delivery/controller/collect_test.go Normal file
View file

@ -0,0 +1,37 @@
package main
import (
"context"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func TestCollectedAttempt(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
revision := "0123456789abcdef0123456789abcdef01234567"
instance := testController(t, pointDelivery(revision), &mutableRefs{}, &now)
instance.reconcileAll(ctx)
name := workflowName("build", "11111111-2222-3333-4444-555555555555", "", revision)
setNamedWorkflowStatus(t, instance, name, "Running", nil)
instance.reconcileAll(ctx)
setNamedWorkflowStatus(t, instance, name, "Succeeded", nil)
if err := instance.client.Resource(workflowGVR).Namespace("delivery").Delete(ctx, name, metav1.DeleteOptions{}); err != nil {
t.Fatal(err)
}
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
if count := countWorkflows(t, instance, "build"); count != 0 {
t.Fatalf("collected execution was resubmitted: %d Workflows", count)
}
points, _, err := unstructured.NestedSlice(getDelivery(t, instance, "point").Object, "status", "points")
if err != nil {
t.Fatal(err)
}
if phase := points[0].(map[string]any)["phase"].(string); !terminalPhase(phase) {
t.Fatalf("collected attempt remains %s", phase)
}
}

759
services/project-delivery/controller/controller.go Normal file
View file

@ -0,0 +1,759 @@
package main
import (
"context"
"crypto/sha256"
"encoding/base32"
"encoding/json"
"errors"
"fmt"
"log/slog"
"slices"
"sort"
"strings"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/util/retry"
)
var (
projectDeliveryGVR = schema.GroupVersionResource{Group: "delivery.quorate.dev", Version: "v1alpha1", Resource: "projectdeliveries"}
workflowGVR = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "workflows"}
)
type controllerConfig struct {
pollInterval time.Duration
bucketInterval time.Duration
requestTimeout time.Duration
templateQuiet time.Duration
}
type controller struct {
client dynamic.Interface
source sourceReader
now func() time.Time
config controllerConfig
qualCursor string
}
func (c *controller) run(ctx context.Context) error {
ticker := time.NewTicker(c.config.pollInterval)
defer ticker.Stop()
for {
c.reconcileAll(ctx)
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
}
func (c *controller) reconcileAll(ctx context.Context) {
if err := c.releaseOrphans(ctx); err != nil {
slog.Error("release orphan records", "error", err)
}
if err := c.reconcileTemplates(ctx); err != nil {
slog.Error("reconcile templates", "error", err)
}
if err := c.qualifyAll(ctx); err != nil {
slog.Error("reconcile qualification", "error", err)
}
items, err := c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
slog.Error("list ProjectDeliveries", "error", err)
return
}
for index := range items.Items {
item := &items.Items[index]
if err := c.reconcileKey(ctx, item.GetNamespace(), item.GetName()); err != nil {
slog.Error("reconcile ProjectDelivery", "namespace", item.GetNamespace(), "name", item.GetName(), "error", err)
c.recordError(ctx, item.GetNamespace(), item.GetName(), err)
}
}
}
func (c *controller) reconcileKey(ctx context.Context, namespace, name string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
object, err := c.client.Resource(projectDeliveryGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
return c.reconcile(ctx, object)
})
}
func (c *controller) reconcile(ctx context.Context, object *unstructured.Unstructured) error {
delivery, err := decodeProjectDelivery(object)
if err != nil {
return err
}
if delivery.Metadata.DeletionTimestamp != nil {
return nil
}
if err := validateTargets(delivery.Spec.Targets); err != nil {
return err
}
recovered, err := c.recoverBuilds(ctx, &delivery)
if err != nil {
return err
}
active, err := c.resumeAttempts(ctx, &delivery)
if err != nil {
return err
}
if active || recovered {
return c.updateStatus(ctx, object, &delivery)
}
retried, err := c.reconcileRetry(ctx, &delivery)
if err != nil {
return err
}
var reconcileErr error
switch {
case retried:
case delivery.Spec.Source.Continuous != nil:
refs, err := c.readRefs(ctx, delivery.Spec.Source.Repository)
if err != nil {
return fmt.Errorf("read Forgejo refs: %w", err)
}
reconcileErr = c.reconcileContinuous(ctx, &delivery, refs)
case delivery.Spec.Source.Point != nil:
reconcileErr = c.reconcilePoint(ctx, &delivery)
default:
reconcileErr = errors.New("delivery source has no mode")
}
if reconcileErr != nil {
return reconcileErr
}
delivery.Status.ObservedGeneration = delivery.Metadata.Generation
return c.updateStatus(ctx, object, &delivery)
}
func (c *controller) resumeAttempts(ctx context.Context, delivery *projectDelivery) (bool, error) {
active := false
if status := delivery.Status.Continuous; status != nil && status.Attempt != nil && !terminalPhase(status.Attempt.Phase) {
if err := c.advanceAttempt(ctx, delivery, status.Attempt); err != nil {
return false, err
}
recordDelivered(status)
delivery.Status.Message = "build attempt: " + phaseOrPending(status.Attempt.Phase)
active = true
}
for index := range delivery.Status.Points {
attempt := &delivery.Status.Points[index]
if terminalPhase(attempt.Phase) {
continue
}
if err := c.advanceAttempt(ctx, delivery, attempt); err != nil {
return false, err
}
delivery.Status.Message = "point attempt: " + phaseOrPending(attempt.Phase)
active = true
}
return active, nil
}
func (c *controller) reconcileRetry(ctx context.Context, delivery *projectDelivery) (bool, error) {
request := delivery.Metadata.Annotations[retryAnnotation]
if request == "" || request == delivery.Status.LastHandledRetry {
return false, nil
}
target, continuous := retryTarget(delivery, request)
if target == nil {
return false, fmt.Errorf("retry Workflow %q is not a recorded attempt", request)
}
if otherAttemptActive(delivery, request) {
return false, nil
}
if !terminalPhase(target.Phase) {
return false, nil
}
successor := attemptStatus{
Ref: target.Ref,
Revision: target.Revision,
SourceIdentity: target.SourceIdentity,
RetryOf: request,
Phase: "Pending",
}
if err := c.advanceAttempt(ctx, delivery, &successor); err != nil {
return false, err
}
if successor.WorkflowName == "" {
delivery.Status.Message = templatePending.Error()
return true, nil
}
if continuous {
rememberAttempt(delivery.Status.Continuous, &successor)
target = delivery.Status.Continuous.Attempt
} else {
delivery.Status.Points = append(delivery.Status.Points, successor)
target = &delivery.Status.Points[len(delivery.Status.Points)-1]
}
delivery.Status.LastHandledRetry = request
delivery.Status.Message = "retry attempt: " + phaseOrPending(target.Phase)
return true, nil
}
func retryTarget(delivery *projectDelivery, workflow string) (*attemptStatus, bool) {
if delivery.Status.Continuous != nil {
attempt := delivery.Status.Continuous.Attempt
if attempt != nil && attempt.WorkflowName == workflow {
return attempt, true
}
}
for index := range delivery.Status.Points {
attempt := &delivery.Status.Points[index]
if attempt.WorkflowName == workflow {
return attempt, false
}
}
return nil, false
}
func otherAttemptActive(delivery *projectDelivery, excluded string) bool {
if delivery.Status.Continuous != nil {
attempt := delivery.Status.Continuous.Attempt
if attempt != nil && attempt.WorkflowName != excluded && !terminalPhase(attempt.Phase) {
return true
}
}
for index := range delivery.Status.Points {
attempt := &delivery.Status.Points[index]
if attempt.WorkflowName != excluded && !terminalPhase(attempt.Phase) {
return true
}
}
return false
}
func (c *controller) reconcileContinuous(
ctx context.Context,
delivery *projectDelivery,
refs map[string]string,
) error {
ref := delivery.Spec.Source.Continuous.Ref
revision, found := refs[ref]
if !found {
return fmt.Errorf("continuous ref %q is absent", ref)
}
if delivery.Status.Continuous == nil {
delivery.Status.Continuous = &continuousStatus{}
}
status := delivery.Status.Continuous
sourceIdentity, err := c.sourceIdentity(ctx, delivery, revision)
if err != nil {
return fmt.Errorf("read continuous path identity: %w", err)
}
filtered := len(delivery.Spec.Source.Continuous.Paths) != 0
if consumedIdentity(status, sourceIdentity) {
clearCandidate(status)
delivery.Status.Message = "continuous source already attempted"
return nil
}
if deliveredMatches(status.Delivered, sourceIdentity) ||
status.Delivered == nil && status.Attempt != nil && status.Attempt.Phase == "Succeeded" &&
attemptMatches(status.Attempt, sourceIdentity, revision, filtered) {
clearCandidate(status)
delivery.Status.Message = "continuous source already delivered"
return nil
}
if status.Attempt != nil && terminalPhase(status.Attempt.Phase) &&
attemptMatches(status.Attempt, sourceIdentity, revision, filtered) {
delivery.Status.Message = "continuous source already attempted"
return nil
}
now := c.now().UTC()
bucket := now.Truncate(c.config.bucketInterval).Format(time.RFC3339)
if status.CandidateIdentity != sourceIdentity {
status.CandidateRevision = revision
status.CandidateIdentity = sourceIdentity
status.CandidateBuckets = []string{bucket}
status.CandidateSince = &now
if delivery.Spec.Source.Continuous.Debounce == "" {
delivery.Status.Message = "observed continuous candidate 1/3"
} else {
delivery.Status.Message = "observed continuous candidate"
}
return nil
}
status.CandidateRevision = revision
if configured := delivery.Spec.Source.Continuous.Debounce; configured != "" {
debounce, err := time.ParseDuration(configured)
if err != nil || debounce <= 0 {
return fmt.Errorf("invalid continuous debounce %q", configured)
}
if status.CandidateSince == nil {
status.CandidateSince = &now
delivery.Status.Message = "observed continuous candidate"
return nil
}
if now.Before(status.CandidateSince.Add(debounce)) {
delivery.Status.Message = "continuous candidate debouncing"
return nil
}
} else {
if !contains(status.CandidateBuckets, bucket) && len(status.CandidateBuckets) < 3 {
status.CandidateBuckets = append(status.CandidateBuckets, bucket)
}
if len(status.CandidateBuckets) < 3 {
delivery.Status.Message = fmt.Sprintf("observed continuous candidate %d/3", len(status.CandidateBuckets))
return nil
}
}
latest, err := c.readRefs(ctx, delivery.Spec.Source.Repository)
if err != nil {
return fmt.Errorf("reread continuous ref: %w", err)
}
latestRevision, found := latest[ref]
if !found {
return fmt.Errorf("continuous ref %q is absent after debounce", ref)
}
latestIdentity, err := c.sourceIdentity(ctx, delivery, latestRevision)
if err != nil {
return fmt.Errorf("reread continuous path identity: %w", err)
}
if consumedIdentity(status, latestIdentity) {
clearCandidate(status)
delivery.Status.Message = "continuous source returned to attempted state"
return nil
}
if deliveredMatches(status.Delivered, latestIdentity) {
clearCandidate(status)
delivery.Status.Message = "continuous source returned to delivered state"
return nil
}
if latestIdentity != sourceIdentity {
status.CandidateRevision = latestRevision
status.CandidateIdentity = latestIdentity
status.CandidateBuckets = nil
status.CandidateSince = nil
delivery.Status.Message = "continuous source changed before Workflow creation"
return nil
}
revision = latestRevision
status.CandidateRevision = revision
attempt := &attemptStatus{
Ref: ref,
Revision: revision,
SourceIdentity: sourceIdentity,
Phase: "Pending",
}
if err := c.advanceAttempt(ctx, delivery, attempt); err != nil {
return err
}
if attempt.WorkflowName == "" {
delivery.Status.Message = templatePending.Error()
return nil
}
rememberAttempt(status, attempt)
recordDelivered(status)
delivery.Status.Message = "build attempt: " + phaseOrPending(attempt.Phase)
return nil
}
func (c *controller) reconcilePoint(
ctx context.Context,
delivery *projectDelivery,
) error {
for _, revision := range delivery.Spec.Source.Point.Revisions {
if pointAttempted(delivery.Status.Points, revision) {
continue
}
attempt := attemptStatus{
Revision: revision,
Phase: "Pending",
}
if err := c.advanceAttempt(ctx, delivery, &attempt); err != nil {
return err
}
if attempt.WorkflowName == "" {
delivery.Status.Message = templatePending.Error()
return nil
}
delivery.Status.Points = append(delivery.Status.Points, attempt)
delivery.Status.Message = "point attempt: " + phaseOrPending(attempt.Phase)
return nil
}
delivery.Status.Message = "point revisions already attempted"
return nil
}
func (c *controller) advanceAttempt(
ctx context.Context,
delivery *projectDelivery,
attempt *attemptStatus,
) error {
if attempt.Input == nil {
targets, err := json.Marshal(delivery.Spec.Targets)
if err != nil {
return err
}
attempt.Input = map[string]string{
"source-repository": delivery.Spec.Source.Repository,
"source-ref": attempt.Ref,
"source-revision": attempt.Revision,
"targets": string(targets),
}
}
name := attempt.WorkflowName
if name == "" {
name = attemptName(delivery, attempt)
if err := c.ensureWorkflow(ctx, delivery, name, delivery.Spec.WorkflowTemplateRef.Name, "build", attempt.Input, map[string]string{sourceKey: attempt.SourceIdentity, retryKey: attempt.RetryOf}); err != nil {
if errors.Is(err, templatePending) {
return nil
}
return err
}
}
state, err := c.workflow(ctx, delivery.Metadata.Namespace, name)
if apierrors.IsNotFound(err) && attempt.WorkflowName != "" {
attempt.Phase = "Error"
return nil
}
if err != nil {
return err
}
attempt.WorkflowName = name
attempt.CreatedAt = state.CreatedAt
attempt.Input = state.Input
attempt.Ref = state.Input["source-ref"]
attempt.Revision = state.Input["source-revision"]
attempt.Phase = phaseOrPending(state.Phase)
if result, found := state.Outputs["delivery-result"]; found {
attempt.Result = opaqueResult(result)
}
observeQualBuild(delivery, attempt)
return nil
}
func (c *controller) ensureWorkflow(
ctx context.Context,
delivery *projectDelivery,
name, templateName, purpose string,
parameters map[string]string,
capture map[string]string,
) error {
resource := c.client.Resource(workflowGVR).Namespace(delivery.Metadata.Namespace)
if _, err := resource.Get(ctx, name, metav1.GetOptions{}); err == nil {
return nil
} else if !apierrors.IsNotFound(err) {
return err
}
published, err := c.readyTemplate(ctx, delivery.Metadata.Namespace, templateName)
if errors.Is(err, templatePending) {
if _, getErr := resource.Get(ctx, name, metav1.GetOptions{}); getErr == nil {
return nil
} else if !apierrors.IsNotFound(getErr) {
return getErr
}
}
if err != nil {
return err
}
metadata := map[string]any{
"name": name,
"namespace": delivery.Metadata.Namespace,
"annotations": map[string]any{templateKey: templateName},
"finalizers": []any{recordKey},
"labels": map[string]any{
"app.kubernetes.io/name": "project-delivery",
"app.kubernetes.io/instance": delivery.Metadata.Name,
"delivery.quorate.dev/purpose": purpose,
recordKey: "pending",
},
"ownerReferences": []any{map[string]any{
"apiVersion": "delivery.quorate.dev/v1alpha1",
"kind": "ProjectDelivery",
"name": delivery.Metadata.Name,
"uid": delivery.Metadata.UID,
"controller": true,
"blockOwnerDeletion": false,
}},
}
for key, value := range capture {
metadata["annotations"].(map[string]any)[key] = value
}
names := make([]string, 0, len(parameters))
for parameter := range parameters {
names = append(names, parameter)
}
sort.Strings(names)
arguments := make([]any, 0, len(names))
for _, parameter := range names {
arguments = append(arguments, map[string]any{"name": parameter, "value": parameters[parameter]})
}
workflow := &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "argoproj.io/v1alpha1",
"kind": "Workflow",
"metadata": metadata,
"spec": map[string]any{
"workflowTemplateRef": map[string]any{"name": published},
"arguments": map[string]any{"parameters": arguments},
"synchronization": map[string]any{
"mutexes": []any{map[string]any{"name": templateName}},
},
},
}}
_, err = resource.Create(ctx, workflow, metav1.CreateOptions{})
if apierrors.IsAlreadyExists(err) {
return nil
}
return err
}
func (c *controller) workflow(ctx context.Context, namespace, name string) (workflowState, error) {
object, err := c.client.Resource(workflowGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return workflowState{}, err
}
phase, _, _ := unstructured.NestedString(object.Object, "status", "phase")
outputs := map[string]string{}
parameters, _, _ := unstructured.NestedSlice(object.Object, "status", "outputs", "parameters")
rootParameters, _, _ := unstructured.NestedSlice(object.Object, "status", "nodes", name, "outputs", "parameters")
parameters = append(parameters, rootParameters...)
for _, item := range parameters {
parameter, ok := item.(map[string]any)
if !ok {
continue
}
name, _ := parameter["name"].(string)
value, _ := parameter["value"].(string)
if name != "" {
outputs[name] = value
}
}
input := map[string]string{}
arguments, _, _ := unstructured.NestedSlice(object.Object, "spec", "arguments", "parameters")
for _, item := range arguments {
parameter := item.(map[string]any)
name, _ := parameter["name"].(string)
value, _ := parameter["value"].(string)
input[name] = value
}
var startedAt *time.Time
if value, _, _ := unstructured.NestedString(object.Object, "status", "startedAt"); value != "" {
timestamp, err := time.Parse(time.RFC3339, value)
if err != nil {
return workflowState{}, err
}
if !timestamp.IsZero() {
startedAt = &timestamp
}
}
ownerUID := ""
for _, owner := range object.GetOwnerReferences() {
if owner.Kind == "ProjectDelivery" && owner.APIVersion == "delivery.quorate.dev/v1alpha1" {
ownerUID = string(owner.UID)
}
}
createdAt := object.GetCreationTimestamp()
return workflowState{Phase: phase, Outputs: outputs, Input: input, StartedAt: startedAt, OwnerUID: ownerUID, CreatedAt: &createdAt.Time}, nil
}
func consumedIdentity(status *continuousStatus, identity string) bool {
return status.Attempt != nil && status.Attempt.History != nil && slices.Contains(status.Attempt.History.Identities, identity)
}
func rememberAttempt(status *continuousStatus, attempt *attemptStatus) {
history := &attemptHistory{}
if previous := status.Attempt; previous != nil {
if previous.History != nil {
history.Workflows = slices.Clone(previous.History.Workflows)
history.Identities = slices.Clone(previous.History.Identities)
}
addHistory(history, previous)
}
addHistory(history, attempt)
attempt.History = history
status.Attempt = attempt
}
func addHistory(history *attemptHistory, attempt *attemptStatus) {
if attempt.WorkflowName != "" && !slices.Contains(history.Workflows, attempt.WorkflowName) {
history.Workflows = append(history.Workflows, attempt.WorkflowName)
}
if attempt.SourceIdentity != "" && !slices.Contains(history.Identities, attempt.SourceIdentity) {
history.Identities = append(history.Identities, attempt.SourceIdentity)
}
}
func (c *controller) updateStatus(ctx context.Context, object *unstructured.Unstructured, delivery *projectDelivery) error {
status, err := toMap(delivery.Status)
if err != nil {
return err
}
copy := object.DeepCopy()
copy.Object["status"] = status
_, err = c.client.Resource(projectDeliveryGVR).Namespace(copy.GetNamespace()).UpdateStatus(ctx, copy, metav1.UpdateOptions{})
if apierrors.IsNotFound(err) {
return c.releaseOrphans(ctx)
}
if err != nil {
return err
}
return c.releaseRecords(ctx, delivery)
}
func (c *controller) recordError(ctx context.Context, namespace, name string, reconcileError error) {
object, err := c.client.Resource(projectDeliveryGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return
}
delivery, err := decodeProjectDelivery(object)
if err != nil {
return
}
delivery.Status.Message = reconcileError.Error()
delivery.Status.ObservedGeneration = delivery.Metadata.Generation
_ = c.updateStatus(ctx, object, &delivery)
}
func decodeProjectDelivery(object *unstructured.Unstructured) (projectDelivery, error) {
content, err := json.Marshal(object.Object)
if err != nil {
return projectDelivery{}, err
}
var delivery projectDelivery
if err := json.Unmarshal(content, &delivery); err != nil {
return projectDelivery{}, err
}
return delivery, nil
}
func validateTargets(targets []json.RawMessage) error {
seen := make(map[string]struct{}, len(targets))
for index, target := range targets {
key := string(target)
if _, found := seen[key]; found {
return fmt.Errorf("duplicate target at index %d", index)
}
seen[key] = struct{}{}
}
return nil
}
func toMap(value any) (map[string]any, error) {
content, err := json.Marshal(value)
if err != nil {
return nil, err
}
var result map[string]any
err = json.Unmarshal(content, &result)
return result, err
}
func workflowName(purpose string, fields ...string) string {
digest := sha256.Sum256([]byte("v1\x00" + purpose + "\x00" + strings.Join(fields, "\x00")))
encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(digest[:20])
return "pd-" + purpose + "-" + strings.ToLower(encoded)
}
func attemptName(delivery *projectDelivery, attempt *attemptStatus) string {
sourceMode := ""
identity := attempt.Revision
if attempt.Ref != "" {
sourceMode = "continuous"
if attempt.SourceIdentity != "" {
identity = attempt.SourceIdentity
}
}
fields := []string{delivery.Metadata.UID, sourceMode, identity}
if attempt.RetryOf != "" {
fields = append(fields, attempt.RetryOf)
}
return workflowName("build", fields...)
}
func (c *controller) readRefs(ctx context.Context, repository string) (map[string]string, error) {
requestContext, cancel := context.WithTimeout(ctx, c.config.requestTimeout)
defer cancel()
return c.source.Refs(requestContext, repository)
}
func (c *controller) sourceIdentity(
ctx context.Context,
delivery *projectDelivery,
revision string,
) (string, error) {
continuous := delivery.Spec.Source.Continuous
paths := append([]string(nil), continuous.Paths...)
sort.Strings(paths)
if len(paths) == 0 {
return revision, nil
}
requestContext, cancel := context.WithTimeout(ctx, c.config.requestTimeout)
defer cancel()
value, err := c.source.PathIdentity(requestContext, delivery.Spec.Source.Repository, revision, paths)
if err != nil {
return "", err
}
return value, nil
}
func recordDelivered(status *continuousStatus) {
attempt := status.Attempt
if attempt == nil || attempt.Phase != "Succeeded" || attempt.SourceIdentity == "" {
return
}
status.Delivered = &deliveredSource{
Revision: attempt.Revision,
Identity: attempt.SourceIdentity,
}
}
func deliveredMatches(delivered *deliveredSource, sourceIdentity string) bool {
return delivered != nil && delivered.Identity == sourceIdentity
}
func attemptMatches(attempt *attemptStatus, sourceIdentity, revision string, filtered bool) bool {
if attempt.SourceIdentity != "" {
return attempt.SourceIdentity == sourceIdentity
}
return !filtered && attempt.Revision == revision
}
func clearCandidate(status *continuousStatus) {
status.CandidateRevision = ""
status.CandidateIdentity = ""
status.CandidateBuckets = nil
status.CandidateSince = nil
}
func contains(values []string, expected string) bool {
for _, value := range values {
if value == expected {
return true
}
}
return false
}
func terminalPhase(phase string) bool {
return phase == "Succeeded" || phase == "Failed" || phase == "Error"
}
func pointAttempted(attempts []attemptStatus, revision string) bool {
for _, attempt := range attempts {
if attempt.Revision == revision {
return true
}
}
return false
}
func phaseOrPending(phase string) string {
if phase == "" {
return "Pending"
}
return phase
}

992
services/project-delivery/controller/controller_test.go Normal file
View file

@ -0,0 +1,992 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"sync"
"testing"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
dynamicfake "k8s.io/client-go/dynamic/fake"
k8stesting "k8s.io/client-go/testing"
)
func TestContinuousCreatesBuildWithOpaqueTargets(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}}
now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
object.Object["spec"].(map[string]any)["source"].(map[string]any)["repository"] = "https://forge.quorate.internal/quorate/cluster"
targets := []any{map[string]any{
"kind": "Git",
"repository": "https://git.quorate.dev/ms2k/quorate",
"ref": "refs/heads/master",
"options": map[string]any{"force": true},
}}
object.Object["spec"].(map[string]any)["targets"] = targets
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(
ctx,
workflowName("build", string(object.GetUID()), "continuous", revision),
metav1.GetOptions{},
)
if err != nil {
t.Fatal(err)
}
if workflowHasParameter(t, build, "input-plan") {
t.Fatal("controller supplied input-plan")
}
if mutex := workflowMutex(t, build); mutex != "nuthatch-delivery" {
t.Fatalf("Workflow mutex = %q", mutex)
}
var passedTargets []any
if err := json.Unmarshal([]byte(workflowParameter(t, build, "targets")), &passedTargets); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(passedTargets, targets) {
t.Fatalf("targets = %#v", passedTargets)
}
}
func TestDuplicateTargetsAreRejected(t *testing.T) {
object := continuousDelivery()
target := map[string]any{"kind": "QuayOCI", "repository": "quorate/nuthatch"}
object.Object["spec"].(map[string]any)["targets"] = []any{target, target}
instance := testController(t, object, &mutableRefs{}, new(time.Time))
err := instance.reconcile(context.Background(), object)
if err == nil || err.Error() != "duplicate target at index 1" {
t.Fatalf("reconcile error = %v", err)
}
}
func TestWorkflowMutexIsScopedByTemplateReference(t *testing.T) {
now := time.Date(2026, time.August, 25, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
instance := testController(t, object, &mutableRefs{}, &now)
delivery, err := decodeProjectDelivery(object)
if err != nil {
t.Fatal(err)
}
ctx := context.Background()
for _, item := range []struct {
name string
template string
}{
{name: "first", template: "shared-delivery"},
{name: "second", template: "shared-delivery"},
{name: "third", template: "other-delivery"},
} {
if err := instance.ensureWorkflow(ctx, &delivery, item.name, item.template, "build", map[string]string{}, nil); err != nil {
t.Fatal(err)
}
workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, item.name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if mutex := workflowMutex(t, workflow); mutex != item.template {
t.Fatalf("Workflow %q mutex = %q", item.name, mutex)
}
}
}
func TestPointCreatesBuildWithoutReadingRefs(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
refs := &mutableRefs{values: map[string]string{}}
now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
object := pointDelivery(revision)
instance := testController(t, object, refs, &now)
instance.reconcileAll(context.Background())
if refs.calls != 0 {
t.Fatalf("Forgejo ref reads = %d", refs.calls)
}
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
}
type mutableRefs struct {
values map[string]string
identities map[string]string
calls int
pathCalls int
}
func (source *mutableRefs) PathIdentity(_ context.Context, _ string, revision string, _ []string) (string, error) {
source.pathCalls++
if value, found := source.identities[revision]; found {
return value, nil
}
return revision, nil
}
func (source *mutableRefs) Refs(context.Context, string) (map[string]string, error) {
source.calls++
result := make(map[string]string, len(source.values))
for name, revision := range source.values {
result[name] = revision
}
return result, nil
}
func TestPointDeliveryConsumesRevisionsWithoutReadingRefs(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
refs := &mutableRefs{values: map[string]string{}}
now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
object := pointDelivery(first)
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
if refs.calls != 0 {
t.Fatalf("Forgejo ref reads = %d", refs.calls)
}
firstBuild := workflowName("build", string(object.GetUID()), "", first)
build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, firstBuild, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if value := workflowParameter(t, build, "source-ref"); value != "" {
t.Fatalf("source-ref = %q", value)
}
if value := workflowParameter(t, build, "source-revision"); value != first {
t.Fatalf("source-revision = %q", value)
}
delivery := getDelivery(t, instance, "point")
if err := unstructured.SetNestedStringSlice(delivery.Object, []string{second}, "spec", "source", "point", "revisions"); err != nil {
t.Fatal(err)
}
if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(ctx, delivery, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
setNamedWorkflowStatus(t, instance, firstBuild, "Succeeded", map[string]string{
"delivery-result": `{"digest":"sha256:first"}`,
})
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
secondBuild := workflowName("build", string(object.GetUID()), "", second)
setNamedWorkflowStatus(t, instance, secondBuild, "Failed", nil)
instance.reconcileAll(ctx)
if refs.calls != 0 {
t.Fatalf("Forgejo ref reads = %d", refs.calls)
}
if countWorkflows(t, instance, "build") != 2 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
delivery = getDelivery(t, instance, "point")
points, found, err := unstructured.NestedSlice(delivery.Object, "status", "points")
if err != nil || !found {
t.Fatalf("read point status: found=%t error=%v", found, err)
}
if len(points) != 2 {
t.Fatalf("point status count = %d", len(points))
}
firstStatus := points[0].(map[string]any)
secondStatus := points[1].(map[string]any)
if firstStatus["revision"] != first || firstStatus["phase"] != "Succeeded" {
t.Fatalf("first point status = %#v", firstStatus)
}
if secondStatus["revision"] != second || secondStatus["phase"] != "Failed" {
t.Fatalf("second point status = %#v", secondStatus)
}
}
func TestPointDeliveryCreatesOneBuildWithTwoReconcilers(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
refs := &mutableRefs{values: map[string]string{}}
now := time.Date(2026, time.August, 20, 0, 0, 0, 0, time.UTC)
object := pointDelivery(revision)
instance := testController(t, object, refs, &now)
peer := *instance
firstObject := getDelivery(t, instance, "point")
secondObject := firstObject.DeepCopy()
errors := make(chan error, 2)
var reconcilers sync.WaitGroup
reconcilers.Add(2)
for index, reconciler := range []*controller{instance, &peer} {
object := []*unstructured.Unstructured{firstObject, secondObject}[index]
go func() {
defer reconcilers.Done()
errors <- reconciler.reconcile(context.Background(), object)
}()
}
reconcilers.Wait()
close(errors)
for err := range errors {
if err != nil {
t.Fatal(err)
}
}
if refs.calls != 0 {
t.Fatalf("Forgejo ref reads = %d", refs.calls)
}
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
delivery := getDelivery(t, instance, "point")
points, found, err := unstructured.NestedSlice(delivery.Object, "status", "points")
if err != nil || !found || len(points) != 1 {
t.Fatalf("point status: count=%d found=%t error=%v", len(points), found, err)
}
}
func TestContinuousDeliveryDebouncesAndDoesNotRetrigger(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}}
now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
instance := testController(t, continuousDelivery(), refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
setWorkflowStatus(t, instance, "build", "Succeeded", map[string]string{"delivery-result": `{"digest":"sha256:abc"}`})
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
delivery := getDelivery(t, instance, "continuous")
phase, _, _ := unstructured.NestedString(delivery.Object, "status", "continuous", "attempt", "phase")
if phase != "Succeeded" {
t.Fatalf("attempt phase = %q", phase)
}
}
func TestContinuousDeliveryUsesConfiguredDebounce(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
refs := &mutableRefs{values: map[string]string{"refs/heads/main": first}}
now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil {
t.Fatal(err)
}
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
now = now.Add(119 * time.Second)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 0 {
t.Fatal("build Workflow created before configured debounce elapsed")
}
refs.values["refs/heads/main"] = second
instance.reconcileAll(ctx)
now = now.Add(time.Second)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 0 {
t.Fatal("build Workflow created without restarting configured debounce")
}
now = now.Add(119 * time.Second)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
}
func TestPathFilteredDebounceIgnoresUnrelatedCommits(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
refs := &mutableRefs{
values: map[string]string{"refs/heads/main": first},
identities: map[string]string{first: "sha256:selected", second: "sha256:selected"},
}
now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil {
t.Fatal(err)
}
if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
t.Fatal(err)
}
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
now = now.Add(119 * time.Second)
refs.values["refs/heads/main"] = second
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 0 {
t.Fatal("build created before filtered debounce elapsed")
}
now = now.Add(time.Second)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
build, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(
ctx,
workflowName("build", string(object.GetUID()), "continuous", refs.identities[second]),
metav1.GetOptions{},
)
if err != nil {
t.Fatal(err)
}
if workflowParameter(t, build, "source-revision") != second {
t.Fatal("build did not advance to the latest unrelated commit")
}
}
func TestPathFilteredDebounceRestartsForSelectedChanges(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
refs := &mutableRefs{
values: map[string]string{"refs/heads/main": first},
identities: map[string]string{first: "sha256:first", second: "sha256:second"},
}
now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
if err := unstructured.SetNestedField(object.Object, "2m", "spec", "source", "continuous", "debounce"); err != nil {
t.Fatal(err)
}
if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
t.Fatal(err)
}
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
now = now.Add(119 * time.Second)
refs.values["refs/heads/main"] = second
instance.reconcileAll(ctx)
now = now.Add(119 * time.Second)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 0 {
t.Fatal("build created before restarted filtered debounce elapsed")
}
now = now.Add(time.Second)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
}
func TestPathFilteredDeliveryUsesSuccessfulBaseline(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
third := "fedcba9876543210fedcba9876543210fedcba98"
refs := &mutableRefs{
values: map[string]string{"refs/heads/main": first},
identities: map[string]string{
first: "sha256:first",
second: "sha256:first",
third: "sha256:third",
},
}
now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil {
t.Fatal(err)
}
if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
t.Fatal(err)
}
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
setWorkflowStatus(t, instance, "build", "Succeeded", nil)
instance.reconcileAll(ctx)
refs.values["refs/heads/main"] = second
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatal("unrelated commit created another build")
}
delivery := getDelivery(t, instance, "continuous")
delivered, found, err := unstructured.NestedMap(delivery.Object, "status", "continuous", "delivered")
if err != nil || !found || delivered["revision"] != first || delivered["identity"] != "sha256:first" {
t.Fatalf("delivered baseline = %#v, found=%t, error=%v", delivered, found, err)
}
refs.values["refs/heads/main"] = third
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 2 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
}
func TestPathFilteredFailureDoesNotAdvanceBaseline(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
third := "fedcba9876543210fedcba9876543210fedcba98"
fourth := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
refs := &mutableRefs{
values: map[string]string{"refs/heads/main": first},
identities: map[string]string{
first: "sha256:first",
second: "sha256:second",
third: "sha256:second",
fourth: "sha256:fourth",
},
}
now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil {
t.Fatal(err)
}
if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
t.Fatal(err)
}
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
setWorkflowStatus(t, instance, "build", "Succeeded", nil)
instance.reconcileAll(ctx)
refs.values["refs/heads/main"] = second
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
setNamedWorkflowStatus(
t,
instance,
workflowName("build", string(object.GetUID()), "continuous", refs.identities[second]),
"Failed",
nil,
)
instance.reconcileAll(ctx)
refs.values["refs/heads/main"] = third
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 2 {
t.Fatal("same failed filtered identity was retried")
}
delivery := getDelivery(t, instance, "continuous")
delivered, _, _ := unstructured.NestedMap(delivery.Object, "status", "continuous", "delivered")
if delivered["identity"] != "sha256:first" {
t.Fatalf("delivered baseline = %#v", delivered)
}
refs.values["refs/heads/main"] = fourth
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 3 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
}
func TestPathFilteredRevertCancelsCandidate(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
third := "fedcba9876543210fedcba9876543210fedcba98"
refs := &mutableRefs{
values: map[string]string{"refs/heads/main": first},
identities: map[string]string{
first: "sha256:first",
second: "sha256:second",
third: "sha256:first",
},
}
now := time.Date(2026, time.August, 21, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
if err := unstructured.SetNestedField(object.Object, "1m", "spec", "source", "continuous", "debounce"); err != nil {
t.Fatal(err)
}
if err := unstructured.SetNestedStringSlice(object.Object, []string{"packages/fsharp-runner"}, "spec", "source", "continuous", "paths"); err != nil {
t.Fatal(err)
}
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
setWorkflowStatus(t, instance, "build", "Succeeded", nil)
instance.reconcileAll(ctx)
refs.values["refs/heads/main"] = second
instance.reconcileAll(ctx)
refs.values["refs/heads/main"] = third
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatal("reverted filtered candidate created a build")
}
delivery := getDelivery(t, instance, "continuous")
candidate, _, _ := unstructured.NestedString(delivery.Object, "status", "continuous", "candidateIdentity")
if candidate != "" {
t.Fatalf("candidate identity = %q", candidate)
}
}
func TestContinuousDeliveryCoalescesUpdatesWhileBuildIsActive(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
third := "fedcba9876543210fedcba9876543210fedcba98"
refs := &mutableRefs{values: map[string]string{"refs/heads/main": first}}
now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
refs.values["refs/heads/main"] = second
instance.reconcileAll(ctx)
refs.values["refs/heads/main"] = third
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build count while first build active = %d", countWorkflows(t, instance, "build"))
}
setWorkflowStatus(t, instance, "build", "Succeeded", nil)
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 2 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(
ctx,
workflowName("build", string(object.GetUID()), "continuous", second),
metav1.GetOptions{},
); !apierrors.IsNotFound(err) {
t.Fatal("intermediate revision received a build workflow")
}
}
func TestDeletionCreatesNoWorkflow(t *testing.T) {
refs := &mutableRefs{values: map[string]string{}}
now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
deletion := metav1.NewTime(now)
object.SetDeletionTimestamp(&deletion)
instance := testController(t, object, refs, &now)
instance.reconcileAll(context.Background())
if countWorkflows(t, instance, "build") != 0 {
t.Fatal("deletion created a build Workflow")
}
}
func TestContinuousRetryAfterWorkflowGC(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
refs := &mutableRefs{values: map[string]string{"refs/heads/main": revision}}
now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC)
object := continuousDelivery()
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
instance.reconcileAll(ctx)
first := workflowName("build", string(object.GetUID()), "continuous", revision)
setNamedWorkflowStatus(t, instance, first, "Failed", nil)
instance.reconcileAll(ctx)
if err := instance.client.Resource(workflowGVR).Namespace("delivery").Delete(ctx, first, metav1.DeleteOptions{}); err != nil {
t.Fatal(err)
}
setRetry(t, instance, "continuous", first)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
delivery := getDelivery(t, instance, "continuous")
handled, _, _ := unstructured.NestedString(delivery.Object, "status", "lastHandledRetry")
if handled != first {
t.Fatalf("last handled retry = %q", handled)
}
successor := workflowName("build", string(object.GetUID()), "continuous", revision, first)
workflow, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, successor, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if workflowHasParameter(t, workflow, "input-plan") {
t.Fatal("retry supplied input-plan")
}
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatal("acknowledged retry created another successor")
}
}
func TestPointRetryReplaysSucceededAttempt(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
refs := &mutableRefs{values: map[string]string{}}
now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC)
object := pointDelivery(revision)
instance := testController(t, object, refs, &now)
ctx := context.Background()
instance.reconcileAll(ctx)
first := workflowName("build", string(object.GetUID()), "", revision)
setNamedWorkflowStatus(t, instance, first, "Running", nil)
instance.reconcileAll(ctx)
setRetry(t, instance, "point", first)
instance.reconcileAll(ctx)
if countWorkflows(t, instance, "build") != 1 {
t.Fatal("active attempt received a retry")
}
delivery := getDelivery(t, instance, "point")
handled, _, _ := unstructured.NestedString(delivery.Object, "status", "lastHandledRetry")
if handled != "" {
t.Fatalf("active retry was acknowledged as %q", handled)
}
setNamedWorkflowStatus(t, instance, first, "Succeeded", nil)
instance.reconcileAll(ctx)
instance.reconcileAll(ctx)
delivery = getDelivery(t, instance, "point")
points, _, _ := unstructured.NestedSlice(delivery.Object, "status", "points")
if len(points) != 2 {
t.Fatalf("point status count = %d", len(points))
}
successor := points[1].(map[string]any)
if successor["revision"] != revision {
t.Fatalf("retry status = %#v", successor)
}
if countWorkflows(t, instance, "build") != 2 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
}
func TestRetryCreatesOneSuccessorWithTwoReconcilers(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
refs := &mutableRefs{values: map[string]string{}}
now := time.Date(2026, time.August, 24, 0, 0, 0, 0, time.UTC)
object := pointDelivery(revision)
instance := testController(t, object, refs, &now)
peer := *instance
ctx := context.Background()
instance.reconcileAll(ctx)
first := workflowName("build", string(object.GetUID()), "", revision)
setNamedWorkflowStatus(t, instance, first, "Failed", nil)
instance.reconcileAll(ctx)
setRetry(t, instance, "point", first)
errors := make(chan error, 2)
var reconcilers sync.WaitGroup
for _, reconciler := range []*controller{instance, &peer} {
reconcilers.Add(1)
go func() {
defer reconcilers.Done()
errors <- reconciler.reconcileKey(ctx, "delivery", "point")
}()
}
reconcilers.Wait()
close(errors)
for err := range errors {
if err != nil {
t.Fatal(err)
}
}
if countWorkflows(t, instance, "build") != 2 {
t.Fatalf("build workflow count = %d", countWorkflows(t, instance, "build"))
}
}
func TestReconcileRetriesResourceConflict(t *testing.T) {
refs := &mutableRefs{values: map[string]string{}}
now := time.Date(2026, time.August, 17, 0, 0, 0, 0, time.UTC)
instance := testController(t, continuousDelivery(), refs, &now)
conflicts := 0
instance.client.(*dynamicfake.FakeDynamicClient).PrependReactor(
"update",
"projectdeliveries",
func(k8stesting.Action) (bool, runtime.Object, error) {
if conflicts > 0 {
return false, nil, nil
}
conflicts++
return true, nil, apierrors.NewConflict(
schema.GroupResource{Group: projectDeliveryGVR.Group, Resource: projectDeliveryGVR.Resource},
"continuous",
errors.New("concurrent update"),
)
},
)
instance.reconcileAll(context.Background())
if conflicts != 1 {
t.Fatalf("conflicts = %d", conflicts)
}
if countWorkflows(t, instance, "build") != 0 {
t.Fatal("candidate created a build too early")
}
}
func testController(t *testing.T, delivery *unstructured.Unstructured, refs sourceReader, now *time.Time) *controller {
t.Helper()
objects := []runtime.Object{delivery}
for _, namespace := range []string{"delivery", "other"} {
for _, name := range []string{"nuthatch-delivery", "shared-delivery", "other-delivery", "qual-tests", "other-template", "other-qual"} {
objects = append(objects, testTemplate(namespace, name))
}
}
client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(
runtime.NewScheme(),
map[schema.GroupVersionResource]string{
projectDeliveryGVR: "ProjectDeliveryList",
workflowGVR: "WorkflowList",
templateGVR: "WorkflowTemplateList",
configMapGVR: "ConfigMapList",
},
objects...,
)
return &controller{
client: client,
source: refs,
now: func() time.Time { return *now },
config: controllerConfig{
pollInterval: time.Second,
bucketInterval: time.Minute,
requestTimeout: time.Second,
},
}
}
func testTemplate(namespace, name string) *unstructured.Unstructured {
return &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "argoproj.io/v1alpha1", "kind": "WorkflowTemplate",
"metadata": map[string]any{
"name": name, "namespace": namespace, "generation": int64(1),
"labels": map[string]any{"app.kubernetes.io/name": "project-delivery", "app.kubernetes.io/instance": "synthetic"},
},
"spec": map[string]any{"entrypoint": "build", "templates": []any{map[string]any{
"name": "build", "container": map[string]any{"image": "example.invalid/builder:v1"},
}}},
}}
}
func continuousDelivery() *unstructured.Unstructured {
return deliveryObject("continuous", map[string]any{"continuous": map[string]any{"ref": "refs/heads/main"}})
}
func pointDelivery(revisions ...string) *unstructured.Unstructured {
values := make([]any, len(revisions))
for index, revision := range revisions {
values[index] = revision
}
return deliveryObject("point", map[string]any{"point": map[string]any{"revisions": values}})
}
func deliveryObject(name string, sourceMode map[string]any) *unstructured.Unstructured {
source := map[string]any{"repository": "https://git.quorate.dev/quorate/nuthatch"}
for key, value := range sourceMode {
source[key] = value
}
return &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "delivery.quorate.dev/v1alpha1",
"kind": "ProjectDelivery",
"metadata": map[string]any{
"name": name,
"namespace": "delivery",
"uid": "11111111-2222-3333-4444-555555555555",
"generation": int64(1),
},
"spec": map[string]any{
"source": source,
"targets": []any{
map[string]any{"kind": "QuayOCI", "repository": "quorate/nuthatch"},
},
"workflowTemplateRef": map[string]any{"name": "nuthatch-delivery"},
},
}}
}
func setWorkflowStatus(t *testing.T, instance *controller, purpose, phase string, outputs map[string]string) {
t.Helper()
items, err := instance.client.Resource(workflowGVR).Namespace("delivery").List(context.Background(), metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
for index := range items.Items {
item := &items.Items[index]
if item.GetLabels()[purposeAnnotation] != purpose {
continue
}
status := map[string]any{"phase": phase}
if len(outputs) != 0 {
parameters := make([]any, 0, len(outputs))
for name, value := range outputs {
parameters = append(parameters, map[string]any{"name": name, "value": value})
}
status["nodes"] = map[string]any{
item.GetName(): map[string]any{"outputs": map[string]any{"parameters": parameters}},
}
}
item.Object["status"] = status
if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").UpdateStatus(context.Background(), item, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
return
}
t.Fatalf("workflow purpose %q not found", purpose)
}
func setNamedWorkflowStatus(t *testing.T, instance *controller, name, phase string, outputs map[string]string) {
t.Helper()
item, err := instance.client.Resource(workflowGVR).Namespace("delivery").Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
status := map[string]any{"phase": phase}
if len(outputs) != 0 {
parameters := make([]any, 0, len(outputs))
for outputName, value := range outputs {
parameters = append(parameters, map[string]any{"name": outputName, "value": value})
}
status["nodes"] = map[string]any{
item.GetName(): map[string]any{"outputs": map[string]any{"parameters": parameters}},
}
}
item.Object["status"] = status
if _, err := instance.client.Resource(workflowGVR).Namespace("delivery").UpdateStatus(context.Background(), item, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
}
func countWorkflows(t *testing.T, instance *controller, purpose string) int {
t.Helper()
items, err := instance.client.Resource(workflowGVR).Namespace("delivery").List(context.Background(), metav1.ListOptions{})
if err != nil {
t.Fatal(err)
}
count := 0
for _, item := range items.Items {
if item.GetLabels()[purposeAnnotation] == purpose {
count++
}
}
return count
}
func getDelivery(t *testing.T, instance *controller, name string) *unstructured.Unstructured {
t.Helper()
object, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
return object
}
func setRetry(t *testing.T, instance *controller, name, workflow string) {
t.Helper()
object := getDelivery(t, instance, name)
annotations := object.GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
}
annotations[retryAnnotation] = workflow
object.SetAnnotations(annotations)
if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
}
func workflowHasParameter(t *testing.T, workflow *unstructured.Unstructured, name string) bool {
t.Helper()
parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters")
if err != nil || !found {
t.Fatalf("read Workflow parameters: found=%t error=%v", found, err)
}
for _, item := range parameters {
parameter := item.(map[string]any)
if parameter["name"] == name {
return true
}
}
return false
}
func workflowParameter(t *testing.T, workflow *unstructured.Unstructured, name string) string {
t.Helper()
parameters, found, err := unstructured.NestedSlice(workflow.Object, "spec", "arguments", "parameters")
if err != nil || !found {
t.Fatalf("read Workflow parameters: found=%t error=%v", found, err)
}
for _, item := range parameters {
parameter := item.(map[string]any)
if parameter["name"] == name {
value, _ := parameter["value"].(string)
return value
}
}
t.Fatalf("Workflow parameter %q not found", name)
return ""
}
func workflowMutex(t *testing.T, workflow *unstructured.Unstructured) string {
t.Helper()
mutexes, found, err := unstructured.NestedSlice(workflow.Object, "spec", "synchronization", "mutexes")
if err != nil || !found || len(mutexes) != 1 {
t.Fatalf("read Workflow mutex: count=%d found=%t error=%v", len(mutexes), found, err)
}
mutex, ok := mutexes[0].(map[string]any)
if !ok {
t.Fatalf("Workflow mutex = %#v", mutexes[0])
}
name, _ := mutex["name"].(string)
return name
}

380
services/project-delivery/controller/forgejo.go Normal file
View file

@ -0,0 +1,380 @@
package main
import (
"bufio"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
)
type sourceReader interface {
Refs(context.Context, string) (map[string]string, error)
PathIdentity(context.Context, string, string, []string) (string, error)
}
type forgejoRouter struct {
routes map[string]sourceReader
}
type forgejoRoute struct {
authority string
source sourceReader
}
type forgejoClient struct {
baseURL *url.URL
http *http.Client
authorization string
}
func newForgejoRouter(routes ...forgejoRoute) (*forgejoRouter, error) {
normalized := make(map[string]sourceReader, len(routes))
for _, route := range routes {
parsed, err := url.Parse("https://" + route.authority)
if err != nil || parsed.Host == "" || parsed.Host != route.authority || parsed.Path != "" || parsed.User != nil {
return nil, fmt.Errorf("invalid Forgejo repository authority %q", route.authority)
}
key := strings.ToLower(parsed.Host)
if route.source == nil {
return nil, fmt.Errorf("Forgejo repository authority %q has no route", route.authority)
}
if _, found := normalized[key]; found {
return nil, fmt.Errorf("duplicate Forgejo repository authority %q", route.authority)
}
normalized[key] = route.source
}
if len(normalized) == 0 {
return nil, errors.New("no Forgejo repository routes configured")
}
return &forgejoRouter{routes: normalized}, nil
}
func (router *forgejoRouter) Refs(ctx context.Context, repository string) (map[string]string, error) {
authority, err := repositoryAuthority(repository)
if err != nil {
return nil, err
}
source, found := router.routes[authority]
if !found {
return nil, fmt.Errorf("no Forgejo route for repository authority %q", authority)
}
return source.Refs(ctx, repository)
}
func (router *forgejoRouter) PathIdentity(
ctx context.Context,
repository, revision string,
paths []string,
) (string, error) {
authority, err := repositoryAuthority(repository)
if err != nil {
return "", err
}
source, found := router.routes[authority]
if !found {
return "", fmt.Errorf("no Forgejo route for repository authority %q", authority)
}
return source.PathIdentity(ctx, repository, revision, paths)
}
func newForgejoClient(base string, client *http.Client, token string) (*forgejoClient, error) {
parsed, err := url.Parse(base)
if err != nil {
return nil, fmt.Errorf("parse Forgejo base URL: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return nil, errors.New("forgejo base URL must use HTTP or HTTPS")
}
if parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" {
return nil, errors.New("invalid Forgejo base URL")
}
parsed.Path = strings.TrimRight(parsed.Path, "/")
authorization := ""
if token != "" {
authorization = "Basic " + base64.StdEncoding.EncodeToString([]byte("oauth2:"+token))
}
return &forgejoClient{baseURL: parsed, http: client, authorization: authorization}, nil
}
func repositoryParts(repository string) (string, string, error) {
parsed, err := url.Parse(repository)
if err != nil {
return "", "", err
}
parts := strings.Split(strings.Trim(strings.TrimSuffix(parsed.EscapedPath(), ".git"), "/"), "/")
if len(parts) != 2 {
return "", "", errors.New("repository URL must contain owner and repository")
}
owner, err := url.PathUnescape(parts[0])
if err != nil {
return "", "", err
}
name, err := url.PathUnescape(parts[1])
if err != nil {
return "", "", err
}
if owner == "" || name == "" || strings.ContainsAny(owner+name, "\x00/\\") {
return "", "", errors.New("invalid repository path")
}
return owner, name, nil
}
func repositoryPath(repository string) (string, error) {
owner, name, err := repositoryParts(repository)
if err != nil {
return "", err
}
return url.PathEscape(owner) + "/" + url.PathEscape(name) + ".git", nil
}
func repositoryAuthority(repository string) (string, error) {
parsed, err := url.Parse(repository)
if err != nil {
return "", err
}
if parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" {
return "", errors.New("repository URL must be an HTTPS URL without credentials, query, or fragment")
}
if _, err := repositoryPath(repository); err != nil {
return "", err
}
return strings.ToLower(parsed.Host), nil
}
func (client *forgejoClient) Refs(ctx context.Context, repository string) (map[string]string, error) {
path, err := repositoryPath(repository)
if err != nil {
return nil, err
}
endpoint := *client.baseURL
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/" + path + "/info/refs"
query := endpoint.Query()
query.Set("service", "git-upload-pack")
endpoint.RawQuery = query.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/x-git-upload-pack-advertisement")
if client.authorization != "" {
req.Header.Set("Authorization", client.authorization)
}
response, err := client.http.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
return nil, fmt.Errorf("forgejo refs returned HTTP %d", response.StatusCode)
}
return parseAdvertisement(response.Body)
}
type treeEntry struct {
Path string `json:"path"`
Mode string `json:"mode"`
Type string `json:"type"`
SHA string `json:"sha"`
}
type treePage struct {
Entries []treeEntry `json:"tree"`
TotalCount int `json:"total_count"`
}
type pathEntry struct {
Found bool
Mode string
SHA string
}
func (client *forgejoClient) PathIdentity(
ctx context.Context,
repository, revision string,
paths []string,
) (string, error) {
owner, name, err := repositoryParts(repository)
if err != nil {
return "", err
}
ordered := append([]string(nil), paths...)
sort.Strings(ordered)
cache := map[string][]treeEntry{}
digest := sha256.New()
for _, path := range ordered {
entry, err := client.pathEntry(ctx, owner, name, revision, path, cache)
if err != nil {
return "", err
}
writeIdentity(digest, path)
if !entry.Found {
writeIdentity(digest, "missing")
continue
}
writeIdentity(digest, entry.Mode)
writeIdentity(digest, entry.SHA)
}
return "sha256:" + hex.EncodeToString(digest.Sum(nil)), nil
}
func (client *forgejoClient) pathEntry(
ctx context.Context,
owner, name, revision, path string,
cache map[string][]treeEntry,
) (pathEntry, error) {
tree := revision
parts := strings.Split(path, "/")
for index, part := range parts {
entries, found := cache[tree]
if !found {
var err error
entries, err = client.tree(ctx, owner, name, tree)
if err != nil {
return pathEntry{}, err
}
cache[tree] = entries
}
entry := treeEntry{}
for _, candidate := range entries {
if candidate.Path == part {
entry = candidate
break
}
}
if entry.Path == "" {
return pathEntry{}, nil
}
if index == len(parts)-1 {
return pathEntry{Found: true, Mode: entry.Mode, SHA: entry.SHA}, nil
}
if entry.Type != "tree" {
return pathEntry{}, nil
}
tree = entry.SHA
}
return pathEntry{}, nil
}
func (client *forgejoClient) tree(
ctx context.Context,
owner, name, revision string,
) ([]treeEntry, error) {
entries := []treeEntry{}
for page := 1; ; page++ {
endpoint := *client.baseURL
endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/api/v1/repos/" +
url.PathEscape(owner) + "/" + url.PathEscape(name) + "/git/trees/" + url.PathEscape(revision)
query := endpoint.Query()
query.Set("page", strconv.Itoa(page))
query.Set("per_page", "100")
endpoint.RawQuery = query.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
if client.authorization != "" {
req.Header.Set("Authorization", client.authorization)
}
response, err := client.http.Do(req)
if err != nil {
return nil, err
}
if response.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096))
_ = response.Body.Close()
return nil, fmt.Errorf("forgejo tree returned HTTP %d", response.StatusCode)
}
var result treePage
err = json.NewDecoder(response.Body).Decode(&result)
_ = response.Body.Close()
if err != nil {
return nil, fmt.Errorf("decode Forgejo tree: %w", err)
}
entries = append(entries, result.Entries...)
if len(result.Entries) < 100 || result.TotalCount > 0 && len(entries) >= result.TotalCount {
return entries, nil
}
}
}
func writeIdentity(output io.Writer, value string) {
var length [8]byte
binary.BigEndian.PutUint64(length[:], uint64(len(value)))
_, _ = output.Write(length[:])
_, _ = io.WriteString(output, value)
}
func parseAdvertisement(input io.Reader) (map[string]string, error) {
reader := bufio.NewReader(input)
refs := map[string]string{}
peeled := map[string]string{}
for {
header := make([]byte, 4)
if _, err := io.ReadFull(reader, header); err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, fmt.Errorf("read pkt-line header: %w", err)
}
length, err := strconv.ParseUint(string(header), 16, 16)
if err != nil {
return nil, errors.New("invalid pkt-line length")
}
if length == 0 || length == 1 || length == 2 {
continue
}
if length < 4 || length > 65520 {
return nil, errors.New("invalid pkt-line size")
}
payload := make([]byte, int(length)-4)
if _, err := io.ReadFull(reader, payload); err != nil {
return nil, fmt.Errorf("read pkt-line payload: %w", err)
}
line := strings.TrimSuffix(string(payload), "\n")
if strings.HasPrefix(line, "# service=") || line == "" {
continue
}
line = strings.SplitN(line, "\x00", 2)[0]
fields := strings.SplitN(line, " ", 2)
if len(fields) != 2 || !validObjectID(fields[0]) {
return nil, errors.New("invalid advertised ref")
}
name := strings.TrimSpace(fields[1])
if !strings.HasPrefix(name, "refs/") {
continue
}
if strings.HasSuffix(name, "^{}") {
peeled[strings.TrimSuffix(name, "^{}")] = fields[0]
continue
}
refs[name] = fields[0]
}
for name, revision := range peeled {
if _, found := refs[name]; found {
refs[name] = revision
}
}
return refs, nil
}
func validObjectID(value string) bool {
if len(value) != 40 && len(value) != 64 {
return false
}
decoded, err := hex.DecodeString(value)
return err == nil && len(decoded)*2 == len(value) && value == strings.ToLower(value)
}

255
services/project-delivery/controller/forgejo_test.go Normal file
View file

@ -0,0 +1,255 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"testing"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (roundTrip roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return roundTrip(request)
}
func TestParseAdvertisement(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
tagObject := "89abcdef0123456789abcdef0123456789abcdef"
content := append(packet("# service=git-upload-pack\n"), []byte("0000")...)
content = append(content, packet(revision+" refs/heads/main\x00multi_ack\n")...)
content = append(content, packet(tagObject+" refs/tags/v1\n")...)
content = append(content, packet(revision+" refs/tags/v1^{}\n")...)
content = append(content, []byte("0000")...)
refs, err := parseAdvertisement(bytes.NewReader(content))
if err != nil {
t.Fatal(err)
}
if len(refs) != 2 || refs["refs/heads/main"] != revision || refs["refs/tags/v1"] != revision {
t.Fatalf("refs = %#v", refs)
}
}
func TestParseAdvertisementKeepsLightweightTag(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
content := append(packet(revision+" refs/tags/v1\x00multi_ack\n"), []byte("0000")...)
refs, err := parseAdvertisement(bytes.NewReader(content))
if err != nil {
t.Fatal(err)
}
if len(refs) != 1 || refs["refs/tags/v1"] != revision {
t.Fatalf("refs = %#v", refs)
}
}
func TestRepositoryPath(t *testing.T) {
path, err := repositoryPath("https://git.quorate.dev/quorate/nuthatch.git")
if err != nil {
t.Fatal(err)
}
if path != "quorate/nuthatch.git" {
t.Fatalf("path = %q", path)
}
}
func TestForgejoClientAuthenticatesRefAdvertisement(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
if request.URL.RequestURI() != "/quorate/nuthatch.git/info/refs?service=git-upload-pack" {
t.Fatalf("request URI = %q", request.URL.RequestURI())
}
expected := "Basic " + base64.StdEncoding.EncodeToString([]byte("oauth2:test-token"))
if request.Header.Get("Authorization") != expected {
t.Fatal("missing source authorization")
}
content := append(packet(revision+" refs/heads/main\x00multi_ack\n"), []byte("0000")...)
return &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": {"application/x-git-upload-pack-advertisement"}},
Body: io.NopCloser(strings.NewReader(string(content))),
}, nil
})
client, err := newForgejoClient("https://forgejo.test", &http.Client{Transport: transport}, "test-token")
if err != nil {
t.Fatal(err)
}
refs, err := client.Refs(context.Background(), "https://git.quorate.dev/quorate/nuthatch")
if err != nil {
t.Fatal(err)
}
if refs["refs/heads/main"] != revision {
t.Fatalf("refs = %#v", refs)
}
}
func TestForgejoClientIdentifiesSelectedPathTree(t *testing.T) {
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
packages := "1111111111111111111111111111111111111111"
runner := "2222222222222222222222222222222222222222"
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
tree := request.URL.Path[strings.LastIndex(request.URL.Path, "/")+1:]
var body string
switch tree {
case first:
body = fmt.Sprintf(`{"tree":[{"path":"docs","mode":"040000","type":"tree","sha":"%s"},{"path":"packages","mode":"040000","type":"tree","sha":"%s"}],"page":1,"total_count":2}`, second, packages)
case second:
body = fmt.Sprintf(`{"tree":[{"path":"packages","mode":"040000","type":"tree","sha":"%s"}],"page":1,"total_count":1}`, packages)
case packages:
body = fmt.Sprintf(`{"tree":[{"path":"fsharp-runner","mode":"040000","type":"tree","sha":"%s"}],"page":1,"total_count":1}`, runner)
default:
t.Fatalf("unexpected tree %q", tree)
}
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body))}, nil
})
client, err := newForgejoClient("https://forgejo.test", &http.Client{Transport: transport}, "")
if err != nil {
t.Fatal(err)
}
firstIdentity, err := client.PathIdentity(context.Background(), "https://git.quorate.dev/ms2k/quorate", first, []string{"packages/fsharp-runner"})
if err != nil {
t.Fatal(err)
}
secondIdentity, err := client.PathIdentity(context.Background(), "https://git.quorate.dev/ms2k/quorate", second, []string{"packages/fsharp-runner"})
if err != nil {
t.Fatal(err)
}
if firstIdentity != secondIdentity {
t.Fatalf("identities differ: %q != %q", firstIdentity, secondIdentity)
}
}
func TestForgejoClientReadsAllTreePages(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
target := "89abcdef0123456789abcdef0123456789abcdef"
requests := 0
transport := roundTripFunc(func(request *http.Request) (*http.Response, error) {
requests++
page, _ := strconv.Atoi(request.URL.Query().Get("page"))
entries := make([]string, 0, 100)
if page == 1 {
for index := range 100 {
entries = append(entries, fmt.Sprintf(`{"path":"entry-%03d","mode":"100644","type":"blob","sha":"%s"}`, index, target))
}
} else if page == 2 {
entries = append(entries, fmt.Sprintf(`{"path":"selected","mode":"100755","type":"blob","sha":"%s"}`, target))
}
body := fmt.Sprintf(`{"tree":[%s],"page":%d,"total_count":101}`, strings.Join(entries, ","), page)
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body))}, nil
})
client, err := newForgejoClient("https://forgejo.test", &http.Client{Transport: transport}, "")
if err != nil {
t.Fatal(err)
}
if _, err := client.PathIdentity(context.Background(), "https://git.quorate.dev/ms2k/quorate", revision, []string{"selected"}); err != nil {
t.Fatal(err)
}
if requests != 2 {
t.Fatalf("tree requests = %d", requests)
}
}
func TestForgejoClientIncludesMissingPathsInIdentity(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
transport := roundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"tree":[],"page":1,"total_count":0}`))}, nil
})
client, err := newForgejoClient("https://forgejo.test", &http.Client{Transport: transport}, "")
if err != nil {
t.Fatal(err)
}
first, err := client.PathIdentity(context.Background(), "https://git.quorate.dev/ms2k/quorate", revision, []string{"missing-a"})
if err != nil {
t.Fatal(err)
}
second, err := client.PathIdentity(context.Background(), "https://git.quorate.dev/ms2k/quorate", revision, []string{"missing-b"})
if err != nil {
t.Fatal(err)
}
if first == second {
t.Fatal("different missing path scopes received the same identity")
}
}
func TestForgejoRouterKeepsCredentialsWithinAuthority(t *testing.T) {
revision := "0123456789abcdef0123456789abcdef01234567"
publicRequests := 0
privateRequests := 0
client := func(expectedToken string, requests *int) *http.Client {
return &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
(*requests)++
expected := "Basic " + base64.StdEncoding.EncodeToString([]byte("oauth2:"+expectedToken))
if request.Header.Get("Authorization") != expected {
t.Fatalf("authorization = %q", request.Header.Get("Authorization"))
}
content := append(packet(revision+" refs/heads/main\x00multi_ack\n"), []byte("0000")...)
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewReader(content)),
}, nil
})}
}
public, err := newForgejoClient("http://public.forgejo.test:3000", client("public-token", &publicRequests), "public-token")
if err != nil {
t.Fatal(err)
}
private, err := newForgejoClient("http://private.forgejo.test:3000", client("private-token", &privateRequests), "private-token")
if err != nil {
t.Fatal(err)
}
router, err := newForgejoRouter(
forgejoRoute{authority: "git.quorate.dev", source: public},
forgejoRoute{authority: "forge.quorate.internal", source: private},
)
if err != nil {
t.Fatal(err)
}
if _, err := router.Refs(context.Background(), "https://git.quorate.dev/ms2k/project"); err != nil {
t.Fatal(err)
}
if publicRequests != 1 || privateRequests != 0 {
t.Fatalf("requests public=%d private=%d", publicRequests, privateRequests)
}
if _, err := router.Refs(context.Background(), "https://forge.quorate.internal/quorate/cluster"); err != nil {
t.Fatal(err)
}
if publicRequests != 1 || privateRequests != 1 {
t.Fatalf("requests public=%d private=%d", publicRequests, privateRequests)
}
if _, err := router.Refs(context.Background(), "https://unknown.example/owner/repository"); err == nil {
t.Fatal("unknown authority received a Forgejo route")
}
if publicRequests != 1 || privateRequests != 1 {
t.Fatalf("unknown authority sent a request: public=%d private=%d", publicRequests, privateRequests)
}
}
func TestForgejoRouterRejectsDuplicateAuthority(t *testing.T) {
client, err := newForgejoClient("http://forgejo.test:3000", http.DefaultClient, "")
if err != nil {
t.Fatal(err)
}
if _, err := newForgejoRouter(
forgejoRoute{authority: "git.quorate.dev", source: client},
forgejoRoute{authority: "GIT.QUORATE.DEV", source: client},
); err == nil {
t.Fatal("duplicate repository authority was accepted")
}
}
func TestRepositoryAuthorityRejectsCredentialBearingURL(t *testing.T) {
if _, err := repositoryAuthority("https://oauth2:[email protected]/quorate/nuthatch"); err == nil {
t.Fatal("credential-bearing repository URL was accepted")
}
}
func packet(value string) []byte {
return []byte(fmt.Sprintf("%04x%s", len(value)+4, value))
}

49
services/project-delivery/controller/go.mod Normal file
View file

@ -0,0 +1,49 @@
module quorate.dev/services/project-delivery-controller
go 1.26.4
require (
k8s.io/apimachinery v0.35.3
k8s.io/client-go v0.35.3
sigs.k8s.io/yaml v1.6.0
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.12.2 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/time v0.9.0 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.35.3 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
)

131
services/project-delivery/controller/go.sum Normal file
View file

@ -0,0 +1,131 @@
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns=
github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY=
golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/api v0.35.3 h1:pA2fiBc6+N9PDf7SAiluKGEBuScsTzd2uYBkA5RzNWQ=
k8s.io/api v0.35.3/go.mod h1:9Y9tkBcFwKNq2sxwZTQh1Njh9qHl81D0As56tu42GA4=
k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8=
k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns=
k8s.io/client-go v0.35.3 h1:s1lZbpN4uI6IxeTM2cpdtrwHcSOBML1ODNTCCfsP1pg=
k8s.io/client-go v0.35.3/go.mod h1:RzoXkc0mzpWIDvBrRnD+VlfXP+lRzqQjCmKtiwZ8Q9c=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE=
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck=
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=
sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

83
services/project-delivery/controller/history_test.go Normal file
View file

@ -0,0 +1,83 @@
package main
import (
"context"
"errors"
"testing"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
dynamicfake "k8s.io/client-go/dynamic/fake"
k8stesting "k8s.io/client-go/testing"
)
func TestConsumedHistory(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
first := "0123456789abcdef0123456789abcdef01234567"
second := "89abcdef0123456789abcdef0123456789abcdef"
refs := &mutableRefs{values: map[string]string{"refs/heads/main": first}}
instance := testController(t, continuousDelivery(), refs, &now)
for _, revision := range []string{first, second} {
refs.values["refs/heads/main"] = revision
for range 3 {
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
}
setNamedWorkflowStatus(t, instance, workflowName("build", "11111111-2222-3333-4444-555555555555", "continuous", revision), "Failed", nil)
instance.reconcileAll(ctx)
}
name := workflowName("build", "11111111-2222-3333-4444-555555555555", "continuous", first)
if err := instance.client.Resource(workflowGVR).Namespace("delivery").Delete(ctx, name, metav1.DeleteOptions{}); err != nil {
t.Fatal(err)
}
refs.values["refs/heads/main"] = first
for range 4 {
instance.reconcileAll(ctx)
now = now.Add(time.Minute)
}
if count := countWorkflows(t, instance, "build"); count != 1 {
t.Fatalf("previously consumed failed identity was submitted again: %d Workflows", count)
}
}
func TestCaptureBeforeStatus(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
revision := "0123456789abcdef0123456789abcdef01234567"
instance := testController(t, pointDelivery(revision), &mutableRefs{values: map[string]string{"refs/heads/main": revision}}, &now)
client := instance.client.(*dynamicfake.FakeDynamicClient)
changed := false
client.PrependReactor("update", "projectdeliveries", func(action k8stesting.Action) (bool, runtime.Object, error) {
if changed || action.GetSubresource() != "status" {
return false, nil, nil
}
changed = true
raw, err := client.Tracker().Get(projectDeliveryGVR, "delivery", "point")
if err != nil {
t.Fatal(err)
}
object := raw.(*unstructured.Unstructured).DeepCopy()
unstructured.RemoveNestedField(object.Object, "spec", "source", "point")
if err := unstructured.SetNestedMap(object.Object, map[string]any{"ref": "refs/heads/main"}, "spec", "source", "continuous"); err != nil {
t.Fatal(err)
}
if err := client.Tracker().Update(projectDeliveryGVR, object, "delivery"); err != nil {
t.Fatal(err)
}
return true, nil, apierrors.NewConflict(projectDeliveryGVR.GroupResource(), "point", errors.New("synthetic source update"))
})
if err := instance.reconcileKey(ctx, "delivery", "point"); err != nil {
t.Fatal(err)
}
points, _, err := unstructured.NestedSlice(getDelivery(t, instance, "point").Object, "status", "points")
if err != nil {
t.Fatal(err)
}
if len(points) != 1 {
t.Fatalf("accepted Point execution disappeared after a source-mode conflict: %d records", len(points))
}
}

223
services/project-delivery/controller/main.go Normal file
View file

@ -0,0 +1,223 @@
package main
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"strings"
"syscall"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/dynamic"
coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Args[1:]); err != nil && !errors.Is(err, context.Canceled) {
slog.Error("project delivery", "error", err)
os.Exit(1)
}
}
func run(ctx context.Context, arguments []string) error {
if len(arguments) != 1 {
return errors.New("expected one command: run")
}
switch arguments[0] {
case "run":
return runController(ctx)
default:
return fmt.Errorf("unknown command %q", arguments[0])
}
}
func runController(ctx context.Context) error {
clientConfig, err := rest.InClusterConfig()
if err != nil {
return fmt.Errorf("load Kubernetes configuration: %w", err)
}
client, err := dynamic.NewForConfig(clientConfig)
if err != nil {
return fmt.Errorf("create Kubernetes client: %w", err)
}
forgejoHTTP, err := httpClient(os.Getenv("FORGEJO_CA_PATH"), 30*time.Second)
if err != nil {
return err
}
publicToken, err := secretFile(os.Getenv("FORGEJO_PUBLIC_TOKEN_PATH"))
if err != nil {
return err
}
privateToken, err := secretFile(os.Getenv("FORGEJO_PRIVATE_TOKEN_PATH"))
if err != nil {
return err
}
publicRefs, err := newForgejoClient(
valueOrDefault("FORGEJO_PUBLIC_URL", "http://forge-git-http.forgejo-public.svc.cluster.local:3000"),
forgejoHTTP,
publicToken,
)
if err != nil {
return err
}
privateRefs, err := newForgejoClient(
valueOrDefault("FORGEJO_PRIVATE_URL", "http://forgejo-http.forgejo.svc.cluster.local:3000"),
forgejoHTTP,
privateToken,
)
if err != nil {
return err
}
refs, err := newForgejoRouter(
forgejoRoute{authority: valueOrDefault("FORGEJO_PUBLIC_AUTHORITY", "git.quorate.dev"), source: publicRefs},
forgejoRoute{authority: valueOrDefault("FORGEJO_PRIVATE_AUTHORITY", "forge.quorate.internal"), source: privateRefs},
)
if err != nil {
return err
}
config, err := loadControllerConfig()
if err != nil {
return err
}
instance := controller{client: client, source: refs, now: time.Now, config: config}
coordination, err := coordinationv1.NewForConfig(clientConfig)
if err != nil {
return err
}
identity, err := os.Hostname()
if err != nil {
return err
}
namespace := os.Getenv("POD_NAMESPACE")
if namespace == "" {
return errors.New("POD_NAMESPACE is required")
}
return instance.lead(ctx, &resourcelock.LeaseLock{
LeaseMeta: metav1.ObjectMeta{Name: "project-delivery", Namespace: namespace},
Client: coordination,
LockConfig: resourcelock.ResourceLockConfig{Identity: identity},
})
}
func secretFile(path string) (string, error) {
if path == "" {
return "", nil
}
value, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read secret file: %w", err)
}
secret := strings.TrimSpace(string(value))
if secret == "" {
return "", errors.New("secret file is empty")
}
return secret, nil
}
func loadControllerConfig() (controllerConfig, error) {
pollInterval, err := durationFromEnvironment("POLL_INTERVAL", 20*time.Second)
if err != nil {
return controllerConfig{}, err
}
bucketInterval, err := durationFromEnvironment("OBSERVATION_BUCKET_INTERVAL", time.Minute)
if err != nil {
return controllerConfig{}, err
}
requestTimeout, err := durationFromEnvironment("REQUEST_TIMEOUT", 15*time.Second)
if err != nil {
return controllerConfig{}, err
}
return controllerConfig{
pollInterval: pollInterval,
bucketInterval: bucketInterval,
requestTimeout: requestTimeout,
templateQuiet: bucketInterval,
}, nil
}
func durationFromEnvironment(name string, fallback time.Duration) (time.Duration, error) {
value := os.Getenv(name)
if value == "" {
return fallback, nil
}
duration, err := time.ParseDuration(value)
if err != nil || duration <= 0 {
return 0, fmt.Errorf("%s must be a positive duration", name)
}
return duration, nil
}
func valueOrDefault(name, fallback string) string {
if value := os.Getenv(name); value != "" {
return value
}
return fallback
}
func httpClient(caPath string, timeout time.Duration) (*http.Client, error) {
transport := http.DefaultTransport.(*http.Transport).Clone()
if caPath != "" {
pem, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("read HTTP CA: %w", err)
}
roots, err := x509.SystemCertPool()
if err != nil {
return nil, fmt.Errorf("load system roots: %w", err)
}
if !roots.AppendCertsFromPEM(pem) {
return nil, errors.New("HTTP CA contains no certificate")
}
transport.TLSClientConfig = &tls.Config{RootCAs: roots}
}
return &http.Client{Transport: transport, Timeout: timeout}, nil
}
func (c *controller) lead(ctx context.Context, lock resourcelock.Interface) error {
electionCtx, cancel := context.WithCancel(ctx)
defer cancel()
controllerErr := make(chan error, 1)
elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
Lock: lock,
LeaseDuration: 30 * time.Second,
RenewDeadline: 20 * time.Second,
RetryPeriod: 5 * time.Second,
Name: "project-delivery",
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(leaderCtx context.Context) {
controllerErr <- c.run(leaderCtx)
cancel()
},
OnStoppedLeading: func() {
if ctx.Err() == nil {
select {
case controllerErr <- errors.New("leadership lost"):
default:
}
}
},
},
})
if err != nil {
return err
}
elector.Run(electionCtx)
select {
case err := <-controllerErr:
return err
default:
return ctx.Err()
}
}

37
services/project-delivery/controller/mode_test.go Normal file
View file

@ -0,0 +1,37 @@
package main
import (
"context"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func TestOldModeCompletes(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
revision := "0123456789abcdef0123456789abcdef01234567"
instance := testController(t, pointDelivery(revision), &mutableRefs{values: map[string]string{"refs/heads/main": revision}}, &now)
instance.reconcileAll(ctx)
workflow := workflowName("build", "11111111-2222-3333-4444-555555555555", "", revision)
setNamedWorkflowStatus(t, instance, workflow, "Succeeded", nil)
object := getDelivery(t, instance, "point")
unstructured.RemoveNestedField(object.Object, "spec", "source", "point")
if err := unstructured.SetNestedMap(object.Object, map[string]any{"ref": "refs/heads/main"}, "spec", "source", "continuous"); err != nil {
t.Fatal(err)
}
if _, err := instance.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(ctx, object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
instance.reconcileAll(ctx)
object = getDelivery(t, instance, "point")
points, _, err := unstructured.NestedSlice(object.Object, "status", "points")
if err != nil {
t.Fatal(err)
}
if phase := points[0].(map[string]any)["phase"]; phase != "Succeeded" {
t.Fatalf("old Point attempt remains %v after its Workflow succeeded", phase)
}
}

290
services/project-delivery/controller/qualification.go Normal file
View file

@ -0,0 +1,290 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"reflect"
"slices"
"strings"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/retry"
)
type qualChoice struct {
namespace string
name string
kind string
index int
}
func (choice qualChoice) key() string {
return choice.namespace + "/" + choice.name + "/" + choice.kind
}
func (c *controller) qualifyAll(ctx context.Context) error {
items, err := c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
return err
}
var failures []error
for _, item := range items.Items {
if err := c.qualifyKey(ctx, item.GetNamespace(), item.GetName(), c.prepareQual); err != nil {
failures = append(failures, err)
}
}
items, err = c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
return errors.Join(append(failures, err)...)
}
var choices []qualChoice
for _, item := range items.Items {
delivery, err := decodeProjectDelivery(&item)
if err != nil {
return err
}
q := delivery.Status.Qualification
if q == nil {
continue
}
choice := qualChoice{namespace: item.GetNamespace(), name: item.GetName()}
for index, attempt := range q.Attempts {
if attempt.Phase == "Pending" || terminalPhase(attempt.Phase) {
continue
}
err := c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error {
return c.advanceQual(ctx, d, index)
})
return errors.Join(append(failures, err)...)
}
if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil {
continue
}
if index := slices.IndexFunc(q.Attempts, func(a qualAttempt) bool { return a.Phase == "Pending" }); index >= 0 {
choice.kind, choice.index = "Manual", index
choices = append(choices, choice)
}
due, err := qualDue(&delivery, c.now())
if err != nil {
failures = append(failures, err)
} else if due {
choice.kind = "Auto"
choices = append(choices, choice)
}
}
if len(choices) == 0 {
return errors.Join(failures...)
}
slices.SortFunc(choices, func(a, b qualChoice) int { return strings.Compare(a.key(), b.key()) })
index := slices.IndexFunc(choices, func(choice qualChoice) bool { return choice.key() > c.qualCursor })
if index < 0 {
index = 0
}
choice := choices[index]
err = c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error {
q := d.Status.Qualification
if choice.kind == "Manual" {
q.Attempts[choice.index].Phase = "Reserved"
return nil
}
due, err := qualDue(d, c.now())
if err != nil || !due {
return err
}
build := len(q.Builds)
q.Attempts = append(q.Attempts, qualAttempt{
Kind: "Auto", Build: build, Phase: "Reserved",
WorkflowName: workflowName("qual", d.Metadata.UID, "Auto", q.Builds[build-1].WorkflowName),
Template: d.Spec.Qualification.WorkflowTemplateRef.Name,
})
return nil
})
if err == nil {
c.qualCursor = choice.key()
}
return errors.Join(append(failures, err)...)
}
func (c *controller) qualifyKey(ctx context.Context, namespace, name string, action func(context.Context, *projectDelivery) error) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
object, err := c.client.Resource(projectDeliveryGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
delivery, err := decodeProjectDelivery(object)
if err != nil {
return err
}
before, _ := toMap(delivery.Status)
actionErr := action(ctx, &delivery)
if delivery.Status.Qualification != nil {
delivery.Status.Qualification.Message = ""
if actionErr != nil {
delivery.Status.Qualification.Message = actionErr.Error()
}
}
after, _ := toMap(delivery.Status)
if !reflect.DeepEqual(before, after) {
if err := c.updateStatus(ctx, object, &delivery); err != nil {
return err
}
}
return actionErr
})
}
func (c *controller) prepareQual(ctx context.Context, delivery *projectDelivery) error {
if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil {
return nil
}
if delivery.Status.Qualification == nil {
delivery.Status.Qualification = &qualStatus{}
}
builds := make([]*attemptStatus, 0, len(delivery.Status.Points)+1)
if status := delivery.Status.Continuous; status != nil && status.Attempt != nil {
builds = append(builds, status.Attempt)
}
for i := range delivery.Status.Points {
builds = append(builds, &delivery.Status.Points[i])
}
for _, build := range builds {
if build.Phase != "Succeeded" {
continue
}
if len(build.Input) == 0 {
state, err := c.workflow(ctx, delivery.Metadata.Namespace, build.WorkflowName)
if apierrors.IsNotFound(err) {
continue
}
if err != nil {
return err
}
build.Input = state.Input
}
observeQualBuild(delivery, build)
}
annotation := delivery.Metadata.Annotations[qualAnnotation]
if annotation == "" {
return nil
}
name, request, found := strings.Cut(annotation, "/")
if !found || name == "" || request == "" {
return fmt.Errorf("qualification request must be <build-workflow-name>/<request-id>")
}
q := delivery.Status.Qualification
for _, attempt := range q.Attempts {
if attempt.Kind == "Manual" && attempt.RequestID == request {
return nil
}
}
index := slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name })
if index < 0 {
state, err := c.workflow(ctx, delivery.Metadata.Namespace, name)
if err != nil {
return err
}
result, found := state.Outputs["delivery-result"]
if state.OwnerUID != delivery.Metadata.UID || state.Phase != "Succeeded" || !found {
return fmt.Errorf("Workflow %q is not a successful build of this ProjectDelivery with a delivery-result", name)
}
observeQualBuild(delivery, &attemptStatus{WorkflowName: name, Phase: state.Phase, Input: state.Input, Result: opaqueResult(result)})
index = slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name })
if index < 0 {
return fmt.Errorf("Workflow %q has no recorded build inputs", name)
}
}
q.Attempts = append(q.Attempts, qualAttempt{
Kind: "Manual", RequestID: request, Build: index + 1, Phase: "Pending",
WorkflowName: workflowName("qual", delivery.Metadata.UID, "Manual", request),
Template: delivery.Spec.Qualification.WorkflowTemplateRef.Name,
})
return nil
}
func observeQualBuild(delivery *projectDelivery, build *attemptStatus) {
if delivery.Spec.Qualification == nil || build.Phase != "Succeeded" || len(build.Input) == 0 || len(build.Result) == 0 {
return
}
if delivery.Status.Qualification == nil {
delivery.Status.Qualification = &qualStatus{}
}
q := delivery.Status.Qualification
if slices.ContainsFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == build.WorkflowName }) {
return
}
input := maps.Clone(build.Input)
input["delivery-result"] = string(build.Result)
q.Builds = append(q.Builds, qualBuild{WorkflowName: build.WorkflowName, Input: input})
}
func qualDue(delivery *projectDelivery, now time.Time) (bool, error) {
spec := delivery.Spec.Qualification
q := delivery.Status.Qualification
if spec == nil || spec.Mode != "OnBuild" || q == nil || len(q.Builds) <= q.Covered {
return false, nil
}
for _, attempt := range q.Attempts {
if attempt.Kind == "Auto" && attempt.Build == len(q.Builds) {
return false, nil
}
}
var interval time.Duration
if spec.Interval != "" {
value, err := time.ParseDuration(spec.Interval)
if err != nil || value < 0 {
return false, fmt.Errorf("invalid qualification interval %q", spec.Interval)
}
interval = value
}
return q.LastStart == nil || !now.Before(q.LastStart.Add(interval)), nil
}
func (c *controller) advanceQual(ctx context.Context, delivery *projectDelivery, index int) error {
q := delivery.Status.Qualification
attempt := &q.Attempts[index]
if attempt.Phase == "Reserved" {
if err := c.ensureWorkflow(ctx, delivery, attempt.WorkflowName, attempt.Template, "qualification", q.Builds[attempt.Build-1].Input, nil); err != nil {
if errors.Is(err, templatePending) {
return nil
}
return err
}
}
state, err := c.workflow(ctx, delivery.Metadata.Namespace, attempt.WorkflowName)
if apierrors.IsNotFound(err) && attempt.Phase != "Reserved" {
attempt.Phase = "Error"
return nil
}
if err != nil {
return err
}
if attempt.StartedAt == nil && state.StartedAt != nil {
attempt.StartedAt = state.StartedAt
q.LastStart = state.StartedAt
q.Covered = max(q.Covered, attempt.Build)
}
attempt.Phase = state.Phase
if attempt.Phase == "" || attempt.Phase == "Pending" {
attempt.Phase = "Queued"
}
if result, found := state.Outputs["qualification-result"]; found {
attempt.Result = opaqueResult(result)
}
return nil
}
func opaqueResult(value string) []byte {
if json.Valid([]byte(value)) {
return []byte(value)
}
result, _ := json.Marshal(value)
return result
}

326
services/project-delivery/controller/qualification_test.go Normal file
View file

@ -0,0 +1,326 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
dynamicfake "k8s.io/client-go/dynamic/fake"
k8stesting "k8s.io/client-go/testing"
)
func TestQualManual(t *testing.T) {
c, now := qualFixture(t, "")
build := publishQual(t, c, "point", strings.Repeat("1", 40))
stepQual(t, c)
if len(readQual(t, c, "point").Attempts) != 0 {
t.Fatal("omitted mode started automatic qualification")
}
requestQual(t, c, "point", build+"/first")
stepQual(t, c)
first := readQual(t, c, "point").Attempts[0]
if first.Phase != "Reserved" || countWorkflows(t, c, "qualification") != 0 {
t.Fatal("reservation was not persisted before submission")
}
stepQual(t, c)
markQual(t, c, "delivery", first.WorkflowName, "Running", now, nil)
stepQual(t, c)
for _, id := range []string{"second", "third"} {
requestQual(t, c, "point", build+"/"+id)
stepQual(t, c)
}
q := readQual(t, c, "point")
if len(q.Attempts) != 3 || q.Attempts[1].Phase != "Pending" || q.Attempts[2].Phase != "Pending" {
t.Fatalf("accepted requests = %#v", q.Attempts)
}
requestQual(t, c, "point", build+"/first")
stepQual(t, c)
if len(readQual(t, c, "point").Attempts) != 3 || countWorkflows(t, c, "qualification") != 1 {
t.Fatal("reapplying a consumed request created another run")
}
markQual(t, c, "delivery", first.WorkflowName, "Succeeded", now, map[string]string{"qualification-result": `{"report":"synthetic://reports/first"}`})
stepQual(t, c)
stepQual(t, c)
stepQual(t, c)
if countWorkflows(t, c, "qualification") != 2 {
t.Fatal("pending manual request did not advance after completion")
}
q = readQual(t, c, "point")
if string(q.Attempts[0].Result) != `{"report":"synthetic://reports/first"}` {
t.Fatalf("qualification result = %s", q.Attempts[0].Result)
}
}
func TestQualInterval(t *testing.T) {
c, now := qualFixture(t, "OnBuild")
firstBuild := publishQual(t, c, "point", strings.Repeat("1", 40))
stepQual(t, c)
stepQual(t, c)
first := readQual(t, c, "point").Attempts[0]
markQual(t, c, "delivery", first.WorkflowName, "Running", now, nil)
stepQual(t, c)
publishQual(t, c, "point", strings.Repeat("2", 40))
thirdBuild := publishQual(t, c, "point", strings.Repeat("3", 40))
markQual(t, c, "delivery", first.WorkflowName, "Failed", now, nil)
stepQual(t, c)
*now = now.Add(167 * time.Hour)
stepQual(t, c)
if len(readQual(t, c, "point").Attempts) != 1 {
t.Fatal("automatic qualification ignored the interval after failure")
}
*now = now.Add(time.Hour)
stepQual(t, c)
q := readQual(t, c, "point")
if len(q.Attempts) != 2 || q.Builds[q.Attempts[1].Build-1].WorkflowName != thirdBuild {
t.Fatalf("expiry did not select latest build: %#v", q.Attempts)
}
stepQual(t, c)
second := q.Attempts[1]
markQual(t, c, "delivery", second.WorkflowName, "Succeeded", now, nil)
stepQual(t, c)
requestQual(t, c, "point", firstBuild+"/old-build")
stepQual(t, c)
stepQual(t, c)
q = readQual(t, c, "point")
manual := q.Attempts[2]
if manual.Kind != "Manual" || manual.Build != 1 {
t.Fatalf("manual selection = %#v", manual)
}
*now = now.Add(time.Hour)
markQual(t, c, "delivery", manual.WorkflowName, "Succeeded", now, nil)
stepQual(t, c)
q = readQual(t, c, "point")
if q.Covered != 3 || !q.LastStart.Equal(*now) {
t.Fatalf("manual start accounting = %#v", q)
}
*now = now.Add(168 * time.Hour)
stepQual(t, c)
if len(readQual(t, c, "point").Attempts) != 3 {
t.Fatal("expiry without a new build repeated qualification")
}
}
func TestQualSnapshot(t *testing.T) {
c, _ := qualFixture(t, "OnBuild")
build := publishQual(t, c, "point", strings.Repeat("1", 40))
original := readQual(t, c, "point").Builds[0].Input
object := getDelivery(t, c, "point")
spec := object.Object["spec"].(map[string]any)
spec["source"].(map[string]any)["repository"] = "https://example.test/other/repo"
spec["targets"] = []any{map[string]any{"kind": "Other", "repository": "changed"}}
if _, err := c.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
if err := c.client.Resource(workflowGVR).Namespace("delivery").Delete(context.Background(), build, metav1.DeleteOptions{}); err != nil {
t.Fatal(err)
}
stepQual(t, c)
object = getDelivery(t, c, "point")
object.Object["spec"].(map[string]any)["qualification"].(map[string]any)["workflowTemplateRef"] = map[string]any{"name": "other-template"}
if _, err := c.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
stepQual(t, c)
attempt := readQual(t, c, "point").Attempts[0]
workflow, err := c.client.Resource(workflowGVR).Namespace("delivery").Get(context.Background(), attempt.WorkflowName, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
template := workflow.GetAnnotations()[templateKey]
if template != "qual-tests" {
t.Fatalf("reserved template changed to %q", template)
}
for key, value := range original {
if got := workflowParameter(t, workflow, key); got != value {
t.Fatalf("parameter %s = %q, want %q", key, got, value)
}
}
if workflowHasParameter(t, workflow, "image-digest") {
t.Fatal("controller interpreted the publication result")
}
}
func TestQualReservation(t *testing.T) {
c, _ := qualFixture(t, "OnBuild")
publishQual(t, c, "point", strings.Repeat("1", 40))
fail := true
c.client.(*dynamicfake.FakeDynamicClient).PrependReactor("update", "projectdeliveries", func(action k8stesting.Action) (bool, runtime.Object, error) {
if fail {
fail = false
return true, nil, errors.New("synthetic status write failure")
}
return false, nil, nil
})
if err := c.qualifyAll(context.Background()); err == nil {
t.Fatal("expected reservation write failure")
}
if countWorkflows(t, c, "qualification") != 0 || len(readQual(t, c, "point").Attempts) != 0 {
t.Fatal("unpersisted reservation created a Workflow")
}
stepQual(t, c)
restarted := *c
restarted.qualCursor = ""
conflicts := 0
c.client.(*dynamicfake.FakeDynamicClient).PrependReactor("update", "projectdeliveries", func(action k8stesting.Action) (bool, runtime.Object, error) {
object := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured)
attempts, _, _ := unstructured.NestedSlice(object.Object, "status", "qualification", "attempts")
queued := len(attempts) != 0 && attempts[0].(map[string]any)["phase"] == "Queued"
if conflicts == 0 && queued {
conflicts++
return true, nil, apierrors.NewConflict(projectDeliveryGVR.GroupResource(), "point", errors.New("synthetic conflict"))
}
return false, nil, nil
})
stepQual(t, &restarted)
stepQual(t, &restarted)
if conflicts != 1 || countWorkflows(t, c, "qualification") != 1 {
t.Fatalf("reservation recovery: conflicts=%d workflows=%d", conflicts, countWorkflows(t, c, "qualification"))
}
}
func TestQualGlobalCap(t *testing.T) {
c, now := qualFixture(t, "OnBuild")
second := getDelivery(t, c, "point").DeepCopy()
second.SetNamespace("other")
second.SetUID("other-delivery")
second.SetResourceVersion("")
second.Object["spec"].(map[string]any)["qualification"].(map[string]any)["workflowTemplateRef"] = map[string]any{"name": "other-qual"}
if _, err := c.client.Resource(projectDeliveryGVR).Namespace("other").Create(context.Background(), second, metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
publishQual(t, c, "point", strings.Repeat("1", 40))
other, err := decodeProjectDelivery(second)
if err != nil {
t.Fatal(err)
}
input := map[string]string{"source-repository": "https://example.test/repo/other", "source-ref": "", "source-revision": strings.Repeat("2", 40), "targets": "[{}]"}
observeQualBuild(&other, &attemptStatus{WorkflowName: "other-build", Phase: "Succeeded", Input: input, Result: json.RawMessage(`{"image":"other"}`)})
if err := c.updateStatus(context.Background(), second, &other); err != nil {
t.Fatal(err)
}
stepQual(t, c)
stepQual(t, c)
first := readQual(t, c, "point").Attempts[0]
markQual(t, c, "delivery", first.WorkflowName, "Running", now, nil)
stepQual(t, c)
objects, err := c.client.Resource(workflowGVR).Namespace(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{LabelSelector: purposeAnnotation + "=qualification"})
if err != nil || len(objects.Items) != 1 {
t.Fatalf("global Workflows = %d, error=%v", len(objects.Items), err)
}
publishQual(t, c, "point", strings.Repeat("3", 40))
markQual(t, c, "delivery", first.WorkflowName, "Succeeded", now, nil)
stepQual(t, c)
*now = now.Add(168 * time.Hour)
stepQual(t, c)
stepQual(t, c)
objects, err = c.client.Resource(workflowGVR).Namespace("other").List(context.Background(), metav1.ListOptions{LabelSelector: purposeAnnotation + "=qualification"})
if err != nil || len(objects.Items) != 1 {
t.Fatalf("waiting namespace did not advance: count=%d error=%v", len(objects.Items), err)
}
}
func TestQualPolicyChange(t *testing.T) {
c, _ := qualFixture(t, "OnBuild")
publishQual(t, c, "point", strings.Repeat("1", 40))
object := getDelivery(t, c, "point")
spec := object.Object["spec"].(map[string]any)["qualification"].(map[string]any)
spec["mode"] = "ManualOnly"
if _, err := c.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
stepQual(t, c)
if len(readQual(t, c, "point").Attempts) != 0 {
t.Fatal("ManualOnly reserved an automatic candidate")
}
object = getDelivery(t, c, "point")
object.Object["spec"].(map[string]any)["qualification"].(map[string]any)["mode"] = "OnBuild"
if _, err := c.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
stepQual(t, c)
if readQual(t, c, "point").Attempts[0].Phase != "Reserved" {
t.Fatal("OnBuild did not reconsider the waiting candidate")
}
}
func qualFixture(t *testing.T, mode string) (*controller, *time.Time) {
t.Helper()
now := time.Date(2026, time.September, 11, 0, 0, 0, 0, time.UTC)
object := pointDelivery(strings.Repeat("1", 40))
object.Object["spec"].(map[string]any)["qualification"] = map[string]any{
"workflowTemplateRef": map[string]any{"name": "qual-tests"}, "mode": mode, "interval": "168h",
}
return testController(t, object, &mutableRefs{}, &now), &now
}
func publishQual(t *testing.T, c *controller, name, revision string) string {
t.Helper()
object := getDelivery(t, c, name)
d, err := decodeProjectDelivery(object)
if err != nil {
t.Fatal(err)
}
attempt := attemptStatus{Revision: revision, Phase: "Pending"}
if err := c.advanceAttempt(context.Background(), &d, &attempt); err != nil {
t.Fatal(err)
}
result := fmt.Sprintf(`{"sourceRevision":%q,"targets":[{"reference":%q}]}`, revision, "synthetic/image@sha256:"+strings.Repeat("a", 64))
setNamedWorkflowStatus(t, c, attempt.WorkflowName, "Succeeded", map[string]string{"delivery-result": result})
if err := c.advanceAttempt(context.Background(), &d, &attempt); err != nil {
t.Fatal(err)
}
d.Status.Points = append(d.Status.Points, attempt)
if err := c.updateStatus(context.Background(), object, &d); err != nil {
t.Fatal(err)
}
return attempt.WorkflowName
}
func stepQual(t *testing.T, c *controller) {
t.Helper()
if err := c.qualifyAll(context.Background()); err != nil {
t.Fatal(err)
}
}
func readQual(t *testing.T, c *controller, name string) *qualStatus {
t.Helper()
d, err := decodeProjectDelivery(getDelivery(t, c, name))
if err != nil {
t.Fatal(err)
}
return d.Status.Qualification
}
func requestQual(t *testing.T, c *controller, name, request string) {
t.Helper()
object := getDelivery(t, c, name)
object.SetAnnotations(map[string]string{qualAnnotation: request})
if _, err := c.client.Resource(projectDeliveryGVR).Namespace("delivery").Update(context.Background(), object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
}
func markQual(t *testing.T, c *controller, namespace, name, phase string, started *time.Time, outputs map[string]string) {
t.Helper()
object, err := c.client.Resource(workflowGVR).Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
params := []any{}
for name, value := range outputs {
params = append(params, map[string]any{"name": name, "value": value})
}
object.Object["status"] = map[string]any{"phase": phase, "startedAt": started.Format(time.RFC3339), "outputs": map[string]any{"parameters": params}}
if _, err := c.client.Resource(workflowGVR).Namespace(namespace).UpdateStatus(context.Background(), object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
}

97
services/project-delivery/controller/records.go Normal file
View file

@ -0,0 +1,97 @@
package main
import (
"context"
"encoding/json"
"slices"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
)
const recordKey = "delivery.quorate.dev/record"
func (c *controller) releaseRecords(ctx context.Context, delivery *projectDelivery) error {
known := make(map[string]bool)
for _, attempt := range delivery.Status.Points {
known[attempt.WorkflowName] = true
}
if status := delivery.Status.Continuous; status != nil && status.Attempt != nil {
known[status.Attempt.WorkflowName] = true
if status.Attempt.History != nil {
for _, name := range status.Attempt.History.Workflows {
known[name] = true
}
}
}
if status := delivery.Status.Qualification; status != nil {
for _, attempt := range status.Attempts {
if attempt.Phase != "Pending" && attempt.Phase != "Reserved" {
known[attempt.WorkflowName] = true
}
}
}
items, err := c.client.Resource(workflowGVR).Namespace(delivery.Metadata.Namespace).List(ctx, metav1.ListOptions{LabelSelector: recordKey + "=pending"})
if err != nil {
return err
}
for _, item := range items.Items {
if known[item.GetName()] && ownedBy(&item, delivery.Metadata.UID) {
if err := c.clearRecord(ctx, &item); err != nil {
return err
}
}
}
return nil
}
func (c *controller) releaseOrphans(ctx context.Context) error {
items, err := c.client.Resource(workflowGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{LabelSelector: recordKey + "=pending"})
if err != nil {
return err
}
for _, item := range items.Items {
for _, owner := range item.GetOwnerReferences() {
if owner.Kind != "ProjectDelivery" || owner.APIVersion != "delivery.quorate.dev/v1alpha1" {
continue
}
parent, err := c.client.Resource(projectDeliveryGVR).Namespace(item.GetNamespace()).Get(ctx, owner.Name, metav1.GetOptions{})
if apierrors.IsNotFound(err) || err == nil && (parent.GetUID() != owner.UID || parent.GetDeletionTimestamp() != nil) {
if err := c.clearRecord(ctx, &item); err != nil {
return err
}
} else if err != nil {
return err
}
}
}
return nil
}
func ownedBy(workflow *unstructured.Unstructured, uid string) bool {
return slices.ContainsFunc(workflow.GetOwnerReferences(), func(owner metav1.OwnerReference) bool {
return owner.Kind == "ProjectDelivery" && owner.APIVersion == "delivery.quorate.dev/v1alpha1" && string(owner.UID) == uid
})
}
func (c *controller) clearRecord(ctx context.Context, workflow *unstructured.Unstructured) error {
finalizers := slices.DeleteFunc(slices.Clone(workflow.GetFinalizers()), func(name string) bool { return name == recordKey })
if finalizers == nil {
finalizers = []string{}
}
patch, err := json.Marshal(map[string]any{"metadata": map[string]any{
"resourceVersion": workflow.GetResourceVersion(),
"finalizers": finalizers,
"labels": map[string]any{recordKey: nil},
}})
if err != nil {
return err
}
_, err = c.client.Resource(workflowGVR).Namespace(workflow.GetNamespace()).Patch(ctx, workflow.GetName(), types.MergePatchType, patch, metav1.PatchOptions{})
if apierrors.IsNotFound(err) {
return nil
}
return err
}

112
services/project-delivery/controller/records_test.go Normal file
View file

@ -0,0 +1,112 @@
package main
import (
"context"
"errors"
"reflect"
"slices"
"testing"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
dynamicfake "k8s.io/client-go/dynamic/fake"
k8stesting "k8s.io/client-go/testing"
)
func TestRecordRelease(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
object := pointDelivery("0123456789abcdef0123456789abcdef01234567")
c := testController(t, object, &mutableRefs{}, &now)
d, err := decodeProjectDelivery(object)
if err != nil {
t.Fatal(err)
}
if err := c.ensureWorkflow(ctx, &d, "accepted", "nuthatch-delivery", "build", nil, nil); err != nil {
t.Fatal(err)
}
workflow, err := c.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, "accepted", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
gc := "workflows.argoproj.io/artifact-gc"
workflow.SetFinalizers([]string{recordKey, gc})
if _, err := c.client.Resource(workflowGVR).Namespace("delivery").Update(ctx, workflow, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
d.Status.Points = []attemptStatus{{WorkflowName: "accepted", Phase: "Pending"}}
client := c.client.(*dynamicfake.FakeDynamicClient)
fail := true
client.PrependReactor("update", "projectdeliveries", func(action k8stesting.Action) (bool, runtime.Object, error) {
if fail && action.GetSubresource() == "status" {
return true, nil, apierrors.NewConflict(projectDeliveryGVR.GroupResource(), "point", errors.New("synthetic conflict"))
}
return false, nil, nil
})
if err := c.updateStatus(ctx, object, &d); !apierrors.IsConflict(err) {
t.Fatalf("status conflict = %v", err)
}
workflow, err = c.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, "accepted", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if !slices.Contains(workflow.GetFinalizers(), recordKey) {
t.Fatal("checkpoint finalizer was removed before status persistence")
}
fail = false
if err := c.updateStatus(ctx, object, &d); err != nil {
t.Fatal(err)
}
workflow, err = c.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, "accepted", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(workflow.GetFinalizers(), []string{gc}) || workflow.GetLabels()[recordKey] != "" {
t.Fatalf("remaining checkpoint: finalizers=%v labels=%v", workflow.GetFinalizers(), workflow.GetLabels())
}
}
func TestRecordOrphans(t *testing.T) {
for _, state := range []string{"missing", "deleting", "replaced"} {
t.Run(state, func(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
object := pointDelivery("0123456789abcdef0123456789abcdef01234567")
c := testController(t, object, &mutableRefs{}, &now)
d, err := decodeProjectDelivery(object)
if err != nil {
t.Fatal(err)
}
if err := c.ensureWorkflow(ctx, &d, "accepted", "nuthatch-delivery", "build", nil, nil); err != nil {
t.Fatal(err)
}
resource := c.client.Resource(projectDeliveryGVR).Namespace("delivery")
if state == "missing" {
if err := resource.Delete(ctx, "point", metav1.DeleteOptions{}); err != nil {
t.Fatal(err)
}
} else {
if state == "deleting" {
object.SetDeletionTimestamp(&metav1.Time{Time: now})
} else {
object.SetUID("replacement")
}
if _, err := resource.Update(ctx, object, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
}
if err := c.releaseOrphans(ctx); err != nil {
t.Fatal(err)
}
workflow, err := c.client.Resource(workflowGVR).Namespace("delivery").Get(ctx, "accepted", metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if slices.Contains(workflow.GetFinalizers(), recordKey) {
t.Fatal("deleted owner retained a checkpoint finalizer")
}
})
}
}

109
services/project-delivery/controller/recovery.go Normal file
View file

@ -0,0 +1,109 @@
package main
import (
"context"
"slices"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func (c *controller) recoverBuilds(ctx context.Context, delivery *projectDelivery) (bool, error) {
known := make(map[string]bool)
for _, attempt := range delivery.Status.Points {
known[attempt.WorkflowName] = true
}
continuous := delivery.Status.Continuous
if continuous != nil && continuous.Attempt != nil {
known[continuous.Attempt.WorkflowName] = true
if continuous.Attempt.History != nil {
for _, name := range continuous.Attempt.History.Workflows {
known[name] = true
}
}
}
items, err := c.client.Resource(workflowGVR).Namespace(delivery.Metadata.Namespace).List(ctx, metav1.ListOptions{LabelSelector: purposeAnnotation + "=build"})
if err != nil {
return false, err
}
slices.SortFunc(items.Items, func(a, b unstructured.Unstructured) int {
left, right := a.GetCreationTimestamp(), b.GetCreationTimestamp()
if order := left.Time.Compare(right.Time); order != 0 {
return order
}
return strings.Compare(a.GetName(), b.GetName())
})
recovered := false
if continuous != nil && continuous.Attempt != nil && continuous.Attempt.CreatedAt == nil {
for _, item := range items.Items {
if item.GetName() == continuous.Attempt.WorkflowName && ownedBy(&item, delivery.Metadata.UID) {
created := item.GetCreationTimestamp()
continuous.Attempt.CreatedAt = &created.Time
recovered = true
break
}
}
}
for _, item := range items.Items {
knownBuild := known[item.GetName()]
if knownBuild {
if continuous == nil || continuous.Attempt == nil || continuous.Attempt.History == nil ||
!slices.Contains(continuous.Attempt.History.Workflows, item.GetName()) || continuous.Attempt.CreatedAt == nil {
continue
}
created := item.GetCreationTimestamp()
if !created.Time.After(*continuous.Attempt.CreatedAt) {
continue
}
}
if !ownedBy(&item, delivery.Metadata.UID) {
continue
}
state, err := c.workflow(ctx, delivery.Metadata.Namespace, item.GetName())
if err != nil {
return false, err
}
attempt := &attemptStatus{
WorkflowName: item.GetName(), Ref: state.Input["source-ref"], Revision: state.Input["source-revision"],
SourceIdentity: item.GetAnnotations()[sourceKey], RetryOf: item.GetAnnotations()[retryKey],
Phase: phaseOrPending(state.Phase), Input: state.Input, CreatedAt: state.CreatedAt,
}
if knownBuild && attempt.SourceIdentity == "" && continuous.Delivered != nil && continuous.Delivered.Revision == attempt.Revision {
attempt.SourceIdentity = continuous.Delivered.Identity
}
if result, found := state.Outputs["delivery-result"]; found {
attempt.Result = opaqueResult(result)
}
if attempt.Ref == "" {
if slices.ContainsFunc(delivery.Status.Points, func(point attemptStatus) bool { return !terminalPhase(point.Phase) }) {
continue
}
delivery.Status.Points = append(delivery.Status.Points, *attempt)
} else {
if continuous == nil {
continuous = &continuousStatus{}
delivery.Status.Continuous = continuous
}
if previous := continuous.Attempt; previous != nil {
if !terminalPhase(previous.Phase) {
continue
}
if previous.CreatedAt != nil && previous.CreatedAt.After(*attempt.CreatedAt) {
rememberAttempt(continuous, previous)
addHistory(previous.History, attempt)
recovered = true
continue
}
}
rememberAttempt(continuous, attempt)
recordDelivered(continuous)
}
if attempt.RetryOf != "" && attempt.RetryOf == delivery.Metadata.Annotations[retryAnnotation] {
delivery.Status.LastHandledRetry = attempt.RetryOf
}
observeQualBuild(delivery, attempt)
recovered = true
}
return recovered, nil
}

87
services/project-delivery/controller/recovery_test.go Normal file
View file

@ -0,0 +1,87 @@
package main
import (
"context"
"slices"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestRecoveryOrder(t *testing.T) {
for _, restored := range []bool{false, true} {
t.Run(map[bool]string{false: "missing-time", true: "known-history"}[restored], func(t *testing.T) {
ctx := context.Background()
now := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC)
instance := testController(t, continuousDelivery(), &mutableRefs{}, &now)
delivery, err := decodeProjectDelivery(continuousDelivery())
if err != nil {
t.Fatal(err)
}
oldTime, newTime := now.Add(-2*time.Hour), now.Add(-time.Hour)
oldRev := "0123456789abcdef0123456789abcdef01234567"
newRev := "89abcdef0123456789abcdef0123456789abcdef"
for _, entry := range []struct {
name, revision, phase string
created time.Time
}{
{"old", oldRev, "Failed", oldTime},
{"new", newRev, "Succeeded", newTime},
} {
input := map[string]string{"source-ref": "refs/heads/main", "source-revision": entry.revision}
if err := instance.ensureWorkflow(ctx, &delivery, entry.name, delivery.Spec.WorkflowTemplateRef.Name, "build", input, nil); err != nil {
t.Fatal(err)
}
resource := instance.client.Resource(workflowGVR).Namespace("delivery")
workflow, err := resource.Get(ctx, entry.name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
workflow.SetCreationTimestamp(metav1.NewTime(entry.created))
if _, err := resource.Update(ctx, workflow, metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
setNamedWorkflowStatus(t, instance, entry.name, entry.phase, nil)
}
delivery.Status.Continuous = &continuousStatus{
Attempt: &attemptStatus{WorkflowName: "new", Ref: "refs/heads/main", Revision: newRev, SourceIdentity: "tree-new", Phase: "Succeeded"},
Delivered: &deliveredSource{Revision: newRev, Identity: "tree-new"},
}
if restored {
delivery.Status.Continuous.Attempt = &attemptStatus{WorkflowName: "old", Ref: "refs/heads/main", Revision: oldRev, Phase: "Failed", CreatedAt: &oldTime,
History: &attemptHistory{Workflows: []string{"old", "new"}, Identities: []string{"tree-new"}},
}
}
if _, err := instance.recoverBuilds(ctx, &delivery); err != nil {
t.Fatal(err)
}
current := delivery.Status.Continuous.Attempt
if current.WorkflowName != "new" || current.Phase != "Succeeded" {
t.Fatalf("latest successful attempt regressed to %s (%s)", current.WorkflowName, current.Phase)
}
if current.CreatedAt == nil || !current.CreatedAt.Equal(newTime) {
t.Fatal("captured creation time missing")
}
if current.SourceIdentity != "tree-new" || !slices.Contains(current.History.Workflows, "old") {
t.Fatal("captured identity or prior history lost")
}
if !slices.Contains(current.History.Workflows, "new") || len(current.History.Workflows) != 2 ||
!slices.Equal(current.History.Identities, []string{"tree-new"}) {
t.Fatal("recovery changed the retained attempt set")
}
if otherAttemptActive(&delivery, "") {
t.Fatal("terminal history recovery occupied an execution slot")
}
if delivered := delivery.Status.Continuous.Delivered; delivered.Revision != newRev || delivered.Identity != "tree-new" {
t.Fatal("terminal history recovery changed the delivered source")
}
if attempt, _ := retryTarget(&delivery, "new"); attempt == nil {
t.Fatal("latest owned terminal attempt is no longer retriable")
}
if changed, err := instance.recoverBuilds(ctx, &delivery); err != nil || changed {
t.Fatalf("repeated recovery changed history: %t %v", changed, err)
}
})
}
}

148
services/project-delivery/controller/render_test.go Normal file
View file

@ -0,0 +1,148 @@
package main
import (
"context"
"io"
"os"
"slices"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
yamlutil "k8s.io/apimachinery/pkg/util/yaml"
dynamicfake "k8s.io/client-go/dynamic/fake"
)
func renderObjects(t *testing.T) ([]runtime.Object, []*unstructured.Unstructured) {
t.Helper()
path := os.Getenv("DELIVERY_RENDER")
if path == "" {
t.Skip("DELIVERY_RENDER is not set")
}
file, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
decoder := yamlutil.NewYAMLOrJSONDecoder(file, 4096)
var objects []runtime.Object
var templates []*unstructured.Unstructured
for {
object := &unstructured.Unstructured{}
if err := decoder.Decode(&object.Object); err == io.EOF {
break
} else if err != nil {
t.Fatal(err)
}
switch object.GetKind() {
case "ConfigMap":
objects = append(objects, object)
case "WorkflowTemplate":
objects = append(objects, object)
templates = append(templates, object)
}
}
if len(templates) == 0 {
t.Fatal("render contains no WorkflowTemplates")
}
return objects, templates
}
func TestMailKeysGraph(t *testing.T) {
_, templates := renderObjects(t)
for _, template := range templates {
if template.GetName() != "mail-keys-build" {
continue
}
stages, _, err := unstructured.NestedSlice(template.Object, "spec", "templates")
if err != nil {
t.Fatal(err)
}
for _, value := range stages {
stage := value.(map[string]any)
if stage["name"] != "delivery" {
continue
}
tasks, _, err := unstructured.NestedSlice(stage, "dag", "tasks")
if err != nil {
t.Fatal(err)
}
want := map[string][]string{
"source-pull": nil,
"cache-pull": nil,
"toolchain-pull": nil,
"build": {"cache-pull", "source-pull", "toolchain-pull"},
"image-publish": {"build"},
"cache-export": {"build"},
}
for _, value := range tasks {
task := value.(map[string]any)
name := task["name"].(string)
expected, found := want[name]
if !found {
t.Fatalf("unexpected or duplicate delivery task %q", name)
}
parents, _, err := unstructured.NestedStringSlice(task, "dependencies")
if err != nil {
t.Fatal(err)
}
slices.Sort(parents)
if !slices.Equal(parents, expected) || task["depends"] != nil {
t.Errorf("%s parents = %v (depends: %v), want %v", name, parents, task["depends"], expected)
}
delete(want, name)
}
if len(want) != 0 {
t.Fatalf("missing delivery tasks: %v", want)
}
return
}
t.Fatal("mail-keys-build has no delivery DAG")
}
t.Fatal("render contains no mail-keys-build")
}
func TestRenderedTemplates(t *testing.T) {
objects, templates := renderObjects(t)
for _, namespace := range []string{"project-delivery"} {
objects = append(objects, &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{"name": "kube-root-ca.crt", "namespace": namespace},
"data": map[string]any{"ca.crt": "synthetic public CA"},
}})
}
client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{
workflowGVR: "WorkflowList", templateGVR: "WorkflowTemplateList", configMapGVR: "ConfigMapList",
}, objects...)
c := &controller{client: client, now: func() time.Time { return time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC) }}
for _, template := range templates {
t.Run(template.GetName(), func(t *testing.T) {
name, err := c.readyTemplate(context.Background(), template.GetNamespace(), template.GetName())
if err != nil {
t.Fatal(err)
}
copy, err := client.Resource(templateGVR).Namespace(template.GetNamespace()).Get(context.Background(), name, metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if copy.GetAnnotations()[templateKey] != template.GetName() {
t.Fatal("captured template lost its logical reference")
}
bundle, err := c.captureBundle(context.Background(), template)
if err != nil {
t.Fatal(err)
}
for _, config := range bundle.configs {
captured, err := client.Resource(configMapGVR).Namespace(template.GetNamespace()).Get(context.Background(), config.GetName(), metav1.GetOptions{})
if err != nil {
t.Fatal(err)
}
if captured.Object["immutable"] != true {
t.Fatalf("action ConfigMap %q is mutable", config.GetName())
}
}
})
}
}

Some files were not shown because too many files have changed in this diff Show more