quorate/services/project-delivery/controller/qualification.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"maps"
	"reflect"
	"slices"
	"strings"
	"time"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/util/retry"
)

type qualChoice struct {
	namespace string
	name      string
	kind      string
	index     int
}

func (choice qualChoice) key() string {
	return choice.namespace + "/" + choice.name + "/" + choice.kind
}

func (c *controller) qualifyAll(ctx context.Context) error {
	items, err := c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
	if err != nil {
		return err
	}
	var failures []error
	for _, item := range items.Items {
		if err := c.qualifyKey(ctx, item.GetNamespace(), item.GetName(), c.prepareQual); err != nil {
			failures = append(failures, err)
		}
	}
	items, err = c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
	if err != nil {
		return errors.Join(append(failures, err)...)
	}
	var choices []qualChoice
	for _, item := range items.Items {
		delivery, err := decodeProjectDelivery(&item)
		if err != nil {
			return err
		}
		q := delivery.Status.Qualification
		if q == nil {
			continue
		}
		choice := qualChoice{namespace: item.GetNamespace(), name: item.GetName()}
		for index, attempt := range q.Attempts {
			if attempt.Phase == "Pending" || terminalPhase(attempt.Phase) {
				continue
			}
			err := c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error {
				return c.advanceQual(ctx, d, index)
			})
			return errors.Join(append(failures, err)...)
		}
		if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil {
			continue
		}
		if index := slices.IndexFunc(q.Attempts, func(a qualAttempt) bool { return a.Phase == "Pending" }); index >= 0 {
			choice.kind, choice.index = "Manual", index
			choices = append(choices, choice)
		}
		due, err := qualDue(&delivery, c.now())
		if err != nil {
			failures = append(failures, err)
		} else if due {
			choice.kind = "Auto"
			choices = append(choices, choice)
		}
	}
	if len(choices) == 0 {
		return errors.Join(failures...)
	}
	slices.SortFunc(choices, func(a, b qualChoice) int { return strings.Compare(a.key(), b.key()) })
	index := slices.IndexFunc(choices, func(choice qualChoice) bool { return choice.key() > c.qualCursor })
	if index < 0 {
		index = 0
	}
	choice := choices[index]
	err = c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error {
		q := d.Status.Qualification
		if choice.kind == "Manual" {
			q.Attempts[choice.index].Phase = "Reserved"
			return nil
		}
		due, err := qualDue(d, c.now())
		if err != nil || !due {
			return err
		}
		build := len(q.Builds)
		q.Attempts = append(q.Attempts, qualAttempt{
			Kind: "Auto", Build: build, Phase: "Reserved",
			WorkflowName: workflowName("qual", d.Metadata.UID, "Auto", q.Builds[build-1].WorkflowName),
			Template:     d.Spec.Qualification.WorkflowTemplateRef.Name,
		})
		return nil
	})
	if err == nil {
		c.qualCursor = choice.key()
	}
	return errors.Join(append(failures, err)...)
}

func (c *controller) qualifyKey(ctx context.Context, namespace, name string, action func(context.Context, *projectDelivery) error) error {
	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
		object, err := c.client.Resource(projectDeliveryGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
		if apierrors.IsNotFound(err) {
			return nil
		}
		if err != nil {
			return err
		}
		delivery, err := decodeProjectDelivery(object)
		if err != nil {
			return err
		}
		before, _ := toMap(delivery.Status)
		actionErr := action(ctx, &delivery)
		if delivery.Status.Qualification != nil {
			delivery.Status.Qualification.Message = ""
			if actionErr != nil {
				delivery.Status.Qualification.Message = actionErr.Error()
			}
		}
		after, _ := toMap(delivery.Status)
		if !reflect.DeepEqual(before, after) {
			if err := c.updateStatus(ctx, object, &delivery); err != nil {
				return err
			}
		}
		return actionErr
	})
}

func (c *controller) prepareQual(ctx context.Context, delivery *projectDelivery) error {
	if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil {
		return nil
	}
	if delivery.Status.Qualification == nil {
		delivery.Status.Qualification = &qualStatus{}
	}
	builds := make([]*attemptStatus, 0, len(delivery.Status.Points)+1)
	if status := delivery.Status.Continuous; status != nil && status.Attempt != nil {
		builds = append(builds, status.Attempt)
	}
	for i := range delivery.Status.Points {
		builds = append(builds, &delivery.Status.Points[i])
	}
	for _, build := range builds {
		if build.Phase != "Succeeded" {
			continue
		}
		if len(build.Input) == 0 {
			state, err := c.workflow(ctx, delivery.Metadata.Namespace, build.WorkflowName)
			if apierrors.IsNotFound(err) {
				continue
			}
			if err != nil {
				return err
			}
			build.Input = state.Input
		}
		observeQualBuild(delivery, build)
	}
	annotation := delivery.Metadata.Annotations[qualAnnotation]
	if annotation == "" {
		return nil
	}
	name, request, found := strings.Cut(annotation, "/")
	if !found || name == "" || request == "" {
		return fmt.Errorf("qualification request must be <build-workflow-name>/<request-id>")
	}
	q := delivery.Status.Qualification
	for _, attempt := range q.Attempts {
		if attempt.Kind == "Manual" && attempt.RequestID == request {
			return nil
		}
	}
	index := slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name })
	if index < 0 {
		state, err := c.workflow(ctx, delivery.Metadata.Namespace, name)
		if err != nil {
			return err
		}
		result, found := state.Outputs["delivery-result"]
		if state.OwnerUID != delivery.Metadata.UID || state.Phase != "Succeeded" || !found {
			return fmt.Errorf("Workflow %q is not a successful build of this ProjectDelivery with a delivery-result", name)
		}
		observeQualBuild(delivery, &attemptStatus{WorkflowName: name, Phase: state.Phase, Input: state.Input, Result: opaqueResult(result)})
		index = slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name })
		if index < 0 {
			return fmt.Errorf("Workflow %q has no recorded build inputs", name)
		}
	}
	q.Attempts = append(q.Attempts, qualAttempt{
		Kind: "Manual", RequestID: request, Build: index + 1, Phase: "Pending",
		WorkflowName: workflowName("qual", delivery.Metadata.UID, "Manual", request),
		Template:     delivery.Spec.Qualification.WorkflowTemplateRef.Name,
	})
	return nil
}

func observeQualBuild(delivery *projectDelivery, build *attemptStatus) {
	if delivery.Spec.Qualification == nil || build.Phase != "Succeeded" || len(build.Input) == 0 || len(build.Result) == 0 {
		return
	}
	if delivery.Status.Qualification == nil {
		delivery.Status.Qualification = &qualStatus{}
	}
	q := delivery.Status.Qualification
	if slices.ContainsFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == build.WorkflowName }) {
		return
	}
	input := maps.Clone(build.Input)
	input["delivery-result"] = string(build.Result)
	q.Builds = append(q.Builds, qualBuild{WorkflowName: build.WorkflowName, Input: input})
}

func qualDue(delivery *projectDelivery, now time.Time) (bool, error) {
	spec := delivery.Spec.Qualification
	q := delivery.Status.Qualification
	if spec == nil || spec.Mode != "OnBuild" || q == nil || len(q.Builds) <= q.Covered {
		return false, nil
	}
	for _, attempt := range q.Attempts {
		if attempt.Kind == "Auto" && attempt.Build == len(q.Builds) {
			return false, nil
		}
	}
	var interval time.Duration
	if spec.Interval != "" {
		value, err := time.ParseDuration(spec.Interval)
		if err != nil || value < 0 {
			return false, fmt.Errorf("invalid qualification interval %q", spec.Interval)
		}
		interval = value
	}
	return q.LastStart == nil || !now.Before(q.LastStart.Add(interval)), nil
}

func (c *controller) advanceQual(ctx context.Context, delivery *projectDelivery, index int) error {
	q := delivery.Status.Qualification
	attempt := &q.Attempts[index]
	if attempt.Phase == "Reserved" {
		if err := c.ensureWorkflow(ctx, delivery, attempt.WorkflowName, attempt.Template, "qualification", q.Builds[attempt.Build-1].Input, nil); err != nil {
			if errors.Is(err, templatePending) {
				return nil
			}
			return err
		}
	}
	state, err := c.workflow(ctx, delivery.Metadata.Namespace, attempt.WorkflowName)
	if apierrors.IsNotFound(err) && attempt.Phase != "Reserved" {
		attempt.Phase = "Error"
		return nil
	}
	if err != nil {
		return err
	}
	if attempt.StartedAt == nil && state.StartedAt != nil {
		attempt.StartedAt = state.StartedAt
		q.LastStart = state.StartedAt
		q.Covered = max(q.Covered, attempt.Build)
	}
	attempt.Phase = state.Phase
	if attempt.Phase == "" || attempt.Phase == "Pending" {
		attempt.Phase = "Queued"
	}
	if result, found := state.Outputs["qualification-result"]; found {
		attempt.Result = opaqueResult(result)
	}
	return nil
}

func opaqueResult(value string) []byte {
	if json.Valid([]byte(value)) {
		return []byte(value)
	}
	result, _ := json.Marshal(value)
	return result
}