package main import ( "context" "encoding/json" "errors" "fmt" "maps" "reflect" "slices" "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" ) type qualChoice struct { namespace string name string kind string index int } func (choice qualChoice) key() string { return choice.namespace + "/" + choice.name + "/" + choice.kind } func (c *controller) qualifyAll(ctx context.Context) error { items, err := c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { return err } var failures []error for _, item := range items.Items { if err := c.qualifyKey(ctx, item.GetNamespace(), item.GetName(), c.prepareQual); err != nil { failures = append(failures, err) } } items, err = c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { return errors.Join(append(failures, err)...) } var choices []qualChoice for _, item := range items.Items { delivery, err := decodeProjectDelivery(&item) if err != nil { return err } q := delivery.Status.Qualification if q == nil { continue } choice := qualChoice{namespace: item.GetNamespace(), name: item.GetName()} for index, attempt := range q.Attempts { if attempt.Phase == "Pending" || terminalPhase(attempt.Phase) { continue } err := c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error { return c.advanceQual(ctx, d, index) }) return errors.Join(append(failures, err)...) } if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil { continue } if index := slices.IndexFunc(q.Attempts, func(a qualAttempt) bool { return a.Phase == "Pending" }); index >= 0 { choice.kind, choice.index = "Manual", index choices = append(choices, choice) } due, err := qualDue(&delivery, c.now()) if err != nil { failures = append(failures, err) } else if due { choice.kind = "Auto" choices = append(choices, choice) } } if len(choices) == 0 { return errors.Join(failures...) } slices.SortFunc(choices, func(a, b qualChoice) int { return strings.Compare(a.key(), b.key()) }) index := slices.IndexFunc(choices, func(choice qualChoice) bool { return choice.key() > c.qualCursor }) if index < 0 { index = 0 } choice := choices[index] err = c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error { q := d.Status.Qualification if choice.kind == "Manual" { q.Attempts[choice.index].Phase = "Reserved" return nil } due, err := qualDue(d, c.now()) if err != nil || !due { return err } build := len(q.Builds) q.Attempts = append(q.Attempts, qualAttempt{ Kind: "Auto", Build: build, Phase: "Reserved", WorkflowName: workflowName("qual", d.Metadata.UID, "Auto", q.Builds[build-1].WorkflowName), Template: d.Spec.Qualification.WorkflowTemplateRef.Name, }) return nil }) if err == nil { c.qualCursor = choice.key() } return errors.Join(append(failures, err)...) } func (c *controller) qualifyKey(ctx context.Context, namespace, name string, action func(context.Context, *projectDelivery) error) error { return retry.RetryOnConflict(retry.DefaultRetry, func() error { object, err := c.client.Resource(projectDeliveryGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { return nil } if err != nil { return err } delivery, err := decodeProjectDelivery(object) if err != nil { return err } before, _ := toMap(delivery.Status) actionErr := action(ctx, &delivery) if delivery.Status.Qualification != nil { delivery.Status.Qualification.Message = "" if actionErr != nil { delivery.Status.Qualification.Message = actionErr.Error() } } after, _ := toMap(delivery.Status) if !reflect.DeepEqual(before, after) { if err := c.updateStatus(ctx, object, &delivery); err != nil { return err } } return actionErr }) } func (c *controller) prepareQual(ctx context.Context, delivery *projectDelivery) error { if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil { return nil } if delivery.Status.Qualification == nil { delivery.Status.Qualification = &qualStatus{} } builds := make([]*attemptStatus, 0, len(delivery.Status.Points)+1) if status := delivery.Status.Continuous; status != nil && status.Attempt != nil { builds = append(builds, status.Attempt) } for i := range delivery.Status.Points { builds = append(builds, &delivery.Status.Points[i]) } for _, build := range builds { if build.Phase != "Succeeded" { continue } if len(build.Input) == 0 { state, err := c.workflow(ctx, delivery.Metadata.Namespace, build.WorkflowName) if apierrors.IsNotFound(err) { continue } if err != nil { return err } build.Input = state.Input } observeQualBuild(delivery, build) } annotation := delivery.Metadata.Annotations[qualAnnotation] if annotation == "" { return nil } name, request, found := strings.Cut(annotation, "/") if !found || name == "" || request == "" { return fmt.Errorf("qualification request must be /") } q := delivery.Status.Qualification for _, attempt := range q.Attempts { if attempt.Kind == "Manual" && attempt.RequestID == request { return nil } } index := slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name }) if index < 0 { state, err := c.workflow(ctx, delivery.Metadata.Namespace, name) if err != nil { return err } result, found := state.Outputs["delivery-result"] if state.OwnerUID != delivery.Metadata.UID || state.Phase != "Succeeded" || !found { return fmt.Errorf("Workflow %q is not a successful build of this ProjectDelivery with a delivery-result", name) } observeQualBuild(delivery, &attemptStatus{WorkflowName: name, Phase: state.Phase, Input: state.Input, Result: opaqueResult(result)}) index = slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name }) if index < 0 { return fmt.Errorf("Workflow %q has no recorded build inputs", name) } } q.Attempts = append(q.Attempts, qualAttempt{ Kind: "Manual", RequestID: request, Build: index + 1, Phase: "Pending", WorkflowName: workflowName("qual", delivery.Metadata.UID, "Manual", request), Template: delivery.Spec.Qualification.WorkflowTemplateRef.Name, }) return nil } func observeQualBuild(delivery *projectDelivery, build *attemptStatus) { if delivery.Spec.Qualification == nil || build.Phase != "Succeeded" || len(build.Input) == 0 || len(build.Result) == 0 { return } if delivery.Status.Qualification == nil { delivery.Status.Qualification = &qualStatus{} } q := delivery.Status.Qualification if slices.ContainsFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == build.WorkflowName }) { return } input := maps.Clone(build.Input) input["delivery-result"] = string(build.Result) q.Builds = append(q.Builds, qualBuild{WorkflowName: build.WorkflowName, Input: input}) } func qualDue(delivery *projectDelivery, now time.Time) (bool, error) { spec := delivery.Spec.Qualification q := delivery.Status.Qualification if spec == nil || spec.Mode != "OnBuild" || q == nil || len(q.Builds) <= q.Covered { return false, nil } for _, attempt := range q.Attempts { if attempt.Kind == "Auto" && attempt.Build == len(q.Builds) { return false, nil } } var interval time.Duration if spec.Interval != "" { value, err := time.ParseDuration(spec.Interval) if err != nil || value < 0 { return false, fmt.Errorf("invalid qualification interval %q", spec.Interval) } interval = value } return q.LastStart == nil || !now.Before(q.LastStart.Add(interval)), nil } func (c *controller) advanceQual(ctx context.Context, delivery *projectDelivery, index int) error { q := delivery.Status.Qualification attempt := &q.Attempts[index] if attempt.Phase == "Reserved" { if err := c.ensureWorkflow(ctx, delivery, attempt.WorkflowName, attempt.Template, "qualification", q.Builds[attempt.Build-1].Input, nil); err != nil { if errors.Is(err, templatePending) { return nil } return err } } state, err := c.workflow(ctx, delivery.Metadata.Namespace, attempt.WorkflowName) if apierrors.IsNotFound(err) && attempt.Phase != "Reserved" { attempt.Phase = "Error" return nil } if err != nil { return err } if attempt.StartedAt == nil && state.StartedAt != nil { attempt.StartedAt = state.StartedAt q.LastStart = state.StartedAt q.Covered = max(q.Covered, attempt.Build) } attempt.Phase = state.Phase if attempt.Phase == "" || attempt.Phase == "Pending" { attempt.Phase = "Queued" } if result, found := state.Outputs["qualification-result"]; found { attempt.Result = opaqueResult(result) } return nil } func opaqueResult(value string) []byte { if json.Valid([]byte(value)) { return []byte(value) } result, _ := json.Marshal(value) return result }