ce7cb702218693d08ea6d83464d2457c9250027c
This commit is contained in:
commit
34c5f8963c
1 changed files with 145336 additions and 0 deletions
290
services/project-delivery/controller/qualification.go
Normal file
290
services/project-delivery/controller/qualification.go
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/util/retry"
|
||||
)
|
||||
|
||||
type qualChoice struct {
|
||||
namespace string
|
||||
name string
|
||||
kind string
|
||||
index int
|
||||
}
|
||||
|
||||
func (choice qualChoice) key() string {
|
||||
return choice.namespace + "/" + choice.name + "/" + choice.kind
|
||||
}
|
||||
|
||||
func (c *controller) qualifyAll(ctx context.Context) error {
|
||||
items, err := c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var failures []error
|
||||
for _, item := range items.Items {
|
||||
if err := c.qualifyKey(ctx, item.GetNamespace(), item.GetName(), c.prepareQual); err != nil {
|
||||
failures = append(failures, err)
|
||||
}
|
||||
}
|
||||
items, err = c.client.Resource(projectDeliveryGVR).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return errors.Join(append(failures, err)...)
|
||||
}
|
||||
var choices []qualChoice
|
||||
for _, item := range items.Items {
|
||||
delivery, err := decodeProjectDelivery(&item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q := delivery.Status.Qualification
|
||||
if q == nil {
|
||||
continue
|
||||
}
|
||||
choice := qualChoice{namespace: item.GetNamespace(), name: item.GetName()}
|
||||
for index, attempt := range q.Attempts {
|
||||
if attempt.Phase == "Pending" || terminalPhase(attempt.Phase) {
|
||||
continue
|
||||
}
|
||||
err := c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error {
|
||||
return c.advanceQual(ctx, d, index)
|
||||
})
|
||||
return errors.Join(append(failures, err)...)
|
||||
}
|
||||
if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil {
|
||||
continue
|
||||
}
|
||||
if index := slices.IndexFunc(q.Attempts, func(a qualAttempt) bool { return a.Phase == "Pending" }); index >= 0 {
|
||||
choice.kind, choice.index = "Manual", index
|
||||
choices = append(choices, choice)
|
||||
}
|
||||
due, err := qualDue(&delivery, c.now())
|
||||
if err != nil {
|
||||
failures = append(failures, err)
|
||||
} else if due {
|
||||
choice.kind = "Auto"
|
||||
choices = append(choices, choice)
|
||||
}
|
||||
}
|
||||
if len(choices) == 0 {
|
||||
return errors.Join(failures...)
|
||||
}
|
||||
slices.SortFunc(choices, func(a, b qualChoice) int { return strings.Compare(a.key(), b.key()) })
|
||||
index := slices.IndexFunc(choices, func(choice qualChoice) bool { return choice.key() > c.qualCursor })
|
||||
if index < 0 {
|
||||
index = 0
|
||||
}
|
||||
choice := choices[index]
|
||||
err = c.qualifyKey(ctx, choice.namespace, choice.name, func(ctx context.Context, d *projectDelivery) error {
|
||||
q := d.Status.Qualification
|
||||
if choice.kind == "Manual" {
|
||||
q.Attempts[choice.index].Phase = "Reserved"
|
||||
return nil
|
||||
}
|
||||
due, err := qualDue(d, c.now())
|
||||
if err != nil || !due {
|
||||
return err
|
||||
}
|
||||
build := len(q.Builds)
|
||||
q.Attempts = append(q.Attempts, qualAttempt{
|
||||
Kind: "Auto", Build: build, Phase: "Reserved",
|
||||
WorkflowName: workflowName("qual", d.Metadata.UID, "Auto", q.Builds[build-1].WorkflowName),
|
||||
Template: d.Spec.Qualification.WorkflowTemplateRef.Name,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
c.qualCursor = choice.key()
|
||||
}
|
||||
return errors.Join(append(failures, err)...)
|
||||
}
|
||||
|
||||
func (c *controller) qualifyKey(ctx context.Context, namespace, name string, action func(context.Context, *projectDelivery) error) error {
|
||||
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
object, err := c.client.Resource(projectDeliveryGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delivery, err := decodeProjectDelivery(object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
before, _ := toMap(delivery.Status)
|
||||
actionErr := action(ctx, &delivery)
|
||||
if delivery.Status.Qualification != nil {
|
||||
delivery.Status.Qualification.Message = ""
|
||||
if actionErr != nil {
|
||||
delivery.Status.Qualification.Message = actionErr.Error()
|
||||
}
|
||||
}
|
||||
after, _ := toMap(delivery.Status)
|
||||
if !reflect.DeepEqual(before, after) {
|
||||
if err := c.updateStatus(ctx, object, &delivery); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return actionErr
|
||||
})
|
||||
}
|
||||
|
||||
func (c *controller) prepareQual(ctx context.Context, delivery *projectDelivery) error {
|
||||
if delivery.Spec.Qualification == nil || delivery.Metadata.DeletionTimestamp != nil {
|
||||
return nil
|
||||
}
|
||||
if delivery.Status.Qualification == nil {
|
||||
delivery.Status.Qualification = &qualStatus{}
|
||||
}
|
||||
builds := make([]*attemptStatus, 0, len(delivery.Status.Points)+1)
|
||||
if status := delivery.Status.Continuous; status != nil && status.Attempt != nil {
|
||||
builds = append(builds, status.Attempt)
|
||||
}
|
||||
for i := range delivery.Status.Points {
|
||||
builds = append(builds, &delivery.Status.Points[i])
|
||||
}
|
||||
for _, build := range builds {
|
||||
if build.Phase != "Succeeded" {
|
||||
continue
|
||||
}
|
||||
if len(build.Input) == 0 {
|
||||
state, err := c.workflow(ctx, delivery.Metadata.Namespace, build.WorkflowName)
|
||||
if apierrors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
build.Input = state.Input
|
||||
}
|
||||
observeQualBuild(delivery, build)
|
||||
}
|
||||
annotation := delivery.Metadata.Annotations[qualAnnotation]
|
||||
if annotation == "" {
|
||||
return nil
|
||||
}
|
||||
name, request, found := strings.Cut(annotation, "/")
|
||||
if !found || name == "" || request == "" {
|
||||
return fmt.Errorf("qualification request must be <build-workflow-name>/<request-id>")
|
||||
}
|
||||
q := delivery.Status.Qualification
|
||||
for _, attempt := range q.Attempts {
|
||||
if attempt.Kind == "Manual" && attempt.RequestID == request {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
index := slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name })
|
||||
if index < 0 {
|
||||
state, err := c.workflow(ctx, delivery.Metadata.Namespace, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, found := state.Outputs["delivery-result"]
|
||||
if state.OwnerUID != delivery.Metadata.UID || state.Phase != "Succeeded" || !found {
|
||||
return fmt.Errorf("Workflow %q is not a successful build of this ProjectDelivery with a delivery-result", name)
|
||||
}
|
||||
observeQualBuild(delivery, &attemptStatus{WorkflowName: name, Phase: state.Phase, Input: state.Input, Result: opaqueResult(result)})
|
||||
index = slices.IndexFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == name })
|
||||
if index < 0 {
|
||||
return fmt.Errorf("Workflow %q has no recorded build inputs", name)
|
||||
}
|
||||
}
|
||||
q.Attempts = append(q.Attempts, qualAttempt{
|
||||
Kind: "Manual", RequestID: request, Build: index + 1, Phase: "Pending",
|
||||
WorkflowName: workflowName("qual", delivery.Metadata.UID, "Manual", request),
|
||||
Template: delivery.Spec.Qualification.WorkflowTemplateRef.Name,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func observeQualBuild(delivery *projectDelivery, build *attemptStatus) {
|
||||
if delivery.Spec.Qualification == nil || build.Phase != "Succeeded" || len(build.Input) == 0 || len(build.Result) == 0 {
|
||||
return
|
||||
}
|
||||
if delivery.Status.Qualification == nil {
|
||||
delivery.Status.Qualification = &qualStatus{}
|
||||
}
|
||||
q := delivery.Status.Qualification
|
||||
if slices.ContainsFunc(q.Builds, func(b qualBuild) bool { return b.WorkflowName == build.WorkflowName }) {
|
||||
return
|
||||
}
|
||||
input := maps.Clone(build.Input)
|
||||
input["delivery-result"] = string(build.Result)
|
||||
q.Builds = append(q.Builds, qualBuild{WorkflowName: build.WorkflowName, Input: input})
|
||||
}
|
||||
|
||||
func qualDue(delivery *projectDelivery, now time.Time) (bool, error) {
|
||||
spec := delivery.Spec.Qualification
|
||||
q := delivery.Status.Qualification
|
||||
if spec == nil || spec.Mode != "OnBuild" || q == nil || len(q.Builds) <= q.Covered {
|
||||
return false, nil
|
||||
}
|
||||
for _, attempt := range q.Attempts {
|
||||
if attempt.Kind == "Auto" && attempt.Build == len(q.Builds) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
var interval time.Duration
|
||||
if spec.Interval != "" {
|
||||
value, err := time.ParseDuration(spec.Interval)
|
||||
if err != nil || value < 0 {
|
||||
return false, fmt.Errorf("invalid qualification interval %q", spec.Interval)
|
||||
}
|
||||
interval = value
|
||||
}
|
||||
return q.LastStart == nil || !now.Before(q.LastStart.Add(interval)), nil
|
||||
}
|
||||
|
||||
func (c *controller) advanceQual(ctx context.Context, delivery *projectDelivery, index int) error {
|
||||
q := delivery.Status.Qualification
|
||||
attempt := &q.Attempts[index]
|
||||
if attempt.Phase == "Reserved" {
|
||||
if err := c.ensureWorkflow(ctx, delivery, attempt.WorkflowName, attempt.Template, "qualification", q.Builds[attempt.Build-1].Input, nil); err != nil {
|
||||
if errors.Is(err, templatePending) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
state, err := c.workflow(ctx, delivery.Metadata.Namespace, attempt.WorkflowName)
|
||||
if apierrors.IsNotFound(err) && attempt.Phase != "Reserved" {
|
||||
attempt.Phase = "Error"
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if attempt.StartedAt == nil && state.StartedAt != nil {
|
||||
attempt.StartedAt = state.StartedAt
|
||||
q.LastStart = state.StartedAt
|
||||
q.Covered = max(q.Covered, attempt.Build)
|
||||
}
|
||||
attempt.Phase = state.Phase
|
||||
if attempt.Phase == "" || attempt.Phase == "Pending" {
|
||||
attempt.Phase = "Queued"
|
||||
}
|
||||
if result, found := state.Outputs["qualification-result"]; found {
|
||||
attempt.Result = opaqueResult(result)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func opaqueResult(value string) []byte {
|
||||
if json.Valid([]byte(value)) {
|
||||
return []byte(value)
|
||||
}
|
||||
result, _ := json.Marshal(value)
|
||||
return result
|
||||
}
|
||||
Loading…
Reference in a new issue