quorate/services/short/controller/run.go

package controller

import (
	"context"
	"encoding/json"
	"log/slog"
	"os"
	"reflect"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"quorate.dev/services/short/cloud"
	"quorate.dev/services/short/config"
	"quorate.dev/services/short/vault"
	"quorate.dev/services/short/wan"
)

var group = schema.GroupVersionResource{Group: "cilium.io", Version: "v2", Resource: "ciliumcidrgroups"}

type Runner struct {
	Cloud    *cloud.Client
	WAN      *wan.Client
	Kube     kubernetes.Interface
	Dynamic  dynamic.Interface
	VaultURL string
}

func Run(ctx context.Context, directory, namespace, identity, vaultURL, opnURL string) error {
	c, err := config.Read[config.Cloud](directory + "/cloud.json")
	if err != nil {
		return err
	}
	key, err := os.ReadFile(directory + "/private_key")
	if err != nil {
		return err
	}
	oci, err := cloud.New(c, string(key))
	if err != nil {
		return err
	}
	opn, err := wan.New(opnURL, directory+"/opnsense")
	if err != nil {
		return err
	}
	kc, err := rest.InClusterConfig()
	if err != nil {
		return err
	}
	kube, err := kubernetes.NewForConfig(kc)
	if err != nil {
		return err
	}
	dyn, err := dynamic.NewForConfig(kc)
	if err != nil {
		return err
	}
	r := Runner{Cloud: oci, WAN: opn, Kube: kube, Dynamic: dyn, VaultURL: vaultURL}
	lock := &resourcelock.LeaseLock{LeaseMeta: metav1.ObjectMeta{Name: "controller", Namespace: namespace}, Client: kube.CoordinationV1(), LockConfig: resourcelock.ResourceLockConfig{Identity: identity}}
	elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
		Lock: lock, LeaseDuration: 30 * time.Second, RenewDeadline: 20 * time.Second, RetryPeriod: 5 * time.Second,
		ReleaseOnCancel: true, Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: r.loop,
			OnStoppedLeading: func() { slog.Info("controller ownership ended") },
		},
	})
	if err != nil {
		return err
	}
	elector.Run(ctx)
	return nil
}

func (r *Runner) loop(ctx context.Context) {
	ticker := time.NewTicker(10 * time.Second)
	defer ticker.Stop()
	var kv *vault.Client
	var login time.Time
	for {
		if kv == nil || time.Since(login) >= 30*time.Minute {
			client, err := vault.Login(ctx, r.VaultURL, "short-controller", "/var/run/secrets/kubernetes.io/serviceaccount/token")
			if err != nil {
				slog.Error("Vault login", "error", err)
			} else {
				kv = client
				login = time.Now()
			}
		}
		if kv != nil {
			request, cancel := context.WithTimeout(ctx, 20*time.Second)
			err := r.step(request, kv)
			cancel()
			if err != nil {
				slog.Error("reconcile short ingress", "error", err)
			}
		}
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
		}
	}
}

func (r *Runner) step(ctx context.Context, kv *vault.Client) error {
	wan, err := r.WAN.Address(ctx)
	if err != nil {
		return err
	}
	state, err := r.Cloud.Reconcile(ctx, wan)
	if err != nil {
		return err
	}
	resource := r.Dynamic.Resource(group)
	current, err := resource.Get(ctx, "short", metav1.GetOptions{})
	if err != nil {
		return err
	}
	want := make([]any, len(state.Peers))
	for i, value := range state.Peers {
		want[i] = value
	}
	got, _, err := unstructured.NestedSlice(current.Object, "spec", "externalCIDRs")
	if err != nil {
		return err
	}
	if !reflect.DeepEqual(want, got) {
		if err := unstructured.SetNestedSlice(current.Object, want, "spec", "externalCIDRs"); err != nil {
			return err
		}
		if _, err = resource.Update(ctx, current, metav1.UpdateOptions{FieldManager: "short"}); err != nil {
			return err
		}
	}
	if !state.Ready {
		slog.Info("waiting for OCI headends")
		return nil
	}
	svc, err := r.Kube.CoreV1().Services("kube-system").Get(ctx, "cilium-gateway-gateway-short", metav1.GetOptions{})
	if err != nil {
		return err
	}
	ports := make([]uint16, 0, len(svc.Spec.Ports))
	for _, p := range svc.Spec.Ports {
		ports = append(ports, uint16(p.Port))
	}
	runtime := config.Runtime{Bundle: state.Bundle, Path: config.Path{Address: r.Cloud.Config.Address, Network: r.Cloud.Config.Network, Gateway: svc.Spec.ClusterIP, Ports: ports}}
	if err = runtime.Bundle.Validate(); err != nil {
		return err
	}
	if err = runtime.Path.Validate(); err != nil {
		return err
	}
	data, err := json.Marshal(runtime)
	if err != nil {
		return err
	}
	record, err := kv.Read(ctx, "short/peers")
	if err != nil {
		return err
	}
	if record.Data["config"] == string(data) {
		return nil
	}
	record.Data = map[string]any{"config": string(data)}
	if err = kv.Write(ctx, "short/peers", record); err != nil {
		return err
	}
	slog.Info("published short tunnel configuration", "wan", wan, "headends", len(state.Bundle.Peers), "ports", ports)
	return nil
}