ce7cb702218693d08ea6d83464d2457c9250027c

services/short/controller/run.go

@@ -0,0 +1,177 @@
+package controller
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"os"
+	"reflect"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"quorate.dev/services/short/cloud"
+	"quorate.dev/services/short/config"
+	"quorate.dev/services/short/vault"
+	"quorate.dev/services/short/wan"
+)
+
+var group = schema.GroupVersionResource{Group: "cilium.io", Version: "v2", Resource: "ciliumcidrgroups"}
+
+type Runner struct {
+	Cloud    *cloud.Client
+	WAN      *wan.Client
+	Kube     kubernetes.Interface
+	Dynamic  dynamic.Interface
+	VaultURL string
+}
+
+func Run(ctx context.Context, directory, namespace, identity, vaultURL, opnURL string) error {
+	c, err := config.Read[config.Cloud](directory + "/cloud.json")
+	if err != nil {
+		return err
+	}
+	key, err := os.ReadFile(directory + "/private_key")
+	if err != nil {
+		return err
+	}
+	oci, err := cloud.New(c, string(key))
+	if err != nil {
+		return err
+	}
+	opn, err := wan.New(opnURL, directory+"/opnsense")
+	if err != nil {
+		return err
+	}
+	kc, err := rest.InClusterConfig()
+	if err != nil {
+		return err
+	}
+	kube, err := kubernetes.NewForConfig(kc)
+	if err != nil {
+		return err
+	}
+	dyn, err := dynamic.NewForConfig(kc)
+	if err != nil {
+		return err
+	}
+	r := Runner{Cloud: oci, WAN: opn, Kube: kube, Dynamic: dyn, VaultURL: vaultURL}
+	lock := &resourcelock.LeaseLock{LeaseMeta: metav1.ObjectMeta{Name: "controller", Namespace: namespace}, Client: kube.CoordinationV1(), LockConfig: resourcelock.ResourceLockConfig{Identity: identity}}
+	elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
+		Lock: lock, LeaseDuration: 30 * time.Second, RenewDeadline: 20 * time.Second, RetryPeriod: 5 * time.Second,
+		ReleaseOnCancel: true, Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: r.loop,
+			OnStoppedLeading: func() { slog.Info("controller ownership ended") },
+		},
+	})
+	if err != nil {
+		return err
+	}
+	elector.Run(ctx)
+	return nil
+}
+
+func (r *Runner) loop(ctx context.Context) {
+	ticker := time.NewTicker(10 * time.Second)
+	defer ticker.Stop()
+	var kv *vault.Client
+	var login time.Time
+	for {
+		if kv == nil || time.Since(login) >= 30*time.Minute {
+			client, err := vault.Login(ctx, r.VaultURL, "short-controller", "/var/run/secrets/kubernetes.io/serviceaccount/token")
+			if err != nil {
+				slog.Error("Vault login", "error", err)
+			} else {
+				kv = client
+				login = time.Now()
+			}
+		}
+		if kv != nil {
+			request, cancel := context.WithTimeout(ctx, 20*time.Second)
+			err := r.step(request, kv)
+			cancel()
+			if err != nil {
+				slog.Error("reconcile short ingress", "error", err)
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (r *Runner) step(ctx context.Context, kv *vault.Client) error {
+	wan, err := r.WAN.Address(ctx)
+	if err != nil {
+		return err
+	}
+	state, err := r.Cloud.Reconcile(ctx, wan)
+	if err != nil {
+		return err
+	}
+	resource := r.Dynamic.Resource(group)
+	current, err := resource.Get(ctx, "short", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	want := make([]any, len(state.Peers))
+	for i, value := range state.Peers {
+		want[i] = value
+	}
+	got, _, err := unstructured.NestedSlice(current.Object, "spec", "externalCIDRs")
+	if err != nil {
+		return err
+	}
+	if !reflect.DeepEqual(want, got) {
+		if err := unstructured.SetNestedSlice(current.Object, want, "spec", "externalCIDRs"); err != nil {
+			return err
+		}
+		if _, err = resource.Update(ctx, current, metav1.UpdateOptions{FieldManager: "short"}); err != nil {
+			return err
+		}
+	}
+	if !state.Ready {
+		slog.Info("waiting for OCI headends")
+		return nil
+	}
+	svc, err := r.Kube.CoreV1().Services("kube-system").Get(ctx, "cilium-gateway-gateway-short", metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	ports := make([]uint16, 0, len(svc.Spec.Ports))
+	for _, p := range svc.Spec.Ports {
+		ports = append(ports, uint16(p.Port))
+	}
+	runtime := config.Runtime{Bundle: state.Bundle, Path: config.Path{Address: r.Cloud.Config.Address, Network: r.Cloud.Config.Network, Gateway: svc.Spec.ClusterIP, Ports: ports}}
+	if err = runtime.Bundle.Validate(); err != nil {
+		return err
+	}
+	if err = runtime.Path.Validate(); err != nil {
+		return err
+	}
+	data, err := json.Marshal(runtime)
+	if err != nil {
+		return err
+	}
+	record, err := kv.Read(ctx, "short/peers")
+	if err != nil {
+		return err
+	}
+	if record.Data["config"] == string(data) {
+		return nil
+	}
+	record.Data = map[string]any{"config": string(data)}
+	if err = kv.Write(ctx, "short/peers", record); err != nil {
+		return err
+	}
+	slog.Info("published short tunnel configuration", "wan", wan, "headends", len(state.Bundle.Peers), "ports", ports)
+	return nil
+}