package controller import ( "context" "encoding/json" "log/slog" "os" "reflect" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" "quorate.dev/services/short/cloud" "quorate.dev/services/short/config" "quorate.dev/services/short/vault" "quorate.dev/services/short/wan" ) var group = schema.GroupVersionResource{Group: "cilium.io", Version: "v2", Resource: "ciliumcidrgroups"} type Runner struct { Cloud *cloud.Client WAN *wan.Client Kube kubernetes.Interface Dynamic dynamic.Interface VaultURL string } func Run(ctx context.Context, directory, namespace, identity, vaultURL, opnURL string) error { c, err := config.Read[config.Cloud](directory + "/cloud.json") if err != nil { return err } key, err := os.ReadFile(directory + "/private_key") if err != nil { return err } oci, err := cloud.New(c, string(key)) if err != nil { return err } opn, err := wan.New(opnURL, directory+"/opnsense") if err != nil { return err } kc, err := rest.InClusterConfig() if err != nil { return err } kube, err := kubernetes.NewForConfig(kc) if err != nil { return err } dyn, err := dynamic.NewForConfig(kc) if err != nil { return err } r := Runner{Cloud: oci, WAN: opn, Kube: kube, Dynamic: dyn, VaultURL: vaultURL} lock := &resourcelock.LeaseLock{LeaseMeta: metav1.ObjectMeta{Name: "controller", Namespace: namespace}, Client: kube.CoordinationV1(), LockConfig: resourcelock.ResourceLockConfig{Identity: identity}} elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ Lock: lock, LeaseDuration: 30 * time.Second, RenewDeadline: 20 * time.Second, RetryPeriod: 5 * time.Second, ReleaseOnCancel: true, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: r.loop, OnStoppedLeading: func() { slog.Info("controller ownership ended") }, }, }) if err != nil { return err } elector.Run(ctx) return nil } func (r *Runner) loop(ctx context.Context) { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() var kv *vault.Client var login time.Time for { if kv == nil || time.Since(login) >= 30*time.Minute { client, err := vault.Login(ctx, r.VaultURL, "short-controller", "/var/run/secrets/kubernetes.io/serviceaccount/token") if err != nil { slog.Error("Vault login", "error", err) } else { kv = client login = time.Now() } } if kv != nil { request, cancel := context.WithTimeout(ctx, 20*time.Second) err := r.step(request, kv) cancel() if err != nil { slog.Error("reconcile short ingress", "error", err) } } select { case <-ctx.Done(): return case <-ticker.C: } } } func (r *Runner) step(ctx context.Context, kv *vault.Client) error { wan, err := r.WAN.Address(ctx) if err != nil { return err } state, err := r.Cloud.Reconcile(ctx, wan) if err != nil { return err } resource := r.Dynamic.Resource(group) current, err := resource.Get(ctx, "short", metav1.GetOptions{}) if err != nil { return err } want := make([]any, len(state.Peers)) for i, value := range state.Peers { want[i] = value } got, _, err := unstructured.NestedSlice(current.Object, "spec", "externalCIDRs") if err != nil { return err } if !reflect.DeepEqual(want, got) { if err := unstructured.SetNestedSlice(current.Object, want, "spec", "externalCIDRs"); err != nil { return err } if _, err = resource.Update(ctx, current, metav1.UpdateOptions{FieldManager: "short"}); err != nil { return err } } if !state.Ready { slog.Info("waiting for OCI headends") return nil } svc, err := r.Kube.CoreV1().Services("kube-system").Get(ctx, "cilium-gateway-gateway-short", metav1.GetOptions{}) if err != nil { return err } ports := make([]uint16, 0, len(svc.Spec.Ports)) for _, p := range svc.Spec.Ports { ports = append(ports, uint16(p.Port)) } runtime := config.Runtime{Bundle: state.Bundle, Path: config.Path{Address: r.Cloud.Config.Address, Network: r.Cloud.Config.Network, Gateway: svc.Spec.ClusterIP, Ports: ports}} if err = runtime.Bundle.Validate(); err != nil { return err } if err = runtime.Path.Validate(); err != nil { return err } data, err := json.Marshal(runtime) if err != nil { return err } record, err := kv.Read(ctx, "short/peers") if err != nil { return err } if record.Data["config"] == string(data) { return nil } record.Data = map[string]any{"config": string(data)} if err = kv.Write(ctx, "short/peers", record); err != nil { return err } slog.Info("published short tunnel configuration", "wan", wan, "headends", len(state.Bundle.Peers), "ports", ports) return nil }