ms2k/quorate
SHA256
Watch
0
Code Releases Activity

ce7cb702218693d08ea6d83464d2457c9250027c

Browse source
This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
2 changed files with 145336 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

177
services/short/controller/run.go Normal file
View file

@ -0,0 +1,177 @@
package controller
import (
"context"
"encoding/json"
"log/slog"
"os"
"reflect"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
"quorate.dev/services/short/cloud"
"quorate.dev/services/short/config"
"quorate.dev/services/short/vault"
"quorate.dev/services/short/wan"
)
var group = schema.GroupVersionResource{Group: "cilium.io", Version: "v2", Resource: "ciliumcidrgroups"}
type Runner struct {
Cloud *cloud.Client
WAN *wan.Client
Kube kubernetes.Interface
Dynamic dynamic.Interface
VaultURL string
}
func Run(ctx context.Context, directory, namespace, identity, vaultURL, opnURL string) error {
c, err := config.Read[config.Cloud](directory + "/cloud.json")
if err != nil {
return err
}
key, err := os.ReadFile(directory + "/private_key")
if err != nil {
return err
}
oci, err := cloud.New(c, string(key))
if err != nil {
return err
}
opn, err := wan.New(opnURL, directory+"/opnsense")
if err != nil {
return err
}
kc, err := rest.InClusterConfig()
if err != nil {
return err
}
kube, err := kubernetes.NewForConfig(kc)
if err != nil {
return err
}
dyn, err := dynamic.NewForConfig(kc)
if err != nil {
return err
}
r := Runner{Cloud: oci, WAN: opn, Kube: kube, Dynamic: dyn, VaultURL: vaultURL}
lock := &resourcelock.LeaseLock{LeaseMeta: metav1.ObjectMeta{Name: "controller", Namespace: namespace}, Client: kube.CoordinationV1(), LockConfig: resourcelock.ResourceLockConfig{Identity: identity}}
elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
Lock: lock, LeaseDuration: 30 * time.Second, RenewDeadline: 20 * time.Second, RetryPeriod: 5 * time.Second,
ReleaseOnCancel: true, Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: r.loop,
OnStoppedLeading: func() { slog.Info("controller ownership ended") },
},
})
if err != nil {
return err
}
elector.Run(ctx)
return nil
}
func (r *Runner) loop(ctx context.Context) {
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
var kv *vault.Client
var login time.Time
for {
if kv == nil || time.Since(login) >= 30*time.Minute {
client, err := vault.Login(ctx, r.VaultURL, "short-controller", "/var/run/secrets/kubernetes.io/serviceaccount/token")
if err != nil {
slog.Error("Vault login", "error", err)
} else {
kv = client
login = time.Now()
}
}
if kv != nil {
request, cancel := context.WithTimeout(ctx, 20*time.Second)
err := r.step(request, kv)
cancel()
if err != nil {
slog.Error("reconcile short ingress", "error", err)
}
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (r *Runner) step(ctx context.Context, kv *vault.Client) error {
wan, err := r.WAN.Address(ctx)
if err != nil {
return err
}
state, err := r.Cloud.Reconcile(ctx, wan)
if err != nil {
return err
}
resource := r.Dynamic.Resource(group)
current, err := resource.Get(ctx, "short", metav1.GetOptions{})
if err != nil {
return err
}
want := make([]any, len(state.Peers))
for i, value := range state.Peers {
want[i] = value
}
got, _, err := unstructured.NestedSlice(current.Object, "spec", "externalCIDRs")
if err != nil {
return err
}
if !reflect.DeepEqual(want, got) {
if err := unstructured.SetNestedSlice(current.Object, want, "spec", "externalCIDRs"); err != nil {
return err
}
if _, err = resource.Update(ctx, current, metav1.UpdateOptions{FieldManager: "short"}); err != nil {
return err
}
}
if !state.Ready {
slog.Info("waiting for OCI headends")
return nil
}
svc, err := r.Kube.CoreV1().Services("kube-system").Get(ctx, "cilium-gateway-gateway-short", metav1.GetOptions{})
if err != nil {
return err
}
ports := make([]uint16, 0, len(svc.Spec.Ports))
for _, p := range svc.Spec.Ports {
ports = append(ports, uint16(p.Port))
}
runtime := config.Runtime{Bundle: state.Bundle, Path: config.Path{Address: r.Cloud.Config.Address, Network: r.Cloud.Config.Network, Gateway: svc.Spec.ClusterIP, Ports: ports}}
if err = runtime.Bundle.Validate(); err != nil {
return err
}
if err = runtime.Path.Validate(); err != nil {
return err
}
data, err := json.Marshal(runtime)
if err != nil {
return err
}
record, err := kv.Read(ctx, "short/peers")
if err != nil {
return err
}
if record.Data["config"] == string(data) {
return nil
}
record.Data = map[string]any{"config": string(data)}
if err = kv.Write(ctx, "short/peers", record); err != nil {
return err
}
slog.Info("published short tunnel configuration", "wan", wan, "headends", len(state.Bundle.Peers), "ports", ports)
return nil
}

177
services/short/controller/run_test.go Normal file
View file

@ -0,0 +1,177 @@
package controller
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"sync"
"sync/atomic"
"testing"
"github.com/oracle/oci-go-sdk/v65/common"
"github.com/oracle/oci-go-sdk/v65/core"
kcore "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
dynfake "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/kubernetes/fake"
"quorate.dev/services/short/cloud"
"quorate.dev/services/short/config"
"quorate.dev/services/short/vault"
"quorate.dev/services/short/wan"
)
type publishedCloud struct {
cloud.Network
phase atomic.Int32
}
func (f *publishedCloud) generations() []int {
switch f.phase.Load() {
case 0:
return []int{0}
case 1:
return []int{0, 1}
default:
return []int{1}
}
}
func (f *publishedCloud) ListCpes(context.Context, core.ListCpesRequest) (core.ListCpesResponse, error) {
var result core.ListCpesResponse
for _, i := range f.generations() {
result.Items = append(result.Items, core.Cpe{Id: common.String(fmt.Sprint(i)), IpAddress: common.String(fmt.Sprintf("192.0.2.%d", i+2)), FreeformTags: map[string]string{"owner": "short"}})
}
return result, nil
}
func (f *publishedCloud) ListIPSecConnections(context.Context, core.ListIPSecConnectionsRequest) (core.ListIPSecConnectionsResponse, error) {
var result core.ListIPSecConnectionsResponse
for _, i := range f.generations() {
result.Items = append(result.Items, core.IpSecConnection{Id: common.String(fmt.Sprint(i)), CpeId: common.String(fmt.Sprint(i)), LifecycleState: core.IpSecConnectionLifecycleStateAvailable, StaticRoutes: []string{"198.18.0.10/32"}, FreeformTags: map[string]string{"owner": "short"}})
}
return result, nil
}
func (f *publishedCloud) ListIPSecConnectionTunnels(_ context.Context, r core.ListIPSecConnectionTunnelsRequest) (core.ListIPSecConnectionTunnelsResponse, error) {
var result core.ListIPSecConnectionTunnelsResponse
for i := 1; i <= 2; i++ {
result.Items = append(result.Items, core.IpSecConnectionTunnel{
Id: common.String(fmt.Sprintf("%s-%d", *r.IpscId, i)), VpnIp: common.String(fmt.Sprintf("192.0.2.%s%d", map[string]string{"0": "1", "1": "2"}[*r.IpscId], i)),
LifecycleState: core.IpSecConnectionTunnelLifecycleStateAvailable, Status: core.IpSecConnectionTunnelStatusDown,
OracleCanInitiate: core.IpSecConnectionTunnelOracleCanInitiateInitiatorOrResponder,
})
}
return result, nil
}
func (f *publishedCloud) GetIPSecConnectionTunnelSharedSecret(_ context.Context, r core.GetIPSecConnectionTunnelSharedSecretRequest) (core.GetIPSecConnectionTunnelSharedSecretResponse, error) {
return core.GetIPSecConnectionTunnelSharedSecretResponse{IpSecConnectionTunnelSharedSecret: core.IpSecConnectionTunnelSharedSecret{SharedSecret: common.String("synthetic-" + *r.TunnelId)}}, nil
}
func TestWANPublication(t *testing.T) {
ctx := context.Background()
api := &publishedCloud{}
dyn := dynfake.NewSimpleDynamicClient(runtime.NewScheme(), &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "cilium.io/v2", "kind": "CiliumCIDRGroup", "metadata": map[string]any{"name": "short"}, "spec": map[string]any{"externalCIDRs": []any{}},
}})
kube := fake.NewClientset(&kcore.Service{ObjectMeta: metav1.ObjectMeta{Name: "cilium-gateway-gateway-short", Namespace: "kube-system"}, Spec: kcore.ServiceSpec{ClusterIP: "192.0.2.100", Ports: []kcore.ServicePort{{Port: 8080}}}})
var mu sync.Mutex
version := 1
data := map[string]any{"config": "{}"}
writes := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
mu.Lock()
defer mu.Unlock()
w.Header().Set("Content-Type", "application/json")
switch req.URL.Path {
case "/api/interfaces/overview/interfaces_info":
if req.Method != http.MethodGet {
t.Error("WAN reader attempted a mutation")
}
address := "192.0.2.2"
if api.phase.Load() > 0 {
address = "192.0.2.3"
}
fmt.Fprintf(w, `{"rows":[{"identifier":"WAN","ipv4":[{"ipaddr":%q}]}]}`, address)
case "/v1/secret/data/short/peers":
if req.Method == http.MethodGet {
json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"data": data, "metadata": map[string]int{"version": version}}})
return
}
var input struct {
Options struct{ CAS int } `json:"options"`
Data map[string]any `json:"data"`
}
if err := json.NewDecoder(req.Body).Decode(&input); err != nil {
t.Error(err)
}
if req.Method != http.MethodPost || input.Options.CAS != version {
t.Error("configuration publication lost its CAS contract")
http.Error(w, "conflict", http.StatusBadRequest)
return
}
var published config.Runtime
if err := json.Unmarshal([]byte(input.Data["config"].(string)), &published); err != nil {
t.Error(err)
}
current, _ := dyn.Resource(group).Get(ctx, "short", metav1.GetOptions{})
allowed, _, _ := unstructured.NestedStringSlice(current.Object, "spec", "externalCIDRs")
for _, p := range published.Peers {
found := false
for _, cidr := range allowed {
found = found || cidr == p.Address+"/32"
}
if !found {
t.Error("published a peer before admitting its address")
}
}
data = input.Data
version++
writes++
default:
t.Errorf("unexpected request: %s", req.URL.Path)
http.NotFound(w, req)
}
}))
defer server.Close()
r := Runner{Cloud: &cloud.Client{API: api, Config: config.Cloud{Address: "198.18.0.10", Network: "198.18.1.0/24"}}, WAN: &wan.Client{URL: server.URL, HTTP: server.Client()}, Kube: kube, Dynamic: dyn}
kv := &vault.Client{URL: server.URL, HTTP: server.Client(), Token: "synthetic-token"}
for phase, wantCIDRs := range []int{2, 4, 2} {
api.phase.Store(int32(phase))
if err := r.step(ctx, kv); err != nil {
t.Fatal(err)
}
mu.Lock()
value := data["config"].(string)
count := writes
mu.Unlock()
var published config.Runtime
if err := json.Unmarshal([]byte(value), &published); err != nil {
t.Fatal(err)
}
wantWAN, wantWrites := "192.0.2.2", 1
if phase > 0 {
wantWAN, wantWrites = "192.0.2.3", 2
}
if published.WAN != wantWAN || count != wantWrites || published.Path.Address != "198.18.0.10" || published.Path.Network != "198.18.1.0/24" || published.Path.Gateway != "192.0.2.100" || !reflect.DeepEqual(published.Path.Ports, []uint16{8080}) {
t.Fatalf("wrong runtime publication: %s; writes=%d", value, count)
}
current, _ := dyn.Resource(group).Get(ctx, "short", metav1.GetOptions{})
cidrs, _, _ := unstructured.NestedStringSlice(current.Object, "spec", "externalCIDRs")
if len(cidrs) != wantCIDRs {
t.Fatalf("wrong peer admission transition: %v", cidrs)
}
if directory := os.Getenv("SHORT_FIXTURE"); directory != "" && phase < 2 {
if err := os.WriteFile(filepath.Join(directory, fmt.Sprintf("runtime%d.json", phase)), []byte(value), 0600); err != nil {
t.Fatal(err)
}
}
}
}