ms2k/quorate
SHA256
Watch
0
Code Releases Activity

ce7cb702218693d08ea6d83464d2457c9250027c

Browse source
This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
8 changed files with 145336 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

118
services/short/tunnel/admission_test.go Normal file
View file

@ -0,0 +1,118 @@
package tunnel
import (
"encoding/json"
"os"
"strings"
"testing"
"github.com/google/cel-go/cel"
admission "k8s.io/api/admissionregistration/v1"
core "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/yaml"
)
func TestElectionPolicy(t *testing.T) {
data, err := os.ReadFile("../k8s/tunnel/admission.yaml")
if err != nil {
t.Fatal(err)
}
var policy admission.ValidatingAdmissionPolicy
if err := yaml.Unmarshal([]byte(strings.Split(string(data), "\n---")[0]), &policy); err != nil {
t.Fatal(err)
}
env, err := cel.NewEnv(
cel.Variable("object", cel.DynType), cel.Variable("oldObject", cel.DynType),
cel.Variable("request", cel.DynType), cel.Variable("variables", cel.DynType),
)
if err != nil {
t.Fatal(err)
}
eval := func(expression string, values map[string]any) any {
t.Helper()
ast, issues := env.Compile(expression)
if issues.Err() != nil {
t.Fatal(issues.Err())
}
program, err := env.Program(ast)
if err != nil {
t.Fatal(err)
}
result, _, err := program.Eval(values)
if err != nil {
t.Fatal(err)
}
return result.Value()
}
object := func(p *core.Pod) map[string]any {
t.Helper()
data, err := json.Marshal(p)
if err != nil {
t.Fatal(err)
}
var result map[string]any
if err := json.Unmarshal(data, &result); err != nil {
t.Fatal(err)
}
return result
}
pod := &core.Pod{
ObjectMeta: metav1.ObjectMeta{Name: "gateway-1", Namespace: "short", Labels: map[string]string{
"app.kubernetes.io/name": "tunnel", "role": "active",
}},
Spec: core.PodSpec{ServiceAccountName: "tunnel", Containers: []core.Container{{Name: "tunnel", Image: "synthetic:v1"}}},
}
tests := []struct {
name string
change func(*core.Pod, *core.Pod)
allow bool
}{
{"claim", func(old, next *core.Pod) { delete(old.Labels, "role") }, true},
{"release", func(old, next *core.Pod) { delete(next.Labels, "role") }, true},
{"unchanged", func(old, next *core.Pod) {}, true},
{"controller", func(old, next *core.Pod) { old.Spec.ServiceAccountName = "short-controller"; next.Spec = old.Spec }, false},
{"forged-label", func(old, next *core.Pod) { delete(old.Labels, "app.kubernetes.io/name") }, false},
{"other-namespace", func(old, next *core.Pod) { old.Namespace = "other"; next.Namespace = "other" }, false},
{"image", func(old, next *core.Pod) { next.Spec.Containers[0].Image = "synthetic:v2" }, false},
{"label-added", func(old, next *core.Pod) { next.Labels["extra"] = "value" }, false},
{"label-removed", func(old, next *core.Pod) { delete(next.Labels, "app.kubernetes.io/name") }, false},
{"label-changed", func(old, next *core.Pod) { next.Labels["app.kubernetes.io/name"] = "other" }, false},
{"role-value", func(old, next *core.Pod) { next.Labels["role"] = "other" }, false},
{"annotation", func(old, next *core.Pod) { next.Annotations = map[string]string{"extra": "value"} }, false},
{"finalizer", func(old, next *core.Pod) { next.Finalizers = []string{"example.test/hold"} }, false},
{"owner", func(old, next *core.Pod) { next.OwnerReferences = []metav1.OwnerReference{{Name: "other"}} }, false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
old, next := pod.DeepCopy(), pod.DeepCopy()
test.change(old, next)
variables := map[string]any{}
values := map[string]any{"object": object(next), "oldObject": object(old), "variables": variables,
"request": map[string]any{"userInfo": map[string]any{"username": "system:serviceaccount:short:tunnel"}}}
for _, match := range policy.Spec.MatchConditions {
if eval(match.Expression, values) != true {
t.Fatal("tunnel admission was skipped")
}
}
for _, variable := range policy.Spec.Variables {
variables[variable.Name] = eval(variable.Expression, values)
}
allowed := true
for _, validation := range policy.Spec.Validations {
if eval(validation.Expression, values) != true {
allowed = false
}
}
if allowed != test.allow {
t.Fatalf("allowed=%v, want %v", allowed, test.allow)
}
})
}
for _, actor := range []string{"system:serviceaccount:flux-system:kustomize-controller", "system:kube-controller-manager"} {
values := map[string]any{"request": map[string]any{"userInfo": map[string]any{"username": actor}}}
if eval(policy.Spec.MatchConditions[0].Expression, values) != false {
t.Fatalf("policy restricts unrelated controller %s", actor)
}
}
}

92
services/short/tunnel/lease.go Normal file
View file

@ -0,0 +1,92 @@
package tunnel
import (
"context"
"encoding/json"
"fmt"
"sync"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/leaderelection"
"k8s.io/client-go/tools/leaderelection/resourcelock"
)
func label(ctx context.Context, kube kubernetes.Interface, namespace, name string, value any) error {
data, err := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]any{"role": value}}})
if err != nil {
return err
}
_, err = kube.CoreV1().Pods(namespace).Patch(ctx, name, types.MergePatchType, data, metav1.PatchOptions{})
return err
}
func claim(ctx context.Context, kube kubernetes.Interface, namespace, name string) error {
pods, err := kube.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=tunnel,app.kubernetes.io/instance=short"})
if err != nil {
return err
}
found := false
for _, pod := range pods.Items {
if pod.Name == name {
found = true
} else if pod.Labels["role"] == "active" {
if err := label(ctx, kube, namespace, pod.Name, nil); err != nil {
return err
}
}
}
if !found {
return fmt.Errorf("tunnel pod is missing")
}
return label(ctx, kube, namespace, name, "active")
}
func ownership(ctx context.Context, kube kubernetes.Interface, namespace, name string, work func(context.Context) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var gate sync.Mutex
var stopped bool
var running sync.WaitGroup
result := make(chan error, 1)
lock := &resourcelock.LeaseLock{LeaseMeta: metav1.ObjectMeta{Name: "tunnel", Namespace: namespace}, Client: kube.CoordinationV1(), LockConfig: resourcelock.ResourceLockConfig{Identity: name}}
elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
// A successful renewal reply can consume 20s before retry, renewal failure, and child shutdown.
Lock: lock, LeaseDuration: 60 * time.Second, RenewDeadline: 20 * time.Second, RetryPeriod: 5 * time.Second,
// Immediate release could start a replacement while charon is stopping.
ReleaseOnCancel: false,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: func(leader context.Context) {
gate.Lock()
if stopped {
gate.Unlock()
return
}
running.Add(1)
gate.Unlock()
defer running.Done()
result <- work(leader)
cancel()
},
OnStoppedLeading: func() {
gate.Lock()
stopped = true
cancel()
gate.Unlock()
running.Wait()
},
},
})
if err != nil {
return err
}
elector.Run(ctx)
select {
case err := <-result:
return err
default:
return nil
}
}

135
services/short/tunnel/lease_test.go Normal file
View file

@ -0,0 +1,135 @@
package tunnel
import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
coordination "k8s.io/api/coordination/v1"
core "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes/fake"
ktest "k8s.io/client-go/testing"
)
func testKube() *fake.Clientset {
kube := fake.NewClientset(&coordination.Lease{ObjectMeta: metav1.ObjectMeta{Name: "tunnel", Namespace: "short", ResourceVersion: "1"}})
for _, name := range []string{"tunnel-0", "tunnel-1"} {
kube.Tracker().Add(&core.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "short", Labels: map[string]string{"app.kubernetes.io/name": "tunnel", "app.kubernetes.io/instance": "short"}}})
}
return kube
}
func TestClaim(t *testing.T) {
kube := testKube()
ctx := context.Background()
if err := label(ctx, kube, "short", "tunnel-0", "active"); err != nil {
t.Fatal(err)
}
if err := claim(ctx, kube, "short", "tunnel-1"); err != nil {
t.Fatal(err)
}
pods, _ := kube.CoreV1().Pods("short").List(ctx, metav1.ListOptions{})
for _, pod := range pods.Items {
if (pod.Labels["role"] == "active") != (pod.Name == "tunnel-1") {
t.Fatalf("wrong backend: %s %v", pod.Name, pod.Labels)
}
}
if err := label(ctx, kube, "short", "tunnel-0", "active"); err != nil {
t.Fatal(err)
}
if err := label(ctx, kube, "short", "tunnel-1", nil); err != nil {
t.Fatal(err)
}
kube.PrependReactor("patch", "pods", func(a ktest.Action) (bool, runtime.Object, error) {
if a.(ktest.PatchAction).GetName() == "tunnel-0" {
return true, nil, fmt.Errorf("API unavailable")
}
return false, nil, nil
})
if err := claim(ctx, kube, "short", "tunnel-1"); err == nil {
t.Fatal("claimed without removing old backend")
}
pod, _ := kube.CoreV1().Pods("short").Get(ctx, "tunnel-1", metav1.GetOptions{})
if pod.Labels["role"] == "active" {
t.Fatal("two active backends after API failure")
}
}
func TestLeaseHandoff(t *testing.T) {
kube := testKube()
var api sync.Mutex
version := 1
kube.PrependReactor("update", "leases", func(a ktest.Action) (bool, runtime.Object, error) {
api.Lock()
defer api.Unlock()
u := a.(ktest.UpdateAction).GetObject().(*coordination.Lease).DeepCopy()
gvr := schema.GroupVersionResource{Group: "coordination.k8s.io", Version: "v1", Resource: "leases"}
old, err := kube.Tracker().Get(gvr, "short", "tunnel")
if err != nil {
return true, nil, err
}
if u.ResourceVersion != old.(*coordination.Lease).ResourceVersion {
return true, nil, apierrors.NewConflict(gvr.GroupResource(), "tunnel", fmt.Errorf("version changed"))
}
version++
u.ResourceVersion = fmt.Sprint(version)
return true, u, kube.Tracker().Update(gvr, u, "short")
})
ctx, cancel := context.WithTimeout(context.Background(), 75*time.Second)
defer cancel()
first, stopFirst := context.WithCancel(ctx)
defer stopFirst()
var count atomic.Int32
started := make(chan string, 2)
done := make(chan error, 2)
work := func(name string) func(context.Context) error {
return func(ctx context.Context) error {
if count.Add(1) != 1 {
t.Error("overlapping tunnel owners")
}
defer count.Add(-1)
if err := claim(ctx, kube, "short", name); err != nil {
return err
}
started <- name
<-ctx.Done()
return nil
}
}
go func() { done <- ownership(first, kube, "short", "tunnel-0", work("tunnel-0")) }()
select {
case name := <-started:
if name != "tunnel-0" {
t.Fatal(name)
}
case <-ctx.Done():
t.Fatal("first owner did not start")
}
go func() { done <- ownership(ctx, kube, "short", "tunnel-1", work("tunnel-1")) }()
stopFirst()
if err := <-done; err != nil {
t.Fatal(err)
}
select {
case name := <-started:
if name != "tunnel-1" {
t.Fatal(name)
}
case <-ctx.Done():
t.Fatal("standby did not acquire expired lease")
}
cancel()
if err := <-done; err != nil {
t.Fatal(err)
}
if count.Load() != 0 {
t.Fatal("owner still running")
}
}

75
services/short/tunnel/native_test.go Normal file
View file

@ -0,0 +1,75 @@
//go:build integration
package tunnel
import (
"context"
"encoding/json"
"net"
"net/http"
"os"
"sync/atomic"
"testing"
"time"
)
func TestNativeReload(t *testing.T) {
nativeLoop(t, false)
}
func TestNativeRuntime(t *testing.T) {
nativeLoop(t, true)
}
func nativeLoop(t *testing.T, observe bool) {
t.Helper()
file := os.Getenv("SHORT_RUNTIME")
if file == "" {
t.Skip("SHORT_RUNTIME requires an isolated native IPsec fixture")
}
ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
defer cancel()
var ready atomic.Bool
var published atomic.Bool
if observe {
listener, err := net.Listen("tcp", "127.0.0.1:9091")
if err != nil {
t.Fatal(err)
}
server := &http.Server{ReadHeaderTimeout: time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/status" {
http.NotFound(w, r)
return
}
json.NewEncoder(w).Encode(map[string]bool{"ready": ready.Load(), "published": published.Load()})
})}
go server.Serve(listener)
defer server.Close()
}
done := make(chan error, 1)
go func() {
done <- serve(ctx, file, &ready, func(context.Context) error {
published.Store(true)
return nil
})
}()
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
select {
case err := <-done:
t.Fatalf("tunnel ended before fixture completion: %v", err)
case <-ctx.Done():
<-done
t.Fatal("native fixture did not finish within its bound")
case <-ticker.C:
if _, err := os.Stat(file + ".done"); err == nil {
cancel()
if err := <-done; err != nil {
t.Fatal(err)
}
return
}
}
}
}

159
services/short/tunnel/network.go Normal file
View file

@ -0,0 +1,159 @@
package tunnel
import (
"encoding/binary"
"fmt"
"net"
"github.com/google/nftables"
"github.com/google/nftables/expr"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netlink/nl"
"golang.org/x/sys/unix"
"quorate.dev/services/short/config"
)
func forwarding(link netlink.Link) error {
req := nl.NewNetlinkRequest(unix.RTM_NEWLINK, unix.NLM_F_ACK)
info := nl.NewIfInfomsg(unix.AF_UNSPEC)
info.Index = int32(link.Attrs().Index)
req.AddData(info)
af := nl.NewRtAttr(unix.IFLA_AF_SPEC|unix.NLA_F_NESTED, nil)
inet := nl.NewRtAttrChild(af, unix.AF_INET|unix.NLA_F_NESTED, nil)
conf := nl.NewRtAttrChild(inet, unix.IFLA_INET_CONF|unix.NLA_F_NESTED, nil)
// Linux UAPI IPV4_DEVCONF_FORWARDING is the first IPv4 interface setting.
nl.NewRtAttrChild(conf, 1, nl.Uint32Attr(1))
req.AddData(af)
_, err := req.Execute(unix.NETLINK_ROUTE, 0)
return err
}
func network(p config.Path) error {
if err := p.Validate(); err != nil {
return err
}
links := make([]netlink.Link, 2)
for index := range links {
name := fmt.Sprintf("ipsec%d", index)
link, err := netlink.LinkByName(name)
if _, ok := err.(netlink.LinkNotFoundError); ok {
link = &netlink.Xfrmi{LinkAttrs: netlink.LinkAttrs{Name: name, MTU: 1280}, Ifid: uint32(42 + index)}
if err = netlink.LinkAdd(link); err != nil {
return err
}
} else if err != nil {
return err
}
if err = netlink.LinkSetUp(link); err != nil {
return err
}
links[index] = link
}
lo, err := netlink.LinkByName("lo")
if err != nil {
return err
}
addr, err := netlink.ParseAddr(p.Address + "/32")
if err != nil {
return err
}
if err = netlink.AddrReplace(lo, addr); err != nil {
return err
}
eth, err := netlink.LinkByName("eth0")
if err != nil {
return err
}
for _, device := range append(links, eth) {
if err = forwarding(device); err != nil {
return fmt.Errorf("enable interface forwarding: %w", err)
}
}
return firewall(p)
}
func returnRoute(network string, index int) error {
link, err := netlink.LinkByName(fmt.Sprintf("ipsec%d", index))
if err != nil {
return err
}
_, destination, err := net.ParseCIDR(network)
if err != nil {
return err
}
return netlink.RouteReplace(&netlink.Route{LinkIndex: link.Attrs().Index, Dst: destination})
}
func inputInterface(name string) []expr.Any {
value := make([]byte, 16)
copy(value, name)
return []expr.Any{&expr.Meta{Key: expr.MetaKeyIIFNAME, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: value}}
}
func clampMSS(name string, key expr.MetaKey) []expr.Any {
value := make([]byte, 16)
copy(value, name)
return []expr.Any{
&expr.Meta{Key: key, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: value},
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 13, Len: 1},
&expr.Bitwise{DestRegister: 1, SourceRegister: 1, Len: 1, Mask: []byte{0x02}, Xor: []byte{0}},
&expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{0x02}},
&expr.Rt{Register: 1, Key: expr.RtTCPMSS},
&expr.Byteorder{DestRegister: 1, SourceRegister: 1, Op: expr.ByteorderHton, Len: 2, Size: 2},
&expr.Exthdr{SourceRegister: 1, Type: 2, Offset: 2, Len: 2, Op: expr.ExthdrOpTcpopt},
}
}
func dest(address string, port uint16) []expr.Any {
buf := make([]byte, 2)
binary.BigEndian.PutUint16(buf, port)
return []expr.Any{
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{unix.IPPROTO_TCP}},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: net.ParseIP(address).To4()},
&expr.Payload{DestRegister: 1, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2}, &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: buf},
}
}
func firewall(p config.Path) error {
c := &nftables.Conn{}
tables, err := c.ListTables()
if err != nil {
return err
}
for _, t := range tables {
if t.Name == "short" && t.Family == nftables.TableFamilyIPv4 {
c.DelTable(t)
}
}
t := c.AddTable(&nftables.Table{Name: "short", Family: nftables.TableFamilyIPv4})
chain := func(name string, kind nftables.ChainType, hook *nftables.ChainHook, priority *nftables.ChainPriority) *nftables.Chain {
return c.AddChain(&nftables.Chain{Name: name, Table: t, Type: kind, Hooknum: hook, Priority: priority})
}
pre := chain("dnat", nftables.ChainTypeNAT, nftables.ChainHookPrerouting, nftables.ChainPriorityNATDest)
post := chain("snat", nftables.ChainTypeNAT, nftables.ChainHookPostrouting, nftables.ChainPriorityNATSource)
forward := chain("forward", nftables.ChainTypeFilter, nftables.ChainHookForward, nftables.ChainPriorityFilter)
input := chain("input", nftables.ChainTypeFilter, nftables.ChainHookInput, nftables.ChainPriorityFilter)
for index := range 2 {
device := fmt.Sprintf("ipsec%d", index)
for _, key := range []expr.MetaKey{expr.MetaKeyIIFNAME, expr.MetaKeyOIFNAME} {
c.AddRule(&nftables.Rule{Table: t, Chain: forward, Exprs: clampMSS(device, key)})
}
for _, port := range p.Ports {
match := append(inputInterface(device), dest(p.Address, port)...)
match = append(match, &expr.Immediate{Register: 1, Data: net.ParseIP(p.Gateway).To4()}, &expr.NAT{Type: expr.NATTypeDestNAT, Family: unix.NFPROTO_IPV4, RegAddrMin: 1})
c.AddRule(&nftables.Rule{Table: t, Chain: pre, Exprs: match})
match = append(inputInterface(device), dest(p.Gateway, port)...)
c.AddRule(&nftables.Rule{Table: t, Chain: post, Exprs: append(match, &expr.Masq{})})
match = append(inputInterface(device), dest(p.Gateway, port)...)
c.AddRule(&nftables.Rule{Table: t, Chain: forward, Exprs: append(match, &expr.Verdict{Kind: expr.VerdictAccept})})
}
for _, chain := range []*nftables.Chain{forward, input} {
c.AddRule(&nftables.Rule{Table: t, Chain: chain, Exprs: append(inputInterface(device), &expr.Verdict{Kind: expr.VerdictDrop})})
}
}
if err := c.Flush(); err != nil {
return fmt.Errorf("apply pod forwarding: %w", err)
}
return nil
}

177
services/short/tunnel/run.go Normal file
View file

@ -0,0 +1,177 @@
package tunnel
import (
"context"
"fmt"
"log/slog"
"net/http"
"os"
"os/exec"
"reflect"
"sync/atomic"
"syscall"
"time"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"quorate.dev/services/short/config"
)
func readRuntime(file string) (config.Runtime, error) {
r, err := config.Read[config.Runtime](file)
if err == nil {
err = r.Bundle.Validate()
}
if err == nil {
err = r.Path.Validate()
}
return r, err
}
func Run(ctx context.Context, file, namespace, name string) error {
kc, err := rest.InClusterConfig()
if err != nil {
return err
}
kube, err := kubernetes.NewForConfig(kc)
if err != nil {
return err
}
if err = label(ctx, kube, namespace, name, nil); err != nil {
return err
}
var ready atomic.Bool
var leading atomic.Bool
server := &http.Server{Addr: ":9090", ReadHeaderTimeout: 5 * time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/ready" {
http.NotFound(w, r)
return
}
_, err := readRuntime(file)
if err != nil || leading.Load() && !ready.Load() {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
fmt.Fprintln(w, "ready")
})}
go func() {
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
slog.Error("health server", "error", err)
}
}()
defer server.Close()
err = ownership(ctx, kube, namespace, name, func(leader context.Context) error {
leading.Store(true)
return serve(leader, file, &ready, func(request context.Context) error {
return claim(request, kube, namespace, name)
})
})
ready.Store(false)
request, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := label(request, kube, namespace, name, nil); err != nil {
slog.Error("remove active tunnel label", "error", err)
}
return err
}
func serve(ctx context.Context, file string, ready *atomic.Bool, publish func(context.Context) error) error {
var stop context.CancelFunc
var exited chan error
var active *config.Runtime
var loaded bool
var published bool
shutdown := func() {
ready.Store(false)
if stop != nil {
stop()
<-exited
stop = nil
}
}
defer shutdown()
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
step := func() error {
r, err := readRuntime(file)
if err != nil {
return err
}
restart := active == nil || !reflect.DeepEqual(active.Bundle, r.Bundle) || active.Path.Address != r.Path.Address || active.Path.Network != r.Path.Network
if restart {
shutdown()
if err = network(r.Path); err != nil {
return err
}
child, cancel := context.WithCancel(ctx)
cmd := exec.CommandContext(child, "/usr/local/libexec/ipsec/charon")
cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }
cmd.WaitDelay = 10 * time.Second
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err = cmd.Start(); err != nil {
cancel()
return err
}
stop = cancel
exited = make(chan error, 1)
go func() { exited <- cmd.Wait() }()
active = &r
loaded = false
} else if !reflect.DeepEqual(active.Path, r.Path) {
if err = firewall(r.Path); err != nil {
return err
}
active = &r
}
if !loaded {
for index, peer := range r.Peers {
if err = load(r.WAN, peer, index); err != nil {
return err
}
}
loaded = true
}
if !published {
if err = publish(ctx); err != nil {
return err
}
published = true
}
up, err := installed()
if err != nil {
return err
}
if index := returnPeer(up); index >= 0 {
if err = returnRoute(r.Path.Network, index); err != nil {
return err
}
}
ready.Store(returnPeer(up) >= 0)
for index, live := range up {
if !live {
if err := initiate(index); err != nil {
slog.Error("initiate headend", "index", index, "error", err)
}
}
}
return nil
}
for {
if ctx.Err() != nil {
return nil
}
if err := step(); err != nil {
ready.Store(false)
slog.Error("reconcile tunnel", "error", err)
}
select {
case <-ctx.Done():
return nil
case err := <-exited:
stop = nil
return fmt.Errorf("charon exited: %w", err)
case <-ticker.C:
}
}
}

150
services/short/tunnel/vici.go Normal file
View file

@ -0,0 +1,150 @@
package tunnel
import (
"context"
"fmt"
"net"
"strconv"
"time"
"github.com/strongswan/govici/vici"
"quorate.dev/services/short/config"
)
type auth struct {
Method string `vici:"auth"`
ID string `vici:"id"`
}
type child struct {
Local []string `vici:"local_ts"`
Remote []string `vici:"remote_ts"`
Proposals []string `vici:"esp_proposals"`
In string `vici:"if_id_in"`
Out string `vici:"if_id_out"`
Rekey string `vici:"rekey_time"`
DPD string `vici:"dpd_action"`
}
type connection struct {
Version string `vici:"version"`
Remote []string `vici:"remote_addrs"`
Proposals []string `vici:"proposals"`
Local auth `vici:"local"`
Peer auth `vici:"remote"`
Children map[string]child `vici:"children"`
Encap string `vici:"encap"`
Mobike string `vici:"mobike"`
DPD string `vici:"dpd_delay"`
Rekey string `vici:"rekey_time"`
}
type shared struct {
ID string `vici:"id"`
Type string `vici:"type"`
Data string `vici:"data"`
Owners []string `vici:"owners"`
}
func dial() (*vici.Session, error) {
return vici.NewSession(vici.WithDialContext(func(ctx context.Context, network, address string) (net.Conn, error) {
conn, err := (&net.Dialer{Timeout: 2 * time.Second}).DialContext(ctx, network, address)
if err == nil {
err = conn.SetDeadline(time.Now().Add(5 * time.Second))
if err != nil {
conn.Close()
}
}
return conn, err
}))
}
func settings(wan string, p config.Peer, index int) connection {
id := strconv.Itoa(42 + index)
return connection{
Version: "2", Remote: []string{p.Address}, Proposals: []string{"aes256-sha256-modp2048"},
Local: auth{Method: "psk", ID: wan}, Peer: auth{Method: "psk", ID: p.Address}, Encap: "yes", Mobike: "no", DPD: "10s", Rekey: "7h",
Children: map[string]child{peerName(index): {Local: []string{"0.0.0.0/0"}, Remote: []string{"0.0.0.0/0"}, Proposals: []string{"aes256gcm16-modp2048"}, In: id, Out: id, Rekey: "50m", DPD: "restart"}},
}
}
func peerName(index int) string { return fmt.Sprintf("short%d", index) }
func load(wan string, p config.Peer, index int) error {
s, err := dial()
if err != nil {
return err
}
defer s.Close()
m, err := vici.MarshalMessage(shared{ID: peerName(index), Type: "IKE", Data: p.Secret, Owners: []string{wan, p.Address}})
if err != nil {
return err
}
if _, err = s.CommandRequest("load-shared", m); err != nil {
return err
}
m, err = vici.MarshalMessage(map[string]connection{peerName(index): settings(wan, p, index)})
if err != nil {
return err
}
_, err = s.CommandRequest("load-conn", m)
return err
}
func initiate(index int) error {
s, err := dial()
if err != nil {
return err
}
defer s.Close()
m := vici.NewMessage()
m.Set("child", peerName(index))
m.Set("timeout", "-1")
_, err = s.CommandRequest("initiate", m)
return err
}
func installed() ([]bool, error) {
s, err := dial()
if err != nil {
return nil, err
}
defer s.Close()
messages, err := s.StreamedCommandRequest("list-sas", "list-sa", nil)
if err != nil {
return nil, err
}
return livePeers(messages), nil
}
func livePeers(messages []*vici.Message) []bool {
up := make([]bool, 2)
for _, m := range messages {
for index := range up {
conn, ok := m.Get(peerName(index)).(*vici.Message)
if !ok {
continue
}
if conn.Get("state") != "ESTABLISHED" {
continue
}
children, ok := conn.Get("child-sas").(*vici.Message)
if !ok {
continue
}
for _, name := range children.Keys() {
c, ok := children.Get(name).(*vici.Message)
if ok && c.Get("state") == "INSTALLED" {
up[index] = true
}
}
}
}
return up
}
func returnPeer(up []bool) int {
for index, live := range up {
if live {
return index
}
}
return -1
}

33
services/short/tunnel/vici_test.go Normal file
View file

@ -0,0 +1,33 @@
package tunnel
import (
"testing"
"github.com/strongswan/govici/vici"
"quorate.dev/services/short/config"
)
func TestReturnPeer(t *testing.T) {
sa := func(index int, state, childState string) *vici.Message {
m, err := vici.MarshalMessage(map[string]any{peerName(index): map[string]any{"state": state, "child-sas": map[string]any{"child": map[string]any{"state": childState}}}})
if err != nil {
t.Fatal(err)
}
return m
}
both := []*vici.Message{sa(0, "ESTABLISHED", "INSTALLED"), sa(1, "ESTABLISHED", "INSTALLED")}
if got := returnPeer(livePeers(both)); got != 0 {
t.Fatal(got)
}
if got := returnPeer(livePeers(both[1:])); got != 1 {
t.Fatal(got)
}
if got := returnPeer(livePeers([]*vici.Message{sa(0, "CONNECTING", "INSTALLED"), sa(1, "ESTABLISHED", "REKEYING")})); got != -1 {
t.Fatal(got)
}
first := settings("192.0.2.1", config.Peer{Address: "192.0.2.2"}, 0)
second := settings("192.0.2.1", config.Peer{Address: "192.0.2.3"}, 1)
if first.Children["short0"].In == second.Children["short1"].In {
t.Fatal("overlapping XFRM policies")
}
}