quorate/services/oci-l4-relay-wan-controller/main.go

package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
)

const leaderLockName = "wan-controller"

type opnsenseClient struct {
	baseURL string
	key     string
	secret  string
	http    *http.Client
}

type interfaceOverview struct {
	Rows []interfaceRow `json:"rows"`
}

type interfaceRow struct {
	Identifier  string `json:"identifier"`
	Description string `json:"description"`
	IPv4        []struct {
		Address string `json:"ipaddr"`
	} `json:"ipv4"`
}

func (c *opnsenseClient) wanIPv4(ctx context.Context, selector string) (net.IP, error) {
	endpoint := strings.TrimRight(c.baseURL, "/") + "/api/interfaces/overview/interfaces_info"
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
	if err != nil {
		return nil, err
	}
	req.SetBasicAuth(c.key, c.secret)
	resp, err := c.http.Do(req)
	if err != nil {
		return nil, err
	}
	defer func() { _ = resp.Body.Close() }()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("OPNsense returned HTTP %d", resp.StatusCode)
	}
	var overview interfaceOverview
	if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&overview); err != nil {
		return nil, err
	}
	var selected *interfaceRow
	for index := range overview.Rows {
		row := &overview.Rows[index]
		if row.Identifier != selector && row.Description != selector {
			continue
		}
		if selected != nil {
			return nil, fmt.Errorf("OPNsense interface selector %q is ambiguous", selector)
		}
		selected = row
	}
	if selected == nil {
		return nil, fmt.Errorf("OPNsense interface selector %q did not match", selector)
	}
	for _, address := range selected.IPv4 {
		if ip := parseInterfaceIPv4(address.Address); validPublicIPv4(ip) {
			return ip.To4(), nil
		}
	}
	return nil, fmt.Errorf("OPNsense interface %q contains no public IPv4", selector)
}

func parseInterfaceIPv4(address string) net.IP {
	if ip := net.ParseIP(address); ip != nil {
		return ip
	}
	ip, _, err := net.ParseCIDR(address)
	if err != nil {
		return nil
	}
	return ip
}

func validPublicIPv4(ip net.IP) bool {
	return ip != nil && ip.To4() != nil && !ip.IsPrivate() && !ip.IsLoopback() &&
		!ip.IsUnspecified() && !ip.IsMulticast() && !ip.IsLinkLocalUnicast()
}

type sourceReconciler interface {
	Reconcile(ctx context.Context, source net.IP) error
}

func poll(ctx context.Context, client *opnsenseClient, selector string, interval time.Duration, minimumObservations int, reconcile sourceReconciler) error {
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	var applied, candidate string
	observations := 0
	for {
		ip, err := client.wanIPv4(ctx, selector)
		if err != nil {
			slog.Error("poll OPNsense WAN", "error", err)
			candidate = ""
			observations = 0
		} else {
			current := ip.String()
			if current == applied {
				candidate = ""
				observations = 0
			} else {
				if current != candidate {
					candidate = current
					observations = 1
					slog.Info("observed new OPNsense WAN candidate", "source", current, "observations", observations)
				} else {
					observations++
				}
				if observations >= minimumObservations {
					if err := reconcile.Reconcile(ctx, ip); err != nil {
						slog.Error("reconcile OCI WireGuard source", "source", current, "error", err)
					} else {
						applied = current
						candidate = ""
						observations = 0
						slog.Info("reconciled OCI WireGuard source", "source", current)
					}
				}
			}
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
		}
	}
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()
	if err := run(ctx); err != nil && !errors.Is(err, context.Canceled) {
		slog.Error("controller stopped", "error", err)
		os.Exit(1)
	}
}

func run(ctx context.Context) error {
	cfg, err := loadConfig()
	if err != nil {
		return err
	}
	restConfig, err := rest.InClusterConfig()
	if err != nil {
		return fmt.Errorf("load Kubernetes client config: %w", err)
	}
	kubeClient, err := coordinationv1.NewForConfig(restConfig)
	if err != nil {
		return fmt.Errorf("create Kubernetes client: %w", err)
	}
	identity, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("read pod hostname: %w", err)
	}
	namespace := os.Getenv("POD_NAMESPACE")
	if namespace == "" {
		return errors.New("POD_NAMESPACE is required")
	}

	electionCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	controllerErr := make(chan error, 1)
	elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
		Lock: &resourcelock.LeaseLock{
			LeaseMeta: metav1.ObjectMeta{Name: leaderLockName, Namespace: namespace},
			Client:    kubeClient,
			LockConfig: resourcelock.ResourceLockConfig{
				Identity: identity,
			},
		},
		LeaseDuration:   30 * time.Second,
		RenewDeadline:   20 * time.Second,
		RetryPeriod:     5 * time.Second,
		ReleaseOnCancel: true,
		Name:            leaderLockName,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: func(leaderCtx context.Context) {
				controllerErr <- runLeader(leaderCtx, cfg)
				cancel()
			},
			OnStoppedLeading: func() {
				if ctx.Err() == nil {
					select {
					case controllerErr <- errors.New("leadership lost"):
					default:
					}
				}
			},
		},
	})
	if err != nil {
		return fmt.Errorf("configure leader election: %w", err)
	}
	elector.Run(electionCtx)
	select {
	case err := <-controllerErr:
		return err
	default:
		return electionCtx.Err()
	}
}

func runLeader(ctx context.Context, cfg config) error {
	reconciler, err := newOCIReconciler(cfg)
	if err != nil {
		return err
	}
	httpClient, err := newOPNsenseHTTPClient(cfg)
	if err != nil {
		return err
	}
	client := &opnsenseClient{
		baseURL: cfg.opnsenseURL,
		key:     cfg.opnsenseKey,
		secret:  cfg.opnsenseSecret,
		http:    httpClient,
	}
	return poll(ctx, client, cfg.opnsenseInterface, cfg.pollInterval, cfg.minimumObservations, reconciler)
}

type config struct {
	opnsenseURL          string
	opnsenseInterface    string
	opnsenseKey          string
	opnsenseSecret       string
	ociTenancy           string
	ociUser              string
	ociFingerprint       string
	ociPrivateKey        string
	ociRegion            string
	networkSecurityGroup string
	securityRule         string
	wireguardPort        int
	pollInterval         time.Duration
	minimumObservations  int
	requestTimeout       time.Duration
	opnsenseCACert       string
	wireguardURL         string
	recoveryTimeout      time.Duration
}

func loadConfig() (config, error) {
	read := func(name string) (string, error) {
		path := os.Getenv(name)
		if path == "" {
			return "", fmt.Errorf("%s is required", name)
		}
		value, err := os.ReadFile(path)
		if err != nil {
			return "", fmt.Errorf("read %s: %w", name, err)
		}
		return strings.TrimSpace(string(value)), nil
	}
	opnsenseKey, err := read("OPNSENSE_KEY_PATH")
	if err != nil {
		return config{}, err
	}
	opnsenseSecret, err := read("OPNSENSE_SECRET_PATH")
	if err != nil {
		return config{}, err
	}
	opnsenseCACert, err := read("OPNSENSE_CA_PATH")
	if err != nil {
		return config{}, err
	}
	ociTenancy, err := read("OCI_TENANCY_OCID_PATH")
	if err != nil {
		return config{}, err
	}
	ociUser, err := read("OCI_USER_OCID_PATH")
	if err != nil {
		return config{}, err
	}
	ociFingerprint, err := read("OCI_FINGERPRINT_PATH")
	if err != nil {
		return config{}, err
	}
	ociPrivateKey, err := read("OCI_PRIVATE_KEY_PATH")
	if err != nil {
		return config{}, err
	}
	ociRegion, err := read("OCI_REGION_PATH")
	if err != nil {
		return config{}, err
	}
	nsg, err := read("OCI_NSG_OCID_PATH")
	if err != nil {
		return config{}, err
	}
	rule, err := read("OCI_SECURITY_RULE_ID_PATH")
	if err != nil {
		return config{}, err
	}
	port, err := strconv.Atoi(envOrDefault("WIREGUARD_PORT", "51820"))
	if err != nil || port < 1 || port > 65535 {
		return config{}, errors.New("WIREGUARD_PORT must be a valid port")
	}
	pollInterval, err := time.ParseDuration(envOrDefault("POLL_INTERVAL", "30s"))
	if err != nil || pollInterval < 5*time.Second {
		return config{}, errors.New("POLL_INTERVAL must be at least 5s")
	}
	minimumObservations, err := strconv.Atoi(envOrDefault("MINIMUM_OBSERVATIONS", "2"))
	if err != nil || minimumObservations < 1 || minimumObservations > 10 {
		return config{}, errors.New("MINIMUM_OBSERVATIONS must be between 1 and 10")
	}
	opnsenseURL, err := requiredHTTPSOrigin("OPNSENSE_URL")
	if err != nil {
		return config{}, err
	}
	wireguardURL, err := requiredMetricsURL("WG_STATUS_URL")
	if err != nil {
		return config{}, err
	}
	return config{
		opnsenseURL:          opnsenseURL,
		opnsenseInterface:    envOrDefault("OPNSENSE_INTERFACE", "WAN"),
		opnsenseKey:          opnsenseKey,
		opnsenseSecret:       opnsenseSecret,
		opnsenseCACert:       opnsenseCACert,
		ociTenancy:           ociTenancy,
		ociUser:              ociUser,
		ociFingerprint:       ociFingerprint,
		ociPrivateKey:        ociPrivateKey,
		ociRegion:            ociRegion,
		networkSecurityGroup: nsg,
		securityRule:         rule,
		wireguardPort:        port,
		pollInterval:         pollInterval,
		minimumObservations:  minimumObservations,
		requestTimeout:       15 * time.Second,
		wireguardURL:         wireguardURL,
		recoveryTimeout:      90 * time.Second,
	}, nil
}

func newOPNsenseHTTPClient(cfg config) (*http.Client, error) {
	block, _ := pem.Decode([]byte(cfg.opnsenseCACert))
	if block == nil || block.Type != "CERTIFICATE" {
		return nil, errors.New("OPNSENSE_CA_PATH contains no PEM certificate")
	}
	certificate, err := x509.ParseCertificate(block.Bytes)
	if err != nil {
		return nil, fmt.Errorf("parse OPNsense CA certificate: %w", err)
	}
	remaining := time.Until(certificate.NotAfter)
	if remaining <= 0 {
		return nil, errors.New("OPNsense CA certificate is expired")
	}
	if remaining < 90*24*time.Hour {
		slog.Warn("OPNsense CA certificate expires soon", "not_after", certificate.NotAfter.UTC())
	}
	if !certificate.IsCA || certificate.KeyUsage&x509.KeyUsageCertSign == 0 {
		return nil, errors.New("OPNSENSE_CA_PATH certificate is not a certificate authority")
	}
	roots := x509.NewCertPool()
	if !roots.AppendCertsFromPEM([]byte(cfg.opnsenseCACert)) {
		return nil, errors.New("OPNSENSE_CA_PATH contains no PEM certificates")
	}
	transport := &http.Transport{TLSClientConfig: &tls.Config{
		MinVersion: tls.VersionTLS12,
		RootCAs:    roots,
	}}
	return &http.Client{Timeout: cfg.requestTimeout, Transport: transport}, nil
}

func requiredHTTPSOrigin(name string) (string, error) {
	raw := os.Getenv(name)
	if raw == "" {
		return "", fmt.Errorf("%s is required", name)
	}
	parsed, err := url.Parse(raw)
	if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || (parsed.Path != "" && parsed.Path != "/") {
		return "", fmt.Errorf("%s must be an HTTPS origin", name)
	}
	return strings.TrimRight(raw, "/"), nil
}

func envOrDefault(name, fallback string) string {
	if value := os.Getenv(name); value != "" {
		return value
	}
	return fallback
}

func requiredMetricsURL(name string) (string, error) {
	value := os.Getenv(name)
	parsed, err := url.Parse(value)
	if err != nil || parsed.Scheme != "http" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path != "/metrics" {
		return "", fmt.Errorf("%s must be an HTTP metrics endpoint", name)
	}
	return value, nil
}