ce7cb702218693d08ea6d83464d2457c9250027c
This commit is contained in:
commit
34c5f8963c
1 changed files with 145336 additions and 0 deletions
419
services/oci-l4-relay-wan-controller/main.go
Normal file
419
services/oci-l4-relay-wan-controller/main.go
Normal file
|
|
@ -0,0 +1,419 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1"
|
||||
"k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/leaderelection"
|
||||
"k8s.io/client-go/tools/leaderelection/resourcelock"
|
||||
)
|
||||
|
||||
const leaderLockName = "wan-controller"
|
||||
|
||||
type opnsenseClient struct {
|
||||
baseURL string
|
||||
key string
|
||||
secret string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
type interfaceOverview struct {
|
||||
Rows []interfaceRow `json:"rows"`
|
||||
}
|
||||
|
||||
type interfaceRow struct {
|
||||
Identifier string `json:"identifier"`
|
||||
Description string `json:"description"`
|
||||
IPv4 []struct {
|
||||
Address string `json:"ipaddr"`
|
||||
} `json:"ipv4"`
|
||||
}
|
||||
|
||||
func (c *opnsenseClient) wanIPv4(ctx context.Context, selector string) (net.IP, error) {
|
||||
endpoint := strings.TrimRight(c.baseURL, "/") + "/api/interfaces/overview/interfaces_info"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.SetBasicAuth(c.key, c.secret)
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("OPNsense returned HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var overview interfaceOverview
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&overview); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var selected *interfaceRow
|
||||
for index := range overview.Rows {
|
||||
row := &overview.Rows[index]
|
||||
if row.Identifier != selector && row.Description != selector {
|
||||
continue
|
||||
}
|
||||
if selected != nil {
|
||||
return nil, fmt.Errorf("OPNsense interface selector %q is ambiguous", selector)
|
||||
}
|
||||
selected = row
|
||||
}
|
||||
if selected == nil {
|
||||
return nil, fmt.Errorf("OPNsense interface selector %q did not match", selector)
|
||||
}
|
||||
for _, address := range selected.IPv4 {
|
||||
if ip := parseInterfaceIPv4(address.Address); validPublicIPv4(ip) {
|
||||
return ip.To4(), nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("OPNsense interface %q contains no public IPv4", selector)
|
||||
}
|
||||
|
||||
func parseInterfaceIPv4(address string) net.IP {
|
||||
if ip := net.ParseIP(address); ip != nil {
|
||||
return ip
|
||||
}
|
||||
ip, _, err := net.ParseCIDR(address)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
func validPublicIPv4(ip net.IP) bool {
|
||||
return ip != nil && ip.To4() != nil && !ip.IsPrivate() && !ip.IsLoopback() &&
|
||||
!ip.IsUnspecified() && !ip.IsMulticast() && !ip.IsLinkLocalUnicast()
|
||||
}
|
||||
|
||||
type sourceReconciler interface {
|
||||
Reconcile(ctx context.Context, source net.IP) error
|
||||
}
|
||||
|
||||
func poll(ctx context.Context, client *opnsenseClient, selector string, interval time.Duration, minimumObservations int, reconcile sourceReconciler) error {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
var applied, candidate string
|
||||
observations := 0
|
||||
for {
|
||||
ip, err := client.wanIPv4(ctx, selector)
|
||||
if err != nil {
|
||||
slog.Error("poll OPNsense WAN", "error", err)
|
||||
candidate = ""
|
||||
observations = 0
|
||||
} else {
|
||||
current := ip.String()
|
||||
if current == applied {
|
||||
candidate = ""
|
||||
observations = 0
|
||||
} else {
|
||||
if current != candidate {
|
||||
candidate = current
|
||||
observations = 1
|
||||
slog.Info("observed new OPNsense WAN candidate", "source", current, "observations", observations)
|
||||
} else {
|
||||
observations++
|
||||
}
|
||||
if observations >= minimumObservations {
|
||||
if err := reconcile.Reconcile(ctx, ip); err != nil {
|
||||
slog.Error("reconcile OCI WireGuard source", "source", current, "error", err)
|
||||
} else {
|
||||
applied = current
|
||||
candidate = ""
|
||||
observations = 0
|
||||
slog.Info("reconciled OCI WireGuard source", "source", current)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
slog.Error("controller stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context) error {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
restConfig, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load Kubernetes client config: %w", err)
|
||||
}
|
||||
kubeClient, err := coordinationv1.NewForConfig(restConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create Kubernetes client: %w", err)
|
||||
}
|
||||
identity, err := os.Hostname()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read pod hostname: %w", err)
|
||||
}
|
||||
namespace := os.Getenv("POD_NAMESPACE")
|
||||
if namespace == "" {
|
||||
return errors.New("POD_NAMESPACE is required")
|
||||
}
|
||||
|
||||
electionCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
controllerErr := make(chan error, 1)
|
||||
elector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{
|
||||
Lock: &resourcelock.LeaseLock{
|
||||
LeaseMeta: metav1.ObjectMeta{Name: leaderLockName, Namespace: namespace},
|
||||
Client: kubeClient,
|
||||
LockConfig: resourcelock.ResourceLockConfig{
|
||||
Identity: identity,
|
||||
},
|
||||
},
|
||||
LeaseDuration: 30 * time.Second,
|
||||
RenewDeadline: 20 * time.Second,
|
||||
RetryPeriod: 5 * time.Second,
|
||||
ReleaseOnCancel: true,
|
||||
Name: leaderLockName,
|
||||
Callbacks: leaderelection.LeaderCallbacks{
|
||||
OnStartedLeading: func(leaderCtx context.Context) {
|
||||
controllerErr <- runLeader(leaderCtx, cfg)
|
||||
cancel()
|
||||
},
|
||||
OnStoppedLeading: func() {
|
||||
if ctx.Err() == nil {
|
||||
select {
|
||||
case controllerErr <- errors.New("leadership lost"):
|
||||
default:
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("configure leader election: %w", err)
|
||||
}
|
||||
elector.Run(electionCtx)
|
||||
select {
|
||||
case err := <-controllerErr:
|
||||
return err
|
||||
default:
|
||||
return electionCtx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func runLeader(ctx context.Context, cfg config) error {
|
||||
reconciler, err := newOCIReconciler(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
httpClient, err := newOPNsenseHTTPClient(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := &opnsenseClient{
|
||||
baseURL: cfg.opnsenseURL,
|
||||
key: cfg.opnsenseKey,
|
||||
secret: cfg.opnsenseSecret,
|
||||
http: httpClient,
|
||||
}
|
||||
return poll(ctx, client, cfg.opnsenseInterface, cfg.pollInterval, cfg.minimumObservations, reconciler)
|
||||
}
|
||||
|
||||
type config struct {
|
||||
opnsenseURL string
|
||||
opnsenseInterface string
|
||||
opnsenseKey string
|
||||
opnsenseSecret string
|
||||
ociTenancy string
|
||||
ociUser string
|
||||
ociFingerprint string
|
||||
ociPrivateKey string
|
||||
ociRegion string
|
||||
networkSecurityGroup string
|
||||
securityRule string
|
||||
wireguardPort int
|
||||
pollInterval time.Duration
|
||||
minimumObservations int
|
||||
requestTimeout time.Duration
|
||||
opnsenseCACert string
|
||||
wireguardURL string
|
||||
recoveryTimeout time.Duration
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
read := func(name string) (string, error) {
|
||||
path := os.Getenv(name)
|
||||
if path == "" {
|
||||
return "", fmt.Errorf("%s is required", name)
|
||||
}
|
||||
value, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s: %w", name, err)
|
||||
}
|
||||
return strings.TrimSpace(string(value)), nil
|
||||
}
|
||||
opnsenseKey, err := read("OPNSENSE_KEY_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
opnsenseSecret, err := read("OPNSENSE_SECRET_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
opnsenseCACert, err := read("OPNSENSE_CA_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
ociTenancy, err := read("OCI_TENANCY_OCID_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
ociUser, err := read("OCI_USER_OCID_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
ociFingerprint, err := read("OCI_FINGERPRINT_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
ociPrivateKey, err := read("OCI_PRIVATE_KEY_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
ociRegion, err := read("OCI_REGION_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
nsg, err := read("OCI_NSG_OCID_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
rule, err := read("OCI_SECURITY_RULE_ID_PATH")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
port, err := strconv.Atoi(envOrDefault("WIREGUARD_PORT", "51820"))
|
||||
if err != nil || port < 1 || port > 65535 {
|
||||
return config{}, errors.New("WIREGUARD_PORT must be a valid port")
|
||||
}
|
||||
pollInterval, err := time.ParseDuration(envOrDefault("POLL_INTERVAL", "30s"))
|
||||
if err != nil || pollInterval < 5*time.Second {
|
||||
return config{}, errors.New("POLL_INTERVAL must be at least 5s")
|
||||
}
|
||||
minimumObservations, err := strconv.Atoi(envOrDefault("MINIMUM_OBSERVATIONS", "2"))
|
||||
if err != nil || minimumObservations < 1 || minimumObservations > 10 {
|
||||
return config{}, errors.New("MINIMUM_OBSERVATIONS must be between 1 and 10")
|
||||
}
|
||||
opnsenseURL, err := requiredHTTPSOrigin("OPNSENSE_URL")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
wireguardURL, err := requiredMetricsURL("WG_STATUS_URL")
|
||||
if err != nil {
|
||||
return config{}, err
|
||||
}
|
||||
return config{
|
||||
opnsenseURL: opnsenseURL,
|
||||
opnsenseInterface: envOrDefault("OPNSENSE_INTERFACE", "WAN"),
|
||||
opnsenseKey: opnsenseKey,
|
||||
opnsenseSecret: opnsenseSecret,
|
||||
opnsenseCACert: opnsenseCACert,
|
||||
ociTenancy: ociTenancy,
|
||||
ociUser: ociUser,
|
||||
ociFingerprint: ociFingerprint,
|
||||
ociPrivateKey: ociPrivateKey,
|
||||
ociRegion: ociRegion,
|
||||
networkSecurityGroup: nsg,
|
||||
securityRule: rule,
|
||||
wireguardPort: port,
|
||||
pollInterval: pollInterval,
|
||||
minimumObservations: minimumObservations,
|
||||
requestTimeout: 15 * time.Second,
|
||||
wireguardURL: wireguardURL,
|
||||
recoveryTimeout: 90 * time.Second,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newOPNsenseHTTPClient(cfg config) (*http.Client, error) {
|
||||
block, _ := pem.Decode([]byte(cfg.opnsenseCACert))
|
||||
if block == nil || block.Type != "CERTIFICATE" {
|
||||
return nil, errors.New("OPNSENSE_CA_PATH contains no PEM certificate")
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse OPNsense CA certificate: %w", err)
|
||||
}
|
||||
remaining := time.Until(certificate.NotAfter)
|
||||
if remaining <= 0 {
|
||||
return nil, errors.New("OPNsense CA certificate is expired")
|
||||
}
|
||||
if remaining < 90*24*time.Hour {
|
||||
slog.Warn("OPNsense CA certificate expires soon", "not_after", certificate.NotAfter.UTC())
|
||||
}
|
||||
if !certificate.IsCA || certificate.KeyUsage&x509.KeyUsageCertSign == 0 {
|
||||
return nil, errors.New("OPNSENSE_CA_PATH certificate is not a certificate authority")
|
||||
}
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM([]byte(cfg.opnsenseCACert)) {
|
||||
return nil, errors.New("OPNSENSE_CA_PATH contains no PEM certificates")
|
||||
}
|
||||
transport := &http.Transport{TLSClientConfig: &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
RootCAs: roots,
|
||||
}}
|
||||
return &http.Client{Timeout: cfg.requestTimeout, Transport: transport}, nil
|
||||
}
|
||||
|
||||
func requiredHTTPSOrigin(name string) (string, error) {
|
||||
raw := os.Getenv(name)
|
||||
if raw == "" {
|
||||
return "", fmt.Errorf("%s is required", name)
|
||||
}
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || (parsed.Path != "" && parsed.Path != "/") {
|
||||
return "", fmt.Errorf("%s must be an HTTPS origin", name)
|
||||
}
|
||||
return strings.TrimRight(raw, "/"), nil
|
||||
}
|
||||
|
||||
func envOrDefault(name, fallback string) string {
|
||||
if value := os.Getenv(name); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func requiredMetricsURL(name string) (string, error) {
|
||||
value := os.Getenv(name)
|
||||
parsed, err := url.Parse(value)
|
||||
if err != nil || parsed.Scheme != "http" || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path != "/metrics" {
|
||||
return "", fmt.Errorf("%s must be an HTTP metrics endpoint", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
Loading…
Reference in a new issue