270 lines
11 KiB
Go
270 lines
11 KiB
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"net"
|
||
|
|
"net/http"
|
||
|
|
"net/http/httptest"
|
||
|
|
"net/netip"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
"sync/atomic"
|
||
|
|
"syscall"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestParseRecordsSortsAndBoundsEvidence(t *testing.T) {
|
||
|
|
now := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC)
|
||
|
|
contents := []byte(`{"nftables":[{"metainfo":{"json_schema_version":1}},{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.2","expires":598,"counter":{"packets":4,"bytes":160}}},{"elem":{"val":"198.18.0.1","expires":599,"counter":{"packets":1,"bytes":40}}}]}}]}`)
|
||
|
|
records, total, err := parseRecords(now, contents, "smtp_syn_quarantine_v4", 1)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if total != 2 || len(records) != 1 || records[0].Address != "198.18.0.1" || records[0].Count != 1 || !records[0].ExpiresAt.Equal(now.Add(599*time.Second)) {
|
||
|
|
t.Fatalf("unexpected records %#v", records)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestParseRecordsRequiresCanonicalIPv4(t *testing.T) {
|
||
|
|
contents := []byte(`{"nftables":[{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"2001:db8::1","expires":598,"counter":{"packets":1}}}]}}]}`)
|
||
|
|
if _, _, err := parseRecords(time.Now(), contents, "smtp_syn_quarantine_v4", 1); err == nil {
|
||
|
|
t.Fatal("IPv6 evidence was accepted")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestParseDropMetricsRequiresEveryBoundedCounter(t *testing.T) {
|
||
|
|
items := make([]string, 0, len(dropMetricSpecs))
|
||
|
|
for index, spec := range dropMetricSpecs {
|
||
|
|
items = append(items, fmt.Sprintf(`{"counter":{"name":%q,"packets":%d}}`, spec.Name, index+1))
|
||
|
|
}
|
||
|
|
metrics, err := parseDropMetrics([]byte(`{"nftables":[`+strings.Join(items, ",")+`]}`), true)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if len(metrics) != len(dropMetricSpecs) || metrics[0].Service != "web" || metrics[0].Reason != "admission" || metrics[0].Packets != 1 {
|
||
|
|
t.Fatalf("unexpected metrics %#v", metrics)
|
||
|
|
}
|
||
|
|
if _, err := parseDropMetrics([]byte(`{"nftables":[]}`), true); err == nil {
|
||
|
|
t.Fatal("missing counters were accepted")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestMetricsHandlerExposesBoundedEvidenceAndDrops(t *testing.T) {
|
||
|
|
generated := time.Date(2026, 7, 30, 1, 0, 0, 0, time.UTC)
|
||
|
|
var latest atomic.Pointer[metricsSnapshot]
|
||
|
|
latest.Store(&metricsSnapshot{
|
||
|
|
GeneratedAt: generated,
|
||
|
|
Services: []serviceMetrics{
|
||
|
|
{Service: "web", Records: []evidenceRecord{{Address: "198.18.0.1", Count: 2, ExpiresAt: generated.Add(time.Minute)}}, Total: 2},
|
||
|
|
{Service: "smtp", Records: []evidenceRecord{{Address: "198.18.0.2", Count: 4, ExpiresAt: generated.Add(10 * time.Minute)}}, Total: 1},
|
||
|
|
},
|
||
|
|
Drops: []dropMetric{{Service: "web", Reason: "source_rate", Packets: 7}},
|
||
|
|
Health: relayHealth{ConntrackCount: 12, ConntrackMaximum: 262144, NftablesActive: true, WebAllowActive: true, WebAllowCached: true, WireGuardActive: true},
|
||
|
|
QdiscDrops: []qdiscMetric{{Channel: "web", Packets: 3}},
|
||
|
|
TelemetryAt: generated,
|
||
|
|
})
|
||
|
|
var conntrackState conntrackMetrics
|
||
|
|
conntrackState.Up.Store(true)
|
||
|
|
conntrackState.Emitted.Store(2)
|
||
|
|
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||
|
|
response := httptest.NewRecorder()
|
||
|
|
metricsHandler{Conntrack: &conntrackState, Latest: &latest, SMTPConfigured: true}.ServeHTTP(response, request)
|
||
|
|
if response.Code != http.StatusOK {
|
||
|
|
t.Fatalf("status = %d", response.Code)
|
||
|
|
}
|
||
|
|
for _, expected := range []string{
|
||
|
|
`oci_l4_relay_tcp_quarantine_packets{service="web",address="198.18.0.1"} 2`,
|
||
|
|
`oci_l4_relay_tcp_quarantine_expires_timestamp_seconds{service="smtp",address="198.18.0.2"} 1785373800`,
|
||
|
|
`oci_l4_relay_tcp_evidence_records{service="smtp"} 1`,
|
||
|
|
`oci_l4_relay_tcp_evidence_truncated{service="web"} 1`,
|
||
|
|
`oci_l4_relay_tcp_evidence_generated_timestamp_seconds 1785373200`,
|
||
|
|
`oci_l4_relay_nftables_dropped_packets_total{service="web",reason="source_rate"} 7`,
|
||
|
|
`oci_l4_relay_qdisc_dropped_packets_total{channel="web"} 3`,
|
||
|
|
`oci_l4_relay_component_healthy{component="web_allow"} 1`,
|
||
|
|
`oci_l4_relay_conntrack_entries 12`,
|
||
|
|
`oci_l4_relay_conntrack_observer_events_total{outcome="emitted"} 2`,
|
||
|
|
`oci_l4_relay_conntrack_observer_up 1`,
|
||
|
|
`oci_l4_relay_smtp_deny_configured 1`,
|
||
|
|
} {
|
||
|
|
if !strings.Contains(response.Body.String(), expected) {
|
||
|
|
t.Errorf("metrics do not contain %q", expected)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestRunCollectsAndServesMetrics(t *testing.T) {
|
||
|
|
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||
|
|
if err != nil {
|
||
|
|
if errors.Is(err, syscall.EPERM) {
|
||
|
|
t.Skip("network namespace does not permit listeners")
|
||
|
|
}
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
listenAddress := listener.Addr().String()
|
||
|
|
listener.Close()
|
||
|
|
items := make([]string, 0, len(dropMetricSpecs))
|
||
|
|
for index, spec := range dropMetricSpecs {
|
||
|
|
items = append(items, fmt.Sprintf(`{"counter":{"name":%q,"packets":%d}}`, spec.Name, index+1))
|
||
|
|
}
|
||
|
|
script := fmt.Sprintf(`#!/bin/sh
|
||
|
|
case "$*" in
|
||
|
|
*smtp_syn_quarantine_v4*) printf '%%s' '{"nftables":[{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.2","expires":599,"counter":{"packets":4}}}]}}]}' ;;
|
||
|
|
*syn_quarantine_v4*) printf '%%s' '{"nftables":[{"set":{"family":"inet","name":"syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.1","expires":59,"counter":{"packets":2}}}]}}]}' ;;
|
||
|
|
*list\ counters*) printf '%%s' '{"nftables":[%s]}' ;;
|
||
|
|
*) exit 1 ;;
|
||
|
|
esac
|
||
|
|
`, strings.Join(items, ","))
|
||
|
|
nftPath := filepath.Join(t.TempDir(), "nft")
|
||
|
|
if err := os.WriteFile(nftPath, []byte(script), 0700); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
snapshotPath := filepath.Join(t.TempDir(), "current.json")
|
||
|
|
tcPath := filepath.Join(t.TempDir(), "tc")
|
||
|
|
if err := os.WriteFile(tcPath, []byte("#!/bin/sh\nprintf '%s' '[{\"parent\":\"1:10\",\"drops\":3}]'\n"), 0700); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
telemetryPath := filepath.Join(t.TempDir(), "telemetry.json")
|
||
|
|
if err := os.WriteFile(telemetryPath, []byte(`{"generated_at":"2026-07-30T01:00:00Z","status":{"conntrack_count":12,"conntrack_maximum":262144,"handshake_age_seconds":10,"nftables_active":true,"smtp_deny_active":true,"smtp_deny_active_source":"current","smtp_deny_bundle_age_seconds":60,"smtp_deny_bundle_remaining_seconds":600,"smtp_deny_cached":true,"smtp_deny_policy_valid":true,"smtp_deny_refresh_completed_age_seconds":30,"smtp_deny_refresh_successful":true,"smtp_deny_remote_current_age_seconds":30,"web_allow_active":true,"web_allow_cached":true,"wireguard_active":true}}`), 0600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
previousObserver := startConntrackObserver
|
||
|
|
startConntrackObserver = func(ctx context.Context, _ relayConfig, _ *conntrackMetrics, _ io.Writer) error {
|
||
|
|
<-ctx.Done()
|
||
|
|
return ctx.Err()
|
||
|
|
}
|
||
|
|
defer func() { startConntrackObserver = previousObserver }()
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
done := make(chan error, 1)
|
||
|
|
go func() {
|
||
|
|
done <- run(ctx, validRelayConfig(listenAddress), nftPath, tcPath, telemetryPath, snapshotPath, false)
|
||
|
|
}()
|
||
|
|
var body []byte
|
||
|
|
client := http.Client{Timeout: time.Second}
|
||
|
|
for attempt := 0; attempt < 50; attempt++ {
|
||
|
|
response, err := client.Get("http://" + listenAddress + "/metrics")
|
||
|
|
if err == nil {
|
||
|
|
body, err = io.ReadAll(response.Body)
|
||
|
|
response.Body.Close()
|
||
|
|
if err == nil && response.StatusCode == http.StatusOK {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
time.Sleep(10 * time.Millisecond)
|
||
|
|
}
|
||
|
|
cancel()
|
||
|
|
if err := <-done; !errors.Is(err, context.Canceled) {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
for _, expected := range []string{
|
||
|
|
`oci_l4_relay_tcp_quarantine_packets{service="web",address="198.18.0.1"} 2`,
|
||
|
|
`oci_l4_relay_tcp_quarantine_packets{service="smtp",address="198.18.0.2"} 4`,
|
||
|
|
`oci_l4_relay_nftables_dropped_packets_total{service="authenticated_mail",reason="aggregate_packet"} 14`,
|
||
|
|
} {
|
||
|
|
if !strings.Contains(string(body), expected) {
|
||
|
|
t.Errorf("metrics do not contain %q", expected)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
contents, err := os.ReadFile(snapshotPath)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if !strings.Contains(string(contents), `"address":"198.18.0.2"`) || strings.Contains(string(contents), `"address":"198.18.0.1"`) {
|
||
|
|
t.Fatalf("unexpected SMTP snapshot %s", contents)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestWriteSnapshotReplacesCurrentFile(t *testing.T) {
|
||
|
|
path := filepath.Join(t.TempDir(), "current.json")
|
||
|
|
if err := os.WriteFile(path, []byte("old"), 0600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
value := snapshot{SchemaVersion: 1, Records: []evidenceRecord{}}
|
||
|
|
if err := writeSnapshot(path, value); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
contents, err := os.ReadFile(path)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
var actual snapshot
|
||
|
|
if err := json.Unmarshal(contents, &actual); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if actual.SchemaVersion != 1 || actual.Records == nil {
|
||
|
|
t.Fatalf("unexpected snapshot %s", contents)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestLoadConfigAcceptsTestedCadences(t *testing.T) {
|
||
|
|
for _, interval := range []int{30, 60} {
|
||
|
|
path := filepath.Join(t.TempDir(), "config.json")
|
||
|
|
contents := relayConfigJSON(interval)
|
||
|
|
if err := os.WriteFile(path, contents, 0600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if _, err := loadConfig(path); err != nil {
|
||
|
|
t.Fatalf("interval %d: %v", interval, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestLoadConfigRejectsUntestedCadence(t *testing.T) {
|
||
|
|
path := filepath.Join(t.TempDir(), "config.json")
|
||
|
|
contents := relayConfigJSON(45)
|
||
|
|
if err := os.WriteFile(path, contents, 0600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if _, err := loadConfig(path); err == nil {
|
||
|
|
t.Fatal("untested cadence was accepted")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestLoadConfigRejectsUnknownField(t *testing.T) {
|
||
|
|
path := filepath.Join(t.TempDir(), "config.json")
|
||
|
|
contents := []byte(strings.TrimSuffix(string(relayConfigJSON(60)), "}") + `,"upload":true}`)
|
||
|
|
if err := os.WriteFile(path, contents, 0600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if _, err := loadConfig(path); err == nil {
|
||
|
|
t.Fatal("unknown field was accepted")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestLoadConfigRejectsNonSpecificMetricsAddress(t *testing.T) {
|
||
|
|
path := filepath.Join(t.TempDir(), "config.json")
|
||
|
|
contents := []byte(strings.Replace(string(relayConfigJSON(60)), "10.40.68.115:19100", "0.0.0.0:19100", 1))
|
||
|
|
if err := os.WriteFile(path, contents, 0600); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if _, err := loadConfig(path); err == nil {
|
||
|
|
t.Fatal("wildcard metrics address was accepted")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func validRelayConfig(listenAddress string) relayConfig {
|
||
|
|
return relayConfig{
|
||
|
|
ConntrackEmissionBurst: 10,
|
||
|
|
ConntrackEmissionRatePerSecond: 1,
|
||
|
|
ConntrackEventBuffer: 4096,
|
||
|
|
ConntrackSampleDenominator: 10,
|
||
|
|
ConntrackServices: []conntrackService{{
|
||
|
|
Name: "web",
|
||
|
|
Ports: []uint16{80, 443},
|
||
|
|
TargetAddress: netip.MustParseAddr("10.104.40.127"),
|
||
|
|
}},
|
||
|
|
MaximumRecords: 4096,
|
||
|
|
MetricsListenAddress: listenAddress,
|
||
|
|
ObservationWindowSeconds: 600,
|
||
|
|
SnapshotIntervalSeconds: 60,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func relayConfigJSON(interval int) []byte {
|
||
|
|
return []byte(fmt.Sprintf(`{"conntrack_emission_burst":10,"conntrack_emission_rate_per_second":1,"conntrack_event_buffer":4096,"conntrack_sample_denominator":10,"conntrack_services":[{"name":"web","ports":[80,443],"target_address":"10.104.40.127"}],"maximum_records":4096,"metrics_listen_address":"10.40.68.115:19100","observation_window_seconds":600,"snapshot_interval_seconds":%d}`, interval))
|
||
|
|
}
|