ms2k/quorate
SHA256
Watch
0
Code Releases Activity

ce7cb702218693d08ea6d83464d2457c9250027c

Browse source
This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
4 changed files with 145336 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

213
services/oci-l4-relay-guest/cmd/smtp-evidence/conntrack.go Normal file
View file

@ -0,0 +1,213 @@
package main
import (
"context"
"encoding/binary"
"encoding/json"
"fmt"
"hash/fnv"
"io"
"net/netip"
"sync/atomic"
"syscall"
"time"
"github.com/ti-mo/conntrack"
"github.com/ti-mo/netfilter"
)
type conntrackMetrics struct {
Emitted atomic.Uint64
Eligible atomic.Uint64
RateDropped atomic.Uint64
SampledOut atomic.Uint64
Up atomic.Bool
}
type conntrackRecord struct {
DurationMilliseconds uint64 `json:"duration_milliseconds"`
EndedAt time.Time `json:"ended_at"`
Original conntrackTuple `json:"original"`
OriginalBytes uint64 `json:"original_bytes"`
OriginalPackets uint64 `json:"original_packets"`
Reply conntrackTuple `json:"reply"`
ReplyBytes uint64 `json:"reply_bytes"`
ReplyPackets uint64 `json:"reply_packets"`
SchemaVersion int `json:"schema_version"`
Service string `json:"service"`
StartedAt time.Time `json:"started_at"`
TCPState string `json:"tcp_state"`
Type string `json:"type"`
}
type conntrackTuple struct {
DestinationAddress string `json:"destination_address"`
DestinationPort uint16 `json:"destination_port"`
SourceAddress string `json:"source_address"`
SourcePort uint16 `json:"source_port"`
}
type tokenBucket struct {
burst float64
last time.Time
rate float64
remaining float64
}
func newTokenBucket(rate, burst int, now time.Time) tokenBucket {
return tokenBucket{burst: float64(burst), last: now, rate: float64(rate), remaining: float64(burst)}
}
func (bucket *tokenBucket) allow(now time.Time) bool {
if now.After(bucket.last) {
bucket.remaining = min(bucket.burst, bucket.remaining+now.Sub(bucket.last).Seconds()*bucket.rate)
bucket.last = now
}
if bucket.remaining < 1 {
return false
}
bucket.remaining--
return true
}
func observeConntrack(ctx context.Context, configuration relayConfig, metrics *conntrackMetrics, output io.Writer) error {
connection, err := conntrack.Dial(nil)
if err != nil {
return err
}
defer connection.Close()
events := make(chan conntrack.Event, configuration.ConntrackEventBuffer)
errors, err := connection.Listen(events, 1, []netfilter.NetlinkGroup{netfilter.GroupCTDestroy})
if err != nil {
return err
}
metrics.Up.Store(true)
defer metrics.Up.Store(false)
go func() {
<-ctx.Done()
connection.Close()
}()
encoder := json.NewEncoder(output)
limiter := newTokenBucket(configuration.ConntrackEmissionRatePerSecond, configuration.ConntrackEmissionBurst, time.Now())
for {
select {
case <-ctx.Done():
return ctx.Err()
case err := <-errors:
return err
case event := <-events:
record, eligible := conntrackRecordForFlow(event.Flow, configuration.ConntrackServices)
if !eligible {
continue
}
metrics.Eligible.Add(1)
if !sampleConntrackFlow(event.Flow, configuration.ConntrackSampleDenominator) {
metrics.SampledOut.Add(1)
continue
}
if !limiter.allow(time.Now()) {
metrics.RateDropped.Add(1)
continue
}
if err := encoder.Encode(record); err != nil {
return err
}
metrics.Emitted.Add(1)
}
}
}
func conntrackRecordForFlow(flow *conntrack.Flow, services []conntrackService) (conntrackRecord, bool) {
if flow == nil || flow.TupleOrig.Proto.Protocol != syscall.IPPROTO_TCP || flow.TupleReply.Proto.Protocol != syscall.IPPROTO_TCP {
return conntrackRecord{}, false
}
service := ""
for _, candidate := range services {
if flow.TupleReply.IP.SourceAddress != candidate.TargetAddress {
continue
}
for _, port := range candidate.Ports {
if flow.TupleOrig.Proto.DestinationPort == port {
service = candidate.Name
break
}
}
}
if service == "" || !validTuple(flow.TupleOrig) || !validTuple(flow.TupleReply) || flow.Timestamp.Start.IsZero() {
return conntrackRecord{}, false
}
endedAt := flow.Timestamp.Stop
if endedAt.IsZero() {
endedAt = time.Now()
}
duration := endedAt.Sub(flow.Timestamp.Start)
if duration < 0 {
return conntrackRecord{}, false
}
state := "unknown"
if flow.ProtoInfo.TCP != nil {
state = tcpState(flow.ProtoInfo.TCP.State)
}
return conntrackRecord{
DurationMilliseconds: uint64(duration.Milliseconds()),
EndedAt: endedAt.UTC(),
Original: recordTuple(flow.TupleOrig),
OriginalBytes: flow.CountersOrig.Bytes,
OriginalPackets: flow.CountersOrig.Packets,
Reply: recordTuple(flow.TupleReply),
ReplyBytes: flow.CountersReply.Bytes,
ReplyPackets: flow.CountersReply.Packets,
SchemaVersion: 1,
Service: service,
StartedAt: flow.Timestamp.Start.UTC(),
TCPState: state,
Type: "oci_l4_relay_conntrack",
}, true
}
func validTuple(tuple conntrack.Tuple) bool {
return tuple.IP.SourceAddress.IsValid() && tuple.IP.DestinationAddress.IsValid() && tuple.Proto.SourcePort != 0 && tuple.Proto.DestinationPort != 0
}
func recordTuple(tuple conntrack.Tuple) conntrackTuple {
return conntrackTuple{
DestinationAddress: tuple.IP.DestinationAddress.String(),
DestinationPort: tuple.Proto.DestinationPort,
SourceAddress: tuple.IP.SourceAddress.String(),
SourcePort: tuple.Proto.SourcePort,
}
}
func sampleConntrackFlow(flow *conntrack.Flow, denominator uint64) bool {
if denominator == 1 {
return true
}
hash := fnv.New64a()
for _, address := range []netip.Addr{
flow.TupleOrig.IP.SourceAddress,
flow.TupleOrig.IP.DestinationAddress,
flow.TupleReply.IP.SourceAddress,
flow.TupleReply.IP.DestinationAddress,
} {
_, _ = hash.Write(address.AsSlice())
}
var values [8]byte
for _, port := range []uint16{
flow.TupleOrig.Proto.SourcePort,
flow.TupleOrig.Proto.DestinationPort,
flow.TupleReply.Proto.SourcePort,
flow.TupleReply.Proto.DestinationPort,
} {
binary.BigEndian.PutUint16(values[:2], port)
_, _ = hash.Write(values[:2])
}
return hash.Sum64()%denominator == 0
}
func tcpState(value uint8) string {
states := []string{"none", "syn_sent", "syn_recv", "established", "fin_wait", "close_wait", "last_ack", "time_wait", "close", "syn_sent2"}
if int(value) >= len(states) {
return fmt.Sprintf("unknown_%d", value)
}
return states[value]
}

63
services/oci-l4-relay-guest/cmd/smtp-evidence/conntrack_test.go Normal file
View file

@ -0,0 +1,63 @@
package main
import (
"net/netip"
"testing"
"time"
"github.com/ti-mo/conntrack"
)
func TestConntrackRecordRequiresDeclaredPortAndReplyTarget(t *testing.T) {
flow := acceptedFlow()
record, ok := conntrackRecordForFlow(&flow, []conntrackService{{
Name: "web",
Ports: []uint16{80, 443},
TargetAddress: netip.MustParseAddr("10.104.40.127"),
}})
if !ok {
t.Fatal("accepted relay flow was not classified")
}
if record.Service != "web" || record.Original.SourceAddress != "198.18.0.1" || record.Reply.SourceAddress != "10.104.40.127" || record.DurationMilliseconds != 1500 || record.TCPState != "time_wait" {
t.Fatalf("unexpected record %#v", record)
}
flow.TupleReply.IP.SourceAddress = netip.MustParseAddr("203.0.113.10")
if _, ok := conntrackRecordForFlow(&flow, []conntrackService{{Name: "web", Ports: []uint16{443}, TargetAddress: netip.MustParseAddr("10.104.40.127")}}); ok {
t.Fatal("outbound connection was classified as accepted relay traffic")
}
}
func TestConntrackSamplingIsDeterministic(t *testing.T) {
flow := acceptedFlow()
first := sampleConntrackFlow(&flow, 10)
for attempt := 0; attempt < 100; attempt++ {
if sampleConntrackFlow(&flow, 10) != first {
t.Fatal("sampling decision changed for an unchanged tuple")
}
}
}
func TestTokenBucketBoundsBurstAndRefills(t *testing.T) {
now := time.Date(2026, 7, 31, 0, 0, 0, 0, time.UTC)
bucket := newTokenBucket(1, 2, now)
if !bucket.allow(now) || !bucket.allow(now) || bucket.allow(now) {
t.Fatal("initial burst was not bounded")
}
if !bucket.allow(now.Add(time.Second)) || bucket.allow(now.Add(time.Second)) {
t.Fatal("sustained rate was not bounded")
}
}
func acceptedFlow() conntrack.Flow {
startedAt := time.Date(2026, 7, 31, 0, 0, 0, 0, time.UTC)
flow := conntrack.NewFlow(6, 0, netip.MustParseAddr("198.18.0.1"), netip.MustParseAddr("10.49.127.43"), 50000, 443, 60, 0)
flow.TupleReply.IP.SourceAddress = netip.MustParseAddr("10.104.40.127")
flow.TupleReply.IP.DestinationAddress = netip.MustParseAddr("198.18.0.1")
flow.TupleReply.Proto.SourcePort = 443
flow.TupleReply.Proto.DestinationPort = 50000
flow.Timestamp = conntrack.Timestamp{Start: startedAt, Stop: startedAt.Add(1500 * time.Millisecond)}
flow.CountersOrig = conntrack.Counter{Packets: 3, Bytes: 180}
flow.CountersReply = conntrack.Counter{Direction: true, Packets: 4, Bytes: 240}
flow.ProtoInfo.TCP = &conntrack.ProtoInfoTCP{State: 7}
return flow
}

618
services/oci-l4-relay-guest/cmd/smtp-evidence/main.go Normal file
View file

@ -0,0 +1,618 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"os"
"os/exec"
"path/filepath"
"sort"
"sync/atomic"
"time"
)
const (
defaultConfigPath = "/etc/oci-l4-relay-smtp-evidence.json"
defaultNftPath = "/usr/sbin/nft"
defaultSnapshotPath = "/var/lib/oci-l4-relay/smtp-evidence/current.json"
defaultTCPath = "/usr/sbin/tc"
defaultTelemetryPath = "/run/oci-l4-relay-telemetry/current.json"
nftablesFamily = "inet"
nftablesTable = "oci_l4_relay"
reason = "syn-rate"
)
var evidenceSets = []evidenceSet{
{Service: "web", Name: "syn_quarantine_v4"},
{Service: "smtp", Name: "smtp_syn_quarantine_v4"},
}
var startConntrackObserver = observeConntrack
var dropMetricSpecs = []dropMetricSpec{
{Name: "web_allow_denied", Service: "web", Reason: "admission"},
{Name: "web_syn_source_dropped", Service: "web", Reason: "source_rate"},
{Name: "web_syn_quarantine_dropped", Service: "web", Reason: "quarantine"},
{Name: "web_syn_aggregate_dropped", Service: "web", Reason: "aggregate_syn"},
{Name: "web_packet_aggregate_dropped", Service: "web", Reason: "aggregate_packet"},
{Name: "smtp_syn_source_dropped", Service: "smtp", Reason: "source_rate"},
{Name: "smtp_syn_quarantine_dropped", Service: "smtp", Reason: "quarantine"},
{Name: "smtp_syn_aggregate_dropped", Service: "smtp", Reason: "aggregate_syn"},
{Name: "smtp_packet_aggregate_dropped", Service: "smtp", Reason: "aggregate_packet"},
{Name: "smtp_denied", Service: "smtp", Reason: "deny_policy", MailOnly: true},
{Name: "auth_syn_source_dropped", Service: "authenticated_mail", Reason: "source_rate"},
{Name: "auth_syn_quarantine_dropped", Service: "authenticated_mail", Reason: "quarantine"},
{Name: "auth_syn_aggregate_dropped", Service: "authenticated_mail", Reason: "aggregate_syn"},
{Name: "auth_packet_aggregate_dropped", Service: "authenticated_mail", Reason: "aggregate_packet"},
}
type relayConfig struct {
ConntrackEmissionBurst int `json:"conntrack_emission_burst"`
ConntrackEmissionRatePerSecond int `json:"conntrack_emission_rate_per_second"`
ConntrackEventBuffer int `json:"conntrack_event_buffer"`
ConntrackSampleDenominator uint64 `json:"conntrack_sample_denominator"`
ConntrackServices []conntrackService `json:"conntrack_services"`
MaximumRecords uint64 `json:"maximum_records"`
MetricsListenAddress string `json:"metrics_listen_address"`
ObservationWindowSeconds int `json:"observation_window_seconds"`
SnapshotIntervalSeconds int `json:"snapshot_interval_seconds"`
}
type conntrackService struct {
Name string `json:"name"`
Ports []uint16 `json:"ports"`
TargetAddress netip.Addr `json:"target_address"`
}
type evidenceSet struct {
Name string
Service string
}
type evidenceRecord struct {
Address string `json:"address"`
Count uint64 `json:"count"`
ExpiresAt time.Time `json:"expires_at"`
}
type snapshot struct {
SchemaVersion int `json:"schema_version"`
GeneratedAt time.Time `json:"generated_at"`
Reason string `json:"reason"`
ObservationWindowSeconds int `json:"observation_window_seconds"`
Records []evidenceRecord `json:"records"`
}
type serviceMetrics struct {
Records []evidenceRecord
Service string
Total uint64
}
type dropMetric struct {
Packets uint64
Reason string
Service string
}
type dropMetricSpec struct {
MailOnly bool
Name string
Reason string
Service string
}
type metricsSnapshot struct {
Drops []dropMetric
GeneratedAt time.Time
Health relayHealth
QdiscDrops []qdiscMetric
Services []serviceMetrics
TelemetryAt time.Time
}
type qdiscMetric struct {
Channel string
Packets uint64
}
type relayHealth struct {
ConntrackCount int64 `json:"conntrack_count"`
ConntrackMaximum int64 `json:"conntrack_maximum"`
HandshakeAgeSeconds int64 `json:"handshake_age_seconds"`
NftablesActive bool `json:"nftables_active"`
SMTPDenyActive bool `json:"smtp_deny_active"`
SMTPDenyActiveSource string `json:"smtp_deny_active_source,omitempty"`
SMTPDenyBundleAgeSeconds int64 `json:"smtp_deny_bundle_age_seconds"`
SMTPDenyBundleRemainingSeconds int64 `json:"smtp_deny_bundle_remaining_seconds"`
SMTPDenyCached bool `json:"smtp_deny_cached"`
SMTPDenyPolicyValid bool `json:"smtp_deny_policy_valid"`
SMTPDenyRefreshCompletedAgeSeconds int64 `json:"smtp_deny_refresh_completed_age_seconds"`
SMTPDenyRefreshSuccessful bool `json:"smtp_deny_refresh_successful"`
SMTPDenyRemoteCurrentAgeSeconds int64 `json:"smtp_deny_remote_current_age_seconds"`
WebAllowActive bool `json:"web_allow_active"`
WebAllowCached bool `json:"web_allow_cached"`
WireGuardActive bool `json:"wireguard_active"`
}
type telemetrySnapshot struct {
GeneratedAt time.Time `json:"generated_at"`
Status relayHealth `json:"status"`
}
type qdiscDocumentItem struct {
Drops uint64 `json:"drops"`
Parent string `json:"parent"`
}
type nftablesSetDocument struct {
Nftables []struct {
Set *struct {
Family string `json:"family"`
Name string `json:"name"`
Table string `json:"table"`
Elem []struct {
Element struct {
Value string `json:"val"`
Expires int64 `json:"expires"`
Counter *struct {
Packets uint64 `json:"packets"`
} `json:"counter"`
} `json:"elem"`
} `json:"elem"`
} `json:"set,omitempty"`
} `json:"nftables"`
}
type nftablesCounterDocument struct {
Nftables []struct {
Counter *struct {
Name string `json:"name"`
Packets uint64 `json:"packets"`
} `json:"counter"`
} `json:"nftables"`
}
func main() {
configPath := flag.String("config", defaultConfigPath, "relay configuration path")
nftPath := flag.String("nft", defaultNftPath, "nft path")
snapshotPath := flag.String("snapshot", defaultSnapshotPath, "snapshot path")
tcPath := flag.String("tc", defaultTCPath, "tc path")
telemetryPath := flag.String("telemetry", defaultTelemetryPath, "telemetry snapshot path")
once := flag.Bool("once", false, "write one snapshot and exit")
flag.Parse()
configuration, err := loadConfig(*configPath)
if err != nil {
fatal(err)
}
if err := run(context.Background(), configuration, *nftPath, *tcPath, *telemetryPath, *snapshotPath, *once); err != nil {
fatal(err)
}
}
func loadConfig(path string) (relayConfig, error) {
contents, err := os.ReadFile(path)
if err != nil {
return relayConfig{}, err
}
var configuration relayConfig
decoder := json.NewDecoder(bytes.NewReader(contents))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&configuration); err != nil {
return relayConfig{}, err
}
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
return relayConfig{}, errors.New("SMTP evidence configuration has trailing data")
}
if configuration.MaximumRecords == 0 || configuration.ObservationWindowSeconds <= 0 {
return relayConfig{}, errors.New("SMTP evidence bounds must be positive")
}
if configuration.ConntrackEmissionBurst <= 0 || configuration.ConntrackEmissionRatePerSecond <= 0 || configuration.ConntrackEventBuffer <= 0 || configuration.ConntrackSampleDenominator == 0 {
return relayConfig{}, errors.New("conntrack observation bounds must be positive")
}
if configuration.ConntrackEventBuffer > 65536 || configuration.ConntrackEmissionBurst > 1000 || configuration.ConntrackEmissionRatePerSecond > 100 {
return relayConfig{}, errors.New("conntrack observation bounds exceed tested limits")
}
if len(configuration.ConntrackServices) == 0 {
return relayConfig{}, errors.New("at least one conntrack service is required")
}
serviceNames := make(map[string]bool)
servicePorts := make(map[uint16]bool)
for _, service := range configuration.ConntrackServices {
if service.Name == "" || serviceNames[service.Name] || !service.TargetAddress.Is4() || service.TargetAddress.IsUnspecified() || len(service.Ports) == 0 {
return relayConfig{}, errors.New("conntrack service is invalid")
}
serviceNames[service.Name] = true
for _, port := range service.Ports {
if port == 0 || servicePorts[port] {
return relayConfig{}, errors.New("conntrack service ports must be nonzero and unique")
}
servicePorts[port] = true
}
}
interval := configuration.SnapshotIntervalSeconds
if interval != 30 && interval != 60 {
return relayConfig{}, errors.New("SMTP evidence snapshot interval must be 30 or 60 seconds")
}
listenAddress, err := netip.ParseAddrPort(configuration.MetricsListenAddress)
if err != nil || !listenAddress.Addr().Is4() || listenAddress.Addr().IsUnspecified() || listenAddress.Port() == 0 || listenAddress.String() != configuration.MetricsListenAddress {
return relayConfig{}, errors.New("metrics listen address must contain a canonical IPv4 address and port")
}
return configuration, nil
}
func run(ctx context.Context, configuration relayConfig, nftPath, tcPath, telemetryPath, snapshotPath string, once bool) error {
var latest atomic.Pointer[metricsSnapshot]
if err := collect(ctx, configuration, nftPath, tcPath, telemetryPath, snapshotPath, &latest); err != nil {
return err
}
if once {
return nil
}
listener, err := net.Listen("tcp4", configuration.MetricsListenAddress)
if err != nil {
return err
}
var conntrackState conntrackMetrics
server := &http.Server{Handler: metricsHandler{Conntrack: &conntrackState, Latest: &latest, SMTPConfigured: conntrackServiceConfigured(configuration.ConntrackServices, "smtp")}, ReadHeaderTimeout: 5 * time.Second}
serverErrors := make(chan error, 1)
observerErrors := make(chan error, 1)
go func() {
serverErrors <- server.Serve(listener)
}()
go func() {
observerErrors <- startConntrackObserver(ctx, configuration, &conntrackState, os.Stdout)
}()
defer server.Close()
for {
timer := time.NewTimer(time.Duration(configuration.SnapshotIntervalSeconds) * time.Second)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case err := <-serverErrors:
timer.Stop()
return err
case err := <-observerErrors:
timer.Stop()
return err
case <-timer.C:
if err := collect(ctx, configuration, nftPath, tcPath, telemetryPath, snapshotPath, &latest); err != nil {
return err
}
}
}
}
func collect(ctx context.Context, configuration relayConfig, nftPath, tcPath, telemetryPath, snapshotPath string, latest *atomic.Pointer[metricsSnapshot]) error {
now := time.Now().UTC()
services := make([]serviceMetrics, 0, len(evidenceSets))
var smtpRecords []evidenceRecord
for _, set := range evidenceSets {
output, err := exec.CommandContext(ctx, nftPath, "-j", "list", "set", nftablesFamily, nftablesTable, set.Name).Output()
if err != nil {
return err
}
records, total, err := parseRecords(now, output, set.Name, configuration.MaximumRecords)
if err != nil {
return err
}
services = append(services, serviceMetrics{Records: records, Service: set.Service, Total: total})
if set.Service == "smtp" {
smtpRecords = records
}
}
counterOutput, err := exec.CommandContext(ctx, nftPath, "-j", "list", "counters").Output()
if err != nil {
return err
}
drops, err := parseDropMetrics(counterOutput, conntrackServiceConfigured(configuration.ConntrackServices, "smtp"))
if err != nil {
return err
}
if err := writeSnapshot(snapshotPath, snapshot{
SchemaVersion: 1,
GeneratedAt: now,
Reason: reason,
ObservationWindowSeconds: configuration.ObservationWindowSeconds,
Records: smtpRecords,
}); err != nil {
return err
}
qdiscOutput, err := exec.CommandContext(ctx, tcPath, "-s", "-j", "qdisc", "show", "dev", "wg-oci0").Output()
if err != nil {
return err
}
qdiscDrops, err := parseQdiscMetrics(qdiscOutput)
if err != nil {
return err
}
telemetryContents, err := os.ReadFile(telemetryPath)
if err != nil {
return err
}
telemetry, err := parseTelemetrySnapshot(telemetryContents)
if err != nil {
return err
}
latest.Store(&metricsSnapshot{Drops: drops, GeneratedAt: now, Health: telemetry.Status, QdiscDrops: qdiscDrops, Services: services, TelemetryAt: telemetry.GeneratedAt})
return nil
}
func parseRecords(now time.Time, contents []byte, expectedSet string, maximum uint64) ([]evidenceRecord, uint64, error) {
var document nftablesSetDocument
decoder := json.NewDecoder(bytes.NewReader(contents))
if err := decoder.Decode(&document); err != nil {
return nil, 0, err
}
records := make([]evidenceRecord, 0)
found := false
seen := make(map[netip.Addr]bool)
for _, object := range document.Nftables {
if object.Set == nil {
continue
}
if object.Set.Family != nftablesFamily || object.Set.Table != nftablesTable || object.Set.Name != expectedSet {
return nil, 0, fmt.Errorf("nftables returned the wrong quarantine set for %q", expectedSet)
}
found = true
for _, item := range object.Set.Elem {
if item.Element.Counter == nil {
return nil, 0, fmt.Errorf("evidence counter is absent for address %q", item.Element.Value)
}
address, err := netip.ParseAddr(item.Element.Value)
if err != nil || !address.Is4() || address.String() != item.Element.Value || seen[address] {
return nil, 0, fmt.Errorf("invalid or duplicate evidence address %q", item.Element.Value)
}
if item.Element.Expires <= 0 {
return nil, 0, fmt.Errorf("invalid expiry for evidence address %q", item.Element.Value)
}
seen[address] = true
records = append(records, evidenceRecord{
Address: address.String(),
Count: item.Element.Counter.Packets,
ExpiresAt: now.Add(time.Duration(item.Element.Expires) * time.Second),
})
}
}
if !found {
return nil, 0, fmt.Errorf("quarantine set %q is absent from nftables output", expectedSet)
}
sort.Slice(records, func(left, right int) bool {
leftAddress := netip.MustParseAddr(records[left].Address)
rightAddress := netip.MustParseAddr(records[right].Address)
return leftAddress.Less(rightAddress)
})
total := uint64(len(records))
if total > maximum {
records = records[:maximum]
}
return records, total, nil
}
func parseDropMetrics(contents []byte, mailConfigured bool) ([]dropMetric, error) {
var document nftablesCounterDocument
if err := json.Unmarshal(contents, &document); err != nil {
return nil, err
}
values := make(map[string]uint64)
for _, object := range document.Nftables {
if object.Counter != nil {
values[object.Counter.Name] = object.Counter.Packets
}
}
metrics := make([]dropMetric, 0, len(dropMetricSpecs))
for _, spec := range dropMetricSpecs {
packets, found := values[spec.Name]
if spec.MailOnly && !mailConfigured && !found {
packets = 0
found = true
}
if !found {
return nil, fmt.Errorf("nftables counter %q is absent", spec.Name)
}
metrics = append(metrics, dropMetric{Packets: packets, Reason: spec.Reason, Service: spec.Service})
}
return metrics, nil
}
func parseQdiscMetrics(contents []byte) ([]qdiscMetric, error) {
var document []qdiscDocumentItem
if err := json.Unmarshal(contents, &document); err != nil {
return nil, err
}
channels := map[string]string{
"1:10": "web",
"1:20": "smtp",
"1:30": "authenticated_mail",
"1:40": "default",
}
values := map[string]uint64{
"authenticated_mail": 0,
"default": 0,
"smtp": 0,
"total": 0,
"web": 0,
}
for _, qdisc := range document {
values["total"] += qdisc.Drops
if channel, ok := channels[qdisc.Parent]; ok {
values[channel] += qdisc.Drops
}
}
metrics := make([]qdiscMetric, 0, len(values))
for _, channel := range []string{"web", "smtp", "authenticated_mail", "default", "total"} {
metrics = append(metrics, qdiscMetric{Channel: channel, Packets: values[channel]})
}
return metrics, nil
}
func parseTelemetrySnapshot(contents []byte) (telemetrySnapshot, error) {
var value telemetrySnapshot
decoder := json.NewDecoder(bytes.NewReader(contents))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&value); err != nil {
return telemetrySnapshot{}, err
}
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
return telemetrySnapshot{}, errors.New("telemetry snapshot has trailing data")
}
if value.GeneratedAt.IsZero() || value.GeneratedAt.After(time.Now().Add(5*time.Minute)) {
return telemetrySnapshot{}, errors.New("telemetry snapshot timestamp is invalid")
}
return value, nil
}
type metricsHandler struct {
Conntrack *conntrackMetrics
Latest *atomic.Pointer[metricsSnapshot]
SMTPConfigured bool
}
func (handler metricsHandler) ServeHTTP(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/metrics" {
http.NotFound(response, request)
return
}
latest := handler.Latest.Load()
if latest == nil {
http.Error(response, "metrics are not ready", http.StatusServiceUnavailable)
return
}
response.Header().Set("Content-Type", "text/plain; version=0.0.4")
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_quarantine_packets gauge")
for _, service := range latest.Services {
for _, record := range service.Records {
fmt.Fprintf(response, "oci_l4_relay_tcp_quarantine_packets{service=%q,address=%q} %d\n", service.Service, record.Address, record.Count)
}
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_quarantine_expires_timestamp_seconds gauge")
for _, service := range latest.Services {
for _, record := range service.Records {
fmt.Fprintf(response, "oci_l4_relay_tcp_quarantine_expires_timestamp_seconds{service=%q,address=%q} %d\n", service.Service, record.Address, record.ExpiresAt.Unix())
}
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_evidence_records gauge")
for _, service := range latest.Services {
fmt.Fprintf(response, "oci_l4_relay_tcp_evidence_records{service=%q} %d\n", service.Service, len(service.Records))
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_evidence_truncated gauge")
for _, service := range latest.Services {
truncated := 0
if service.Total > uint64(len(service.Records)) {
truncated = 1
}
fmt.Fprintf(response, "oci_l4_relay_tcp_evidence_truncated{service=%q} %d\n", service.Service, truncated)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_evidence_generated_timestamp_seconds gauge")
fmt.Fprintf(response, "oci_l4_relay_tcp_evidence_generated_timestamp_seconds %d\n", latest.GeneratedAt.Unix())
fmt.Fprintln(response, "# TYPE oci_l4_relay_nftables_dropped_packets_total counter")
for _, metric := range latest.Drops {
fmt.Fprintf(response, "oci_l4_relay_nftables_dropped_packets_total{service=%q,reason=%q} %d\n", metric.Service, metric.Reason, metric.Packets)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_qdisc_dropped_packets_total counter")
for _, metric := range latest.QdiscDrops {
fmt.Fprintf(response, "oci_l4_relay_qdisc_dropped_packets_total{channel=%q} %d\n", metric.Channel, metric.Packets)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_component_healthy gauge")
for component, healthy := range map[string]bool{
"nftables": latest.Health.NftablesActive,
"smtp_deny": !handler.SMTPConfigured || latest.Health.SMTPDenyActive && latest.Health.SMTPDenyPolicyValid,
"web_allow": latest.Health.WebAllowActive && latest.Health.WebAllowCached,
"wireguard": latest.Health.WireGuardActive,
} {
fmt.Fprintf(response, "oci_l4_relay_component_healthy{component=%q} %d\n", component, booleanValue(healthy))
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_admission_cached gauge")
fmt.Fprintf(response, "oci_l4_relay_admission_cached{service=%q} %d\n", "web", booleanValue(latest.Health.WebAllowCached))
fmt.Fprintf(response, "oci_l4_relay_admission_cached{service=%q} %d\n", "smtp", booleanValue(latest.Health.SMTPDenyCached))
fmt.Fprintln(response, "# TYPE oci_l4_relay_smtp_deny_refresh_successful gauge")
fmt.Fprintf(response, "oci_l4_relay_smtp_deny_refresh_successful %d\n", booleanValue(latest.Health.SMTPDenyRefreshSuccessful))
fmt.Fprintln(response, "# TYPE oci_l4_relay_smtp_deny_configured gauge")
fmt.Fprintf(response, "oci_l4_relay_smtp_deny_configured %d\n", booleanValue(handler.SMTPConfigured))
for name, value := range map[string]int64{
"oci_l4_relay_conntrack_entries": latest.Health.ConntrackCount,
"oci_l4_relay_conntrack_maximum": latest.Health.ConntrackMaximum,
"oci_l4_relay_smtp_deny_bundle_age_seconds": latest.Health.SMTPDenyBundleAgeSeconds,
"oci_l4_relay_smtp_deny_bundle_remaining_seconds": latest.Health.SMTPDenyBundleRemainingSeconds,
"oci_l4_relay_smtp_deny_refresh_completed_age_seconds": latest.Health.SMTPDenyRefreshCompletedAgeSeconds,
"oci_l4_relay_smtp_deny_remote_current_age_seconds": latest.Health.SMTPDenyRemoteCurrentAgeSeconds,
"oci_l4_relay_wireguard_handshake_age_seconds": latest.Health.HandshakeAgeSeconds,
"oci_l4_relay_telemetry_generated_timestamp_seconds": latest.TelemetryAt.Unix(),
} {
fmt.Fprintf(response, "# TYPE %s gauge\n%s %d\n", name, name, value)
}
if handler.Conntrack != nil {
fmt.Fprintln(response, "# TYPE oci_l4_relay_conntrack_observer_events_total counter")
for outcome, value := range map[string]uint64{
"eligible": handler.Conntrack.Eligible.Load(),
"emitted": handler.Conntrack.Emitted.Load(),
"rate_dropped": handler.Conntrack.RateDropped.Load(),
"sampled_out": handler.Conntrack.SampledOut.Load(),
} {
fmt.Fprintf(response, "oci_l4_relay_conntrack_observer_events_total{outcome=%q} %d\n", outcome, value)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_conntrack_observer_up gauge")
fmt.Fprintf(response, "oci_l4_relay_conntrack_observer_up %d\n", booleanValue(handler.Conntrack.Up.Load()))
}
}
func booleanValue(value bool) int {
if value {
return 1
}
return 0
}
func conntrackServiceConfigured(services []conntrackService, name string) bool {
for _, service := range services {
if service.Name == name {
return true
}
}
return false
}
func writeSnapshot(path string, value snapshot) error {
contents, err := json.Marshal(value)
if err != nil {
return err
}
directory := filepath.Dir(path)
temporary, err := os.CreateTemp(directory, ".smtp-evidence-*.tmp")
if err != nil {
return err
}
temporaryPath := temporary.Name()
defer os.Remove(temporaryPath)
if err := temporary.Chmod(0600); err != nil {
temporary.Close()
return err
}
if _, err := temporary.Write(append(contents, '\n')); err != nil {
temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(temporaryPath, path)
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}

270
services/oci-l4-relay-guest/cmd/smtp-evidence/main_test.go Normal file
View file

@ -0,0 +1,270 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
)
func TestParseRecordsSortsAndBoundsEvidence(t *testing.T) {
now := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC)
contents := []byte(`{"nftables":[{"metainfo":{"json_schema_version":1}},{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.2","expires":598,"counter":{"packets":4,"bytes":160}}},{"elem":{"val":"198.18.0.1","expires":599,"counter":{"packets":1,"bytes":40}}}]}}]}`)
records, total, err := parseRecords(now, contents, "smtp_syn_quarantine_v4", 1)
if err != nil {
t.Fatal(err)
}
if total != 2 || len(records) != 1 || records[0].Address != "198.18.0.1" || records[0].Count != 1 || !records[0].ExpiresAt.Equal(now.Add(599*time.Second)) {
t.Fatalf("unexpected records %#v", records)
}
}
func TestParseRecordsRequiresCanonicalIPv4(t *testing.T) {
contents := []byte(`{"nftables":[{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"2001:db8::1","expires":598,"counter":{"packets":1}}}]}}]}`)
if _, _, err := parseRecords(time.Now(), contents, "smtp_syn_quarantine_v4", 1); err == nil {
t.Fatal("IPv6 evidence was accepted")
}
}
func TestParseDropMetricsRequiresEveryBoundedCounter(t *testing.T) {
items := make([]string, 0, len(dropMetricSpecs))
for index, spec := range dropMetricSpecs {
items = append(items, fmt.Sprintf(`{"counter":{"name":%q,"packets":%d}}`, spec.Name, index+1))
}
metrics, err := parseDropMetrics([]byte(`{"nftables":[`+strings.Join(items, ",")+`]}`), true)
if err != nil {
t.Fatal(err)
}
if len(metrics) != len(dropMetricSpecs) || metrics[0].Service != "web" || metrics[0].Reason != "admission" || metrics[0].Packets != 1 {
t.Fatalf("unexpected metrics %#v", metrics)
}
if _, err := parseDropMetrics([]byte(`{"nftables":[]}`), true); err == nil {
t.Fatal("missing counters were accepted")
}
}
func TestMetricsHandlerExposesBoundedEvidenceAndDrops(t *testing.T) {
generated := time.Date(2026, 7, 30, 1, 0, 0, 0, time.UTC)
var latest atomic.Pointer[metricsSnapshot]
latest.Store(&metricsSnapshot{
GeneratedAt: generated,
Services: []serviceMetrics{
{Service: "web", Records: []evidenceRecord{{Address: "198.18.0.1", Count: 2, ExpiresAt: generated.Add(time.Minute)}}, Total: 2},
{Service: "smtp", Records: []evidenceRecord{{Address: "198.18.0.2", Count: 4, ExpiresAt: generated.Add(10 * time.Minute)}}, Total: 1},
},
Drops: []dropMetric{{Service: "web", Reason: "source_rate", Packets: 7}},
Health: relayHealth{ConntrackCount: 12, ConntrackMaximum: 262144, NftablesActive: true, WebAllowActive: true, WebAllowCached: true, WireGuardActive: true},
QdiscDrops: []qdiscMetric{{Channel: "web", Packets: 3}},
TelemetryAt: generated,
})
var conntrackState conntrackMetrics
conntrackState.Up.Store(true)
conntrackState.Emitted.Store(2)
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
response := httptest.NewRecorder()
metricsHandler{Conntrack: &conntrackState, Latest: &latest, SMTPConfigured: true}.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d", response.Code)
}
for _, expected := range []string{
`oci_l4_relay_tcp_quarantine_packets{service="web",address="198.18.0.1"} 2`,
`oci_l4_relay_tcp_quarantine_expires_timestamp_seconds{service="smtp",address="198.18.0.2"} 1785373800`,
`oci_l4_relay_tcp_evidence_records{service="smtp"} 1`,
`oci_l4_relay_tcp_evidence_truncated{service="web"} 1`,
`oci_l4_relay_tcp_evidence_generated_timestamp_seconds 1785373200`,
`oci_l4_relay_nftables_dropped_packets_total{service="web",reason="source_rate"} 7`,
`oci_l4_relay_qdisc_dropped_packets_total{channel="web"} 3`,
`oci_l4_relay_component_healthy{component="web_allow"} 1`,
`oci_l4_relay_conntrack_entries 12`,
`oci_l4_relay_conntrack_observer_events_total{outcome="emitted"} 2`,
`oci_l4_relay_conntrack_observer_up 1`,
`oci_l4_relay_smtp_deny_configured 1`,
} {
if !strings.Contains(response.Body.String(), expected) {
t.Errorf("metrics do not contain %q", expected)
}
}
}
func TestRunCollectsAndServesMetrics(t *testing.T) {
listener, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
if errors.Is(err, syscall.EPERM) {
t.Skip("network namespace does not permit listeners")
}
t.Fatal(err)
}
listenAddress := listener.Addr().String()
listener.Close()
items := make([]string, 0, len(dropMetricSpecs))
for index, spec := range dropMetricSpecs {
items = append(items, fmt.Sprintf(`{"counter":{"name":%q,"packets":%d}}`, spec.Name, index+1))
}
script := fmt.Sprintf(`#!/bin/sh
case "$*" in
*smtp_syn_quarantine_v4*) printf '%%s' '{"nftables":[{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.2","expires":599,"counter":{"packets":4}}}]}}]}' ;;
*syn_quarantine_v4*) printf '%%s' '{"nftables":[{"set":{"family":"inet","name":"syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.1","expires":59,"counter":{"packets":2}}}]}}]}' ;;
*list\ counters*) printf '%%s' '{"nftables":[%s]}' ;;
*) exit 1 ;;
esac
`, strings.Join(items, ","))
nftPath := filepath.Join(t.TempDir(), "nft")
if err := os.WriteFile(nftPath, []byte(script), 0700); err != nil {
t.Fatal(err)
}
snapshotPath := filepath.Join(t.TempDir(), "current.json")
tcPath := filepath.Join(t.TempDir(), "tc")
if err := os.WriteFile(tcPath, []byte("#!/bin/sh\nprintf '%s' '[{\"parent\":\"1:10\",\"drops\":3}]'\n"), 0700); err != nil {
t.Fatal(err)
}
telemetryPath := filepath.Join(t.TempDir(), "telemetry.json")
if err := os.WriteFile(telemetryPath, []byte(`{"generated_at":"2026-07-30T01:00:00Z","status":{"conntrack_count":12,"conntrack_maximum":262144,"handshake_age_seconds":10,"nftables_active":true,"smtp_deny_active":true,"smtp_deny_active_source":"current","smtp_deny_bundle_age_seconds":60,"smtp_deny_bundle_remaining_seconds":600,"smtp_deny_cached":true,"smtp_deny_policy_valid":true,"smtp_deny_refresh_completed_age_seconds":30,"smtp_deny_refresh_successful":true,"smtp_deny_remote_current_age_seconds":30,"web_allow_active":true,"web_allow_cached":true,"wireguard_active":true}}`), 0600); err != nil {
t.Fatal(err)
}
previousObserver := startConntrackObserver
startConntrackObserver = func(ctx context.Context, _ relayConfig, _ *conntrackMetrics, _ io.Writer) error {
<-ctx.Done()
return ctx.Err()
}
defer func() { startConntrackObserver = previousObserver }()
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
done <- run(ctx, validRelayConfig(listenAddress), nftPath, tcPath, telemetryPath, snapshotPath, false)
}()
var body []byte
client := http.Client{Timeout: time.Second}
for attempt := 0; attempt < 50; attempt++ {
response, err := client.Get("http://" + listenAddress + "/metrics")
if err == nil {
body, err = io.ReadAll(response.Body)
response.Body.Close()
if err == nil && response.StatusCode == http.StatusOK {
break
}
}
time.Sleep(10 * time.Millisecond)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
for _, expected := range []string{
`oci_l4_relay_tcp_quarantine_packets{service="web",address="198.18.0.1"} 2`,
`oci_l4_relay_tcp_quarantine_packets{service="smtp",address="198.18.0.2"} 4`,
`oci_l4_relay_nftables_dropped_packets_total{service="authenticated_mail",reason="aggregate_packet"} 14`,
} {
if !strings.Contains(string(body), expected) {
t.Errorf("metrics do not contain %q", expected)
}
}
contents, err := os.ReadFile(snapshotPath)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(contents), `"address":"198.18.0.2"`) || strings.Contains(string(contents), `"address":"198.18.0.1"`) {
t.Fatalf("unexpected SMTP snapshot %s", contents)
}
}
func TestWriteSnapshotReplacesCurrentFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "current.json")
if err := os.WriteFile(path, []byte("old"), 0600); err != nil {
t.Fatal(err)
}
value := snapshot{SchemaVersion: 1, Records: []evidenceRecord{}}
if err := writeSnapshot(path, value); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var actual snapshot
if err := json.Unmarshal(contents, &actual); err != nil {
t.Fatal(err)
}
if actual.SchemaVersion != 1 || actual.Records == nil {
t.Fatalf("unexpected snapshot %s", contents)
}
}
func TestLoadConfigAcceptsTestedCadences(t *testing.T) {
for _, interval := range []int{30, 60} {
path := filepath.Join(t.TempDir(), "config.json")
contents := relayConfigJSON(interval)
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err != nil {
t.Fatalf("interval %d: %v", interval, err)
}
}
}
func TestLoadConfigRejectsUntestedCadence(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
contents := relayConfigJSON(45)
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err == nil {
t.Fatal("untested cadence was accepted")
}
}
func TestLoadConfigRejectsUnknownField(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
contents := []byte(strings.TrimSuffix(string(relayConfigJSON(60)), "}") + `,"upload":true}`)
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err == nil {
t.Fatal("unknown field was accepted")
}
}
func TestLoadConfigRejectsNonSpecificMetricsAddress(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
contents := []byte(strings.Replace(string(relayConfigJSON(60)), "10.40.68.115:19100", "0.0.0.0:19100", 1))
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err == nil {
t.Fatal("wildcard metrics address was accepted")
}
}
func validRelayConfig(listenAddress string) relayConfig {
return relayConfig{
ConntrackEmissionBurst: 10,
ConntrackEmissionRatePerSecond: 1,
ConntrackEventBuffer: 4096,
ConntrackSampleDenominator: 10,
ConntrackServices: []conntrackService{{
Name: "web",
Ports: []uint16{80, 443},
TargetAddress: netip.MustParseAddr("10.104.40.127"),
}},
MaximumRecords: 4096,
MetricsListenAddress: listenAddress,
ObservationWindowSeconds: 600,
SnapshotIntervalSeconds: 60,
}
}
func relayConfigJSON(interval int) []byte {
return []byte(fmt.Sprintf(`{"conntrack_emission_burst":10,"conntrack_emission_rate_per_second":1,"conntrack_event_buffer":4096,"conntrack_sample_denominator":10,"conntrack_services":[{"name":"web","ports":[80,443],"target_address":"10.104.40.127"}],"maximum_records":4096,"metrics_listen_address":"10.40.68.115:19100","observation_window_seconds":600,"snapshot_interval_seconds":%d}`, interval))
}