ms2k/quorate
SHA256
Watch
0
Code Releases Activity

ce7cb702218693d08ea6d83464d2457c9250027c

Browse source
This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
8 changed files with 145336 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

574
services/oci-l4-relay-guest/cmd/admission/main.go Normal file
View file

@ -0,0 +1,574 @@
package main
import (
"bufio"
"compress/gzip"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/netip"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
const (
tableName = "oci_l4_relay_web_allow"
maxBundleBytes = 64 << 20
maxIntervals = 500000
maxExceptions = 10000
minimumLifetime = time.Hour
maximumLifetime = 7 * 24 * time.Hour
)
type addressRange struct {
Start string `json:"start"`
End string `json:"end"`
}
type bundle struct {
SchemaVersion int `json:"schema_version"`
GeneratedAt time.Time `json:"generated_at"`
ExpiresAt time.Time `json:"expires_at"`
AdmittedIPv4 []addressRange `json:"admitted_ipv4"`
ExceptionIPv4 []addressRange `json:"exception_ipv4"`
AdmittedIntervalCount int `json:"admitted_interval_count"`
AdmittedAddressCount uint64 `json:"admitted_address_count"`
ExceptionIntervalCount int `json:"exception_interval_count"`
ExceptionAddressCount uint64 `json:"exception_address_count"`
}
type parsedRange struct {
start uint32
end uint32
}
type validatedBundle struct {
bundle
admitted []parsedRange
exceptions []parsedRange
}
type updateConfig struct {
bucket string
object string
publicInterface string
stateDirectory string
ociCLI string
nft string
minimumAddresses uint64
maximumAddresses uint64
minimumIntervals int
maximumExceptionAddresses uint64
maximumExceptionIntervals int
staticExceptions []string
}
type stringValues []string
func (values *stringValues) String() string {
return strings.Join(*values, ",")
}
func (values *stringValues) Set(value string) error {
*values = append(*values, value)
return nil
}
func addUpdateFlags(flags *flag.FlagSet, configuration *updateConfig) {
flags.StringVar(&configuration.publicInterface, "public-interface", "", "public network interface")
flags.StringVar(&configuration.stateDirectory, "state-directory", "/var/lib/oci-l4-relay/web-allow", "state directory")
flags.StringVar(&configuration.nft, "nft", "/usr/sbin/nft", "nft path")
flags.Uint64Var(&configuration.minimumAddresses, "minimum-admitted-addresses", 1, "minimum admitted IPv4 address count")
flags.Uint64Var(&configuration.maximumAddresses, "maximum-admitted-addresses", 1<<32, "maximum admitted IPv4 address count")
flags.IntVar(&configuration.minimumIntervals, "minimum-admitted-intervals", 1, "minimum admitted interval count")
flags.Uint64Var(&configuration.maximumExceptionAddresses, "maximum-exception-addresses", 1<<32, "maximum exception IPv4 address count")
flags.IntVar(&configuration.maximumExceptionIntervals, "maximum-exception-intervals", maxExceptions, "maximum exception interval count")
flags.Var((*stringValues)(&configuration.staticExceptions), "static-exception-ipv4", "static IPv4 exception")
}
func writeJSONFile(path string, value any) error {
contents, err := json.Marshal(value)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return err
}
return os.WriteFile(path, append(contents, '\n'), 0644)
}
func main() {
if len(os.Args) < 2 {
fatal(errors.New("expected apply, update, restore, validate, or render"))
}
switch os.Args[1] {
case "apply":
flags := flag.NewFlagSet("apply", flag.ExitOnError)
configuration := updateConfig{}
input := flags.String("input", "", "downloaded bundle path")
addUpdateFlags(flags, &configuration)
_ = flags.Parse(os.Args[2:])
now := time.Now().UTC()
if *input == "" {
fatal(errors.New("input is required"))
}
if err := activateBundle(now, configuration, *input, true); err != nil {
fmt.Fprintln(os.Stderr, err)
if restoreErr := restore(now, configuration); restoreErr != nil {
if failClosedErr := failClosed(configuration); failClosedErr != nil {
fatal(errors.Join(restoreErr, failClosedErr))
}
fatal(restoreErr)
}
fmt.Println("local_apply=false cache_restored=true")
}
case "update":
flags := flag.NewFlagSet("update", flag.ExitOnError)
configuration := updateConfig{}
flags.StringVar(&configuration.bucket, "bucket", "", "OCI Object Storage bucket")
flags.StringVar(&configuration.object, "object", "", "OCI Object Storage object")
flags.StringVar(&configuration.ociCLI, "oci-cli", "/opt/oci-cli/bin/oci", "OCI CLI path")
addUpdateFlags(flags, &configuration)
_ = flags.Parse(os.Args[2:])
now := time.Now().UTC()
if err := update(now, configuration); err != nil {
fmt.Fprintln(os.Stderr, err)
if restoreErr := restore(now, configuration); restoreErr != nil {
if failClosedErr := failClosed(configuration); failClosedErr != nil {
fatal(errors.Join(restoreErr, failClosedErr))
}
fatal(restoreErr)
}
fmt.Println("remote_update=false cache_restored=true")
}
case "restore":
flags := flag.NewFlagSet("restore", flag.ExitOnError)
configuration := updateConfig{}
addUpdateFlags(flags, &configuration)
_ = flags.Parse(os.Args[2:])
if err := restore(time.Now().UTC(), configuration); err != nil {
if failClosedErr := failClosed(configuration); failClosedErr != nil {
fatal(errors.Join(err, failClosedErr))
}
fatal(err)
}
case "validate", "render":
flags := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
input := flags.String("input", "", "bundle path")
publicInterface := flags.String("public-interface", "", "public network interface")
minimumAddresses := flags.Uint64("minimum-admitted-addresses", 1, "minimum admitted IPv4 address count")
maximumAddresses := flags.Uint64("maximum-admitted-addresses", 1<<32, "maximum admitted IPv4 address count")
minimumIntervals := flags.Int("minimum-admitted-intervals", 1, "minimum admitted interval count")
maximumExceptionAddresses := flags.Uint64("maximum-exception-addresses", 1<<32, "maximum exception IPv4 address count")
maximumExceptionIntervals := flags.Int("maximum-exception-intervals", maxExceptions, "maximum exception interval count")
summaryOutput := flags.String("summary-output", "", "write validated bundle summary")
_ = flags.Parse(os.Args[2:])
validated, err := readBundle(time.Now().UTC(), *input)
if err != nil {
fatal(err)
}
if err := validatePolicy(updateConfig{
minimumAddresses: *minimumAddresses,
maximumAddresses: *maximumAddresses,
minimumIntervals: *minimumIntervals,
maximumExceptionAddresses: *maximumExceptionAddresses,
maximumExceptionIntervals: *maximumExceptionIntervals,
}, validated); err != nil {
fatal(err)
}
if *summaryOutput != "" {
summary := struct {
GeneratedAt time.Time `json:"generated_at"`
ExpiresAt time.Time `json:"expires_at"`
AdmittedIntervalCount int `json:"admitted_interval_count"`
AdmittedAddressCount uint64 `json:"admitted_address_count"`
ExceptionIntervalCount int `json:"exception_interval_count"`
ExceptionAddressCount uint64 `json:"exception_address_count"`
}{
GeneratedAt: validated.GeneratedAt,
ExpiresAt: validated.ExpiresAt,
AdmittedIntervalCount: validated.AdmittedIntervalCount,
AdmittedAddressCount: validated.AdmittedAddressCount,
ExceptionIntervalCount: validated.ExceptionIntervalCount,
ExceptionAddressCount: validated.ExceptionAddressCount,
}
if err := writeJSONFile(*summaryOutput, summary); err != nil {
fatal(err)
}
}
if os.Args[1] == "render" {
if err := render(os.Stdout, validated, *publicInterface); err != nil {
fatal(err)
}
}
default:
fatal(fmt.Errorf("unknown command %q", os.Args[1]))
}
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
func update(now time.Time, configuration updateConfig) error {
if configuration.bucket == "" || configuration.object == "" || configuration.publicInterface == "" {
return errors.New("bucket, object, and public-interface are required")
}
if err := os.MkdirAll(configuration.stateDirectory, 0700); err != nil {
return err
}
temporaryDirectory, err := os.MkdirTemp(configuration.stateDirectory, ".update-")
if err != nil {
return err
}
defer func() { _ = os.RemoveAll(temporaryDirectory) }()
bundlePath := filepath.Join(temporaryDirectory, "bundle.json.gz")
command := exec.Command(configuration.ociCLI, "os", "object", "get", "--auth", "instance_principal", "--bucket-name", configuration.bucket, "--name", configuration.object, "--file", bundlePath)
command.Stdout = os.Stdout
command.Stderr = os.Stderr
if err := command.Run(); err != nil {
return fmt.Errorf("fetch address bundle: %w", err)
}
return activateBundle(now, configuration, bundlePath, true)
}
func restore(now time.Time, configuration updateConfig) error {
if configuration.publicInterface == "" {
return errors.New("public-interface is required")
}
return activateBundle(now, configuration, filepath.Join(configuration.stateDirectory, "last-good.json.gz"), false)
}
func activateBundle(now time.Time, configuration updateConfig, bundlePath string, persist bool) error {
validated, err := readBundle(now, bundlePath)
if err != nil {
return err
}
if err := validatePolicy(configuration, validated); err != nil {
return err
}
validated, err = addStaticExceptions(validated, configuration.staticExceptions)
if err != nil {
return err
}
temporaryDirectory, err := os.MkdirTemp(configuration.stateDirectory, ".activate-")
if err != nil {
return err
}
defer func() { _ = os.RemoveAll(temporaryDirectory) }()
rulesetPath := filepath.Join(temporaryDirectory, "admission.nft")
ruleset, err := os.OpenFile(rulesetPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return err
}
if err := render(ruleset, validated, configuration.publicInterface); err != nil {
_ = ruleset.Close()
return err
}
if err := ruleset.Close(); err != nil {
return err
}
if err := ensureTable(configuration.nft); err != nil {
return err
}
if output, err := exec.Command(configuration.nft, "--check", "--file", rulesetPath).CombinedOutput(); err != nil {
return fmt.Errorf("validate rendered ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
if output, err := exec.Command(configuration.nft, "--file", rulesetPath).CombinedOutput(); err != nil {
return fmt.Errorf("load rendered ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
if persist {
if err := replaceFile(bundlePath, filepath.Join(configuration.stateDirectory, "last-good.json.gz")); err != nil {
return err
}
if err := replaceFile(rulesetPath, filepath.Join(configuration.stateDirectory, "last-good.nft")); err != nil {
return err
}
}
fmt.Printf("admitted_intervals=%d admitted_addresses=%d exception_intervals=%d exception_addresses=%d expires_at=%s\n", len(validated.admitted), validated.AdmittedAddressCount, len(validated.exceptions), validated.ExceptionAddressCount, validated.ExpiresAt.Format(time.RFC3339))
return nil
}
func addStaticExceptions(document validatedBundle, values []string) (validatedBundle, error) {
exceptions := append([]parsedRange(nil), document.exceptions...)
for _, value := range values {
address, err := parseIPv4(value)
if err != nil {
return validatedBundle{}, fmt.Errorf("static exception: %w", err)
}
exceptions = append(exceptions, parsedRange{start: address, end: address})
}
sort.Slice(exceptions, func(left, right int) bool {
return exceptions[left].start < exceptions[right].start
})
merged := make([]parsedRange, 0, len(exceptions))
for _, candidate := range exceptions {
if len(merged) == 0 || uint64(candidate.start) > uint64(merged[len(merged)-1].end)+1 {
merged = append(merged, candidate)
continue
}
if candidate.end > merged[len(merged)-1].end {
merged[len(merged)-1].end = candidate.end
}
}
if len(merged) > maxExceptions {
return validatedBundle{}, errors.New("effective exception interval count exceeds limit")
}
document.exceptions = merged
return document, nil
}
func validatePolicy(configuration updateConfig, validated validatedBundle) error {
if validated.AdmittedAddressCount < configuration.minimumAddresses || validated.AdmittedAddressCount > configuration.maximumAddresses {
return errors.New("admitted address count is outside configured policy bounds")
}
if len(validated.admitted) < configuration.minimumIntervals {
return errors.New("admitted interval count is below the configured policy bound")
}
if validated.ExceptionAddressCount > configuration.maximumExceptionAddresses {
return errors.New("exception address count exceeds the configured policy bound")
}
if len(validated.exceptions) > configuration.maximumExceptionIntervals {
return errors.New("exception interval count exceeds the configured policy bound")
}
return nil
}
func failClosed(configuration updateConfig) error {
if configuration.publicInterface == "" {
return errors.New("public-interface is required")
}
if err := ensureTable(configuration.nft); err != nil {
return err
}
ruleset, err := os.CreateTemp("", "oci-l4-relay-fail-closed-*.nft")
if err != nil {
return err
}
path := ruleset.Name()
defer func() { _ = os.Remove(path) }()
if err := renderFailClosed(ruleset, configuration.publicInterface); err != nil {
_ = ruleset.Close()
return err
}
if err := ruleset.Close(); err != nil {
return err
}
if output, err := exec.Command(configuration.nft, "--check", "--file", path).CombinedOutput(); err != nil {
return fmt.Errorf("validate fail-closed ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
if output, err := exec.Command(configuration.nft, "--file", path).CombinedOutput(); err != nil {
return fmt.Errorf("load fail-closed ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
fmt.Println("web_allow_fail_closed=true")
return nil
}
func ensureTable(nft string) error {
if err := exec.Command(nft, "list", "table", "inet", tableName).Run(); err == nil {
return nil
}
if output, err := exec.Command(nft, "add", "table", "inet", tableName).CombinedOutput(); err != nil {
return fmt.Errorf("create admission table: %w: %s", err, strings.TrimSpace(string(output)))
}
return nil
}
func replaceFile(source, destination string) error {
contents, err := os.ReadFile(source)
if err != nil {
return err
}
temporary, err := os.CreateTemp(filepath.Dir(destination), ".last-good-")
if err != nil {
return err
}
name := temporary.Name()
defer func() { _ = os.Remove(name) }()
if err := temporary.Chmod(0600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(contents); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(name, destination)
}
func readBundle(now time.Time, path string) (validatedBundle, error) {
file, err := os.Open(path)
if err != nil {
return validatedBundle{}, err
}
defer func() { _ = file.Close() }()
reader := io.LimitReader(file, maxBundleBytes+1)
if strings.HasSuffix(path, ".gz") {
compressed, err := gzip.NewReader(reader)
if err != nil {
return validatedBundle{}, fmt.Errorf("open bundle gzip: %w", err)
}
defer func() { _ = compressed.Close() }()
reader = io.LimitReader(compressed, maxBundleBytes+1)
}
decoder := json.NewDecoder(reader)
decoder.DisallowUnknownFields()
document := bundle{}
if err := decoder.Decode(&document); err != nil {
return validatedBundle{}, fmt.Errorf("decode bundle: %w", err)
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return validatedBundle{}, errors.New("bundle contains trailing data")
}
return validateBundle(now, document)
}
func validateBundle(now time.Time, document bundle) (validatedBundle, error) {
if document.SchemaVersion != 1 {
return validatedBundle{}, fmt.Errorf("unsupported schema version %d", document.SchemaVersion)
}
if document.GeneratedAt.After(now.Add(5 * time.Minute)) {
return validatedBundle{}, errors.New("bundle generation time is in the future")
}
if document.ExpiresAt.Before(now.Add(minimumLifetime)) {
return validatedBundle{}, errors.New("bundle expires too soon")
}
if !document.ExpiresAt.After(document.GeneratedAt) {
return validatedBundle{}, errors.New("bundle expiry does not follow generation")
}
if document.ExpiresAt.Sub(document.GeneratedAt) > maximumLifetime || document.GeneratedAt.Before(now.Add(-maximumLifetime)) {
return validatedBundle{}, errors.New("bundle lifetime or generation age exceeds bounds")
}
if len(document.AdmittedIPv4) == 0 || len(document.AdmittedIPv4) > maxIntervals {
return validatedBundle{}, errors.New("admitted interval count is outside bounds")
}
if len(document.ExceptionIPv4) > maxExceptions {
return validatedBundle{}, errors.New("exception interval count is outside bounds")
}
admitted, admittedAddresses, err := validateRanges(document.AdmittedIPv4)
if err != nil {
return validatedBundle{}, fmt.Errorf("admitted_ipv4: %w", err)
}
exceptions, exceptionAddresses, err := validateRanges(document.ExceptionIPv4)
if err != nil {
return validatedBundle{}, fmt.Errorf("exception_ipv4: %w", err)
}
if document.AdmittedIntervalCount != len(admitted) || document.AdmittedAddressCount != admittedAddresses {
return validatedBundle{}, errors.New("admitted statistics do not match payload")
}
if document.ExceptionIntervalCount != len(exceptions) || document.ExceptionAddressCount != exceptionAddresses {
return validatedBundle{}, errors.New("exception statistics do not match payload")
}
return validatedBundle{bundle: document, admitted: admitted, exceptions: exceptions}, nil
}
func validateRanges(values []addressRange) ([]parsedRange, uint64, error) {
parsed := make([]parsedRange, 0, len(values))
var count uint64
var previousEnd uint32
for index, value := range values {
start, err := parseIPv4(value.Start)
if err != nil {
return nil, 0, fmt.Errorf("range %d start: %w", index, err)
}
end, err := parseIPv4(value.End)
if err != nil {
return nil, 0, fmt.Errorf("range %d end: %w", index, err)
}
if start > end {
return nil, 0, fmt.Errorf("range %d is reversed", index)
}
if index > 0 && start <= previousEnd {
return nil, 0, fmt.Errorf("range %d overlaps or is unsorted", index)
}
parsed = append(parsed, parsedRange{start: start, end: end})
count += uint64(end) - uint64(start) + 1
previousEnd = end
}
return parsed, count, nil
}
func parseIPv4(value string) (uint32, error) {
address, err := netip.ParseAddr(value)
if err != nil || !address.Is4() || address.String() != value {
return 0, fmt.Errorf("invalid canonical IPv4 address %q", value)
}
bytes := address.As4()
return uint32(bytes[0])<<24 | uint32(bytes[1])<<16 | uint32(bytes[2])<<8 | uint32(bytes[3]), nil
}
func ipv4String(value uint32) string {
return netip.AddrFrom4([4]byte{byte(value >> 24), byte(value >> 16), byte(value >> 8), byte(value)}).String()
}
func render(output io.Writer, document validatedBundle, publicInterface string) error {
if publicInterface == "" || strings.ContainsAny(publicInterface, "\"\\\n\r\t ") {
return errors.New("invalid public interface")
}
writer := bufio.NewWriter(output)
if _, err := fmt.Fprintf(writer, "delete table inet %s\ntable inet %s {\n", tableName, tableName); err != nil {
return err
}
if err := renderSet(writer, "admitted_v4", document.admitted); err != nil {
return err
}
if err := renderSet(writer, "exception_v4", document.exceptions); err != nil {
return err
}
if _, err := fmt.Fprintf(writer, " counter web_allow_denied {}\n chain ingress {\n type filter hook prerouting priority raw - 20; policy accept;\n iifname %s tcp dport { 80, 443, 465, 993 } ip saddr @exception_v4 accept\n iifname %s tcp dport { 80, 443, 465, 993 } ip saddr != @admitted_v4 counter name web_allow_denied drop\n }\n}\n", strconv.Quote(publicInterface), strconv.Quote(publicInterface)); err != nil {
return err
}
return writer.Flush()
}
func renderFailClosed(output io.Writer, publicInterface string) error {
if publicInterface == "" || strings.ContainsAny(publicInterface, "\"\\\n\r\t ") {
return errors.New("invalid public interface")
}
_, err := fmt.Fprintf(output, "delete table inet %s\ntable inet %s {\n counter web_allow_denied {}\n chain ingress {\n type filter hook prerouting priority raw - 20; policy accept;\n iifname %s tcp dport { 80, 443, 465, 993 } counter name web_allow_denied drop\n }\n}\n", tableName, tableName, strconv.Quote(publicInterface))
return err
}
func renderSet(writer *bufio.Writer, name string, ranges []parsedRange) error {
if _, err := fmt.Fprintf(writer, " set %s {\n type ipv4_addr\n flags interval\n auto-merge\n", name); err != nil {
return err
}
if len(ranges) > 0 {
if _, err := fmt.Fprintln(writer, " elements = {"); err != nil {
return err
}
for index, item := range ranges {
value := ipv4String(item.start)
if item.start != item.end {
value += "-" + ipv4String(item.end)
}
separator := ","
if index == len(ranges)-1 {
separator = ""
}
if _, err := fmt.Fprintf(writer, " %s%s\n", value, separator); err != nil {
return err
}
}
if _, err := fmt.Fprintln(writer, " }"); err != nil {
return err
}
}
_, err := fmt.Fprintln(writer, " }")
return err
}

155
services/oci-l4-relay-guest/cmd/admission/main_test.go Normal file
View file

@ -0,0 +1,155 @@
package main
import (
"bytes"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestWriteJSONFileCreatesParentDirectory(t *testing.T) {
path := filepath.Join(t.TempDir(), "workflow", "validation-summary.json")
if err := writeJSONFile(path, map[string]bool{"valid": true}); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(contents) != "{\"valid\":true}\n" {
t.Fatalf("unexpected contents %q", contents)
}
}
func validBundle(now time.Time) bundle {
return bundle{
SchemaVersion: 1,
GeneratedAt: now.Add(-time.Minute),
ExpiresAt: now.Add(24 * time.Hour),
AdmittedIPv4: []addressRange{{Start: "192.0.2.0", End: "192.0.2.255"}, {Start: "198.51.100.1", End: "198.51.100.1"}},
ExceptionIPv4: []addressRange{{Start: "203.0.113.0", End: "203.0.113.15"}},
AdmittedIntervalCount: 2,
AdmittedAddressCount: 257,
ExceptionIntervalCount: 1,
ExceptionAddressCount: 16,
}
}
func TestValidateAndRender(t *testing.T) {
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
validated, err := validateBundle(now, validBundle(now))
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
if err := render(&output, validated, "ens3"); err != nil {
t.Fatal(err)
}
for _, expected := range []string{
"delete table inet oci_l4_relay_web_allow",
"192.0.2.0-192.0.2.255",
"198.51.100.1",
"203.0.113.0-203.0.113.15",
"iifname \"ens3\" tcp dport { 80, 443, 465, 993 }",
"counter name web_allow_denied drop",
} {
if !strings.Contains(output.String(), expected) {
t.Errorf("rendered output does not contain %q", expected)
}
}
if strings.Contains(output.String(), "587") {
t.Fatal("rendered output admits cluster-local submission")
}
}
func TestValidationFailures(t *testing.T) {
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
tests := map[string]func(*bundle){
"schema": func(value *bundle) { value.SchemaVersion = 2 },
"future": func(value *bundle) { value.GeneratedAt = now.Add(10 * time.Minute) },
"expiry": func(value *bundle) { value.ExpiresAt = now.Add(30 * time.Minute) },
"lifetime": func(value *bundle) { value.ExpiresAt = now.Add(8 * 24 * time.Hour) },
"overlap": func(value *bundle) { value.AdmittedIPv4[1] = addressRange{Start: "192.0.2.128", End: "192.0.2.200"} },
"reversed": func(value *bundle) { value.AdmittedIPv4[0] = addressRange{Start: "192.0.2.10", End: "192.0.2.1"} },
"ipv6": func(value *bundle) { value.AdmittedIPv4[0] = addressRange{Start: "2001:db8::1", End: "2001:db8::2"} },
"noncanonical": func(value *bundle) { value.AdmittedIPv4[0].Start = "192.000.002.000" },
"address count": func(value *bundle) { value.AdmittedAddressCount++ },
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
value := validBundle(now)
mutate(&value)
if _, err := validateBundle(now, value); err == nil {
t.Fatal("expected validation failure")
}
})
}
}
func TestRejectsUnsafeInterface(t *testing.T) {
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
validated, err := validateBundle(now, validBundle(now))
if err != nil {
t.Fatal(err)
}
if err := render(&bytes.Buffer{}, validated, "ens3\nflush ruleset"); err == nil {
t.Fatal("expected unsafe interface rejection")
}
}
func TestRenderFailClosed(t *testing.T) {
var output bytes.Buffer
if err := renderFailClosed(&output, "ens3"); err != nil {
t.Fatal(err)
}
if !strings.Contains(output.String(), "iifname \"ens3\" tcp dport { 80, 443, 465, 993 } counter name web_allow_denied drop") {
t.Fatal("fail-closed ruleset does not drop every positive-policy surface")
}
}
func TestPolicyBounds(t *testing.T) {
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
validated, err := validateBundle(now, validBundle(now))
if err != nil {
t.Fatal(err)
}
if err := validatePolicy(updateConfig{minimumAddresses: 257, maximumAddresses: 257, minimumIntervals: 2, maximumExceptionAddresses: 16, maximumExceptionIntervals: 1}, validated); err != nil {
t.Fatal(err)
}
for name, configuration := range map[string]updateConfig{
"minimum addresses": {minimumAddresses: 258, maximumAddresses: 1 << 32, minimumIntervals: 1, maximumExceptionAddresses: 1 << 32, maximumExceptionIntervals: maxExceptions},
"maximum addresses": {minimumAddresses: 1, maximumAddresses: 256, minimumIntervals: 1, maximumExceptionAddresses: 1 << 32, maximumExceptionIntervals: maxExceptions},
"minimum intervals": {minimumAddresses: 1, maximumAddresses: 1 << 32, minimumIntervals: 3, maximumExceptionAddresses: 1 << 32, maximumExceptionIntervals: maxExceptions},
"maximum exception addresses": {minimumAddresses: 1, maximumAddresses: 1 << 32, minimumIntervals: 1, maximumExceptionAddresses: 15, maximumExceptionIntervals: maxExceptions},
"maximum exception intervals": {minimumAddresses: 1, maximumAddresses: 1 << 32, minimumIntervals: 1, maximumExceptionAddresses: 1 << 32, maximumExceptionIntervals: 0},
} {
t.Run(name, func(t *testing.T) {
if err := validatePolicy(configuration, validated); err == nil {
t.Fatal("expected policy-bound failure")
}
})
}
}
func TestAddStaticExceptions(t *testing.T) {
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
validated, err := validateBundle(now, validBundle(now))
if err != nil {
t.Fatal(err)
}
validated, err = addStaticExceptions(validated, []string{"203.0.113.16", "198.51.100.50"})
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
if err := render(&output, validated, "ens3"); err != nil {
t.Fatal(err)
}
for _, expected := range []string{"198.51.100.50", "203.0.113.0-203.0.113.16"} {
if !strings.Contains(output.String(), expected) {
t.Errorf("rendered output does not contain %q", expected)
}
}
}

627
services/oci-l4-relay-guest/cmd/smtp-deny/main.go Normal file
View file

@ -0,0 +1,627 @@
package main
import (
"bufio"
"compress/gzip"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/netip"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
tableName = "oci_l4_relay_smtp_deny"
activePolicyStateFile = "active-state.json"
maximumBundleBytes = 16 << 20
absoluteMaxEntries = 100000
absoluteMaxLife = 7 * 24 * time.Hour
)
var transportReasons = map[string]struct{}{
"connection-churn": {},
"connection-rate": {},
"syn-rate": {},
}
type denyEntry struct {
Start string `json:"start"`
End string `json:"end"`
Reason string `json:"reason"`
ObservationStarted time.Time `json:"observation_started"`
ObservationEnded time.Time `json:"observation_ended"`
ObservationCount uint64 `json:"observation_count"`
ExpiresAt time.Time `json:"expires_at"`
}
type bundle struct {
SchemaVersion int `json:"schema_version"`
GeneratedAt time.Time `json:"generated_at"`
ExpiresAt time.Time `json:"expires_at"`
DeniedIPv4 []denyEntry `json:"denied_ipv4"`
DeniedIntervalCount int `json:"denied_interval_count"`
DeniedAddressCount uint64 `json:"denied_address_count"`
}
type parsedEntry struct {
start uint32
end uint32
expiresAt time.Time
}
type validatedBundle struct {
bundle
entries []parsedEntry
}
type activePolicyState struct {
ActivatedAt time.Time `json:"activated_at"`
Source string `json:"source"`
BundleGeneratedAt time.Time `json:"bundle_generated_at,omitempty"`
BundleExpiresAt time.Time `json:"bundle_expires_at,omitempty"`
LastRemoteCurrentAt time.Time `json:"last_remote_current_at,omitempty"`
}
type activationConfig struct {
publicInterface string
stateDirectory string
nft string
minimumObservations uint64
maximumAddresses uint64
maximumIntervals int
maximumLifetime time.Duration
maximumObservationWindow time.Duration
}
func main() {
if len(os.Args) < 2 {
fatal(errors.New("expected apply, empty, restore, validate, or render"))
}
switch os.Args[1] {
case "apply":
flags := flag.NewFlagSet("apply", flag.ExitOnError)
configuration := activationConfig{}
input := flags.String("input", "", "downloaded SMTP deny bundle")
addActivationFlags(flags, &configuration)
_ = flags.Parse(os.Args[2:])
if *input == "" {
fatal(errors.New("input is required"))
}
now := time.Now().UTC()
if err := activate(now, configuration, *input, true, "current"); err != nil {
applyErr := err
if restoreErr := restore(now, configuration); restoreErr != nil {
if failOpenErr := failOpen(now, configuration); failOpenErr != nil {
fatal(errors.Join(applyErr, restoreErr, failOpenErr))
}
fatal(errors.Join(applyErr, restoreErr))
}
fmt.Println("local_apply=false cache_restored=true")
fatal(applyErr)
}
case "empty":
flags := flag.NewFlagSet("empty", flag.ExitOnError)
output := flags.String("output", "", "empty SMTP deny bundle output")
lifetime := flags.Duration("lifetime", 24*time.Hour, "empty bundle lifetime")
_ = flags.Parse(os.Args[2:])
if *output == "" || *lifetime <= 0 || *lifetime > absoluteMaxLife {
fatal(errors.New("output and a bounded lifetime are required"))
}
now := time.Now().UTC().Truncate(time.Second)
document := bundle{
SchemaVersion: 1,
GeneratedAt: now,
ExpiresAt: now.Add(*lifetime),
DeniedIPv4: []denyEntry{},
}
if err := writeBundle(*output, document); err != nil {
fatal(err)
}
case "restore":
flags := flag.NewFlagSet("restore", flag.ExitOnError)
configuration := activationConfig{}
addActivationFlags(flags, &configuration)
_ = flags.Parse(os.Args[2:])
now := time.Now().UTC()
if err := restore(now, configuration); err != nil {
if failOpenErr := failOpen(now, configuration); failOpenErr != nil {
fatal(errors.Join(err, failOpenErr))
}
fatal(err)
}
case "validate", "render":
flags := flag.NewFlagSet(os.Args[1], flag.ExitOnError)
configuration := activationConfig{}
input := flags.String("input", "", "SMTP deny bundle")
previousInput := flags.String("previous-input", "", "previous SMTP deny bundle")
summaryOutput := flags.String("summary-output", "", "sanitized validation summary")
addActivationFlags(flags, &configuration)
_ = flags.Parse(os.Args[2:])
now := time.Now().UTC()
validated, err := readBundle(now, configuration, *input)
if err != nil {
fatal(err)
}
removedIntervals := 0
if *previousInput != "" {
previous, err := readBundle(now, configuration, *previousInput)
if err != nil {
fatal(err)
}
removedIntervals = removedIntervalCount(previous, validated)
}
if *summaryOutput != "" {
summary := struct {
GeneratedAt time.Time `json:"generated_at"`
ExpiresAt time.Time `json:"expires_at"`
DeniedIntervalCount int `json:"denied_interval_count"`
DeniedAddressCount uint64 `json:"denied_address_count"`
RemovedIntervalCount int `json:"removed_interval_count"`
}{
GeneratedAt: validated.GeneratedAt,
ExpiresAt: validated.ExpiresAt,
DeniedIntervalCount: validated.DeniedIntervalCount,
DeniedAddressCount: validated.DeniedAddressCount,
RemovedIntervalCount: removedIntervals,
}
if err := writeJSONFile(*summaryOutput, summary); err != nil {
fatal(err)
}
}
if os.Args[1] == "render" {
if err := render(os.Stdout, now, validated, configuration.publicInterface); err != nil {
fatal(err)
}
}
default:
fatal(fmt.Errorf("unknown command %q", os.Args[1]))
}
}
func writeBundle(path string, document bundle) error {
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return err
}
temporary, err := os.CreateTemp(filepath.Dir(path), ".bundle-")
if err != nil {
return err
}
name := temporary.Name()
defer func() { _ = os.Remove(name) }()
writer := io.Writer(temporary)
var compressed *gzip.Writer
if strings.HasSuffix(path, ".gz") {
compressed = gzip.NewWriter(temporary)
compressed.ModTime = time.Time{}
writer = compressed
}
encoder := json.NewEncoder(writer)
if err := encoder.Encode(document); err != nil {
_ = temporary.Close()
return err
}
if compressed != nil {
if err := compressed.Close(); err != nil {
_ = temporary.Close()
return err
}
}
if err := temporary.Chmod(0600); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(name, path)
}
func writeJSONFile(path string, value any) error {
contents, err := json.Marshal(value)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return err
}
return os.WriteFile(path, append(contents, '\n'), 0644)
}
func removedIntervalCount(previous, current validatedBundle) int {
retained := make(map[[2]uint32]struct{}, len(current.entries))
for _, entry := range current.entries {
retained[[2]uint32{entry.start, entry.end}] = struct{}{}
}
removed := 0
for _, entry := range previous.entries {
if _, exists := retained[[2]uint32{entry.start, entry.end}]; !exists {
removed++
}
}
return removed
}
func addActivationFlags(flags *flag.FlagSet, configuration *activationConfig) {
flags.StringVar(&configuration.publicInterface, "public-interface", "", "public network interface")
flags.StringVar(&configuration.stateDirectory, "state-directory", "/var/lib/oci-l4-relay/smtp-deny", "state directory")
flags.StringVar(&configuration.nft, "nft", "/usr/sbin/nft", "nft path")
flags.Uint64Var(&configuration.minimumObservations, "minimum-observations", 1, "minimum observations for each entry")
flags.Uint64Var(&configuration.maximumAddresses, "maximum-addresses", 1, "maximum covered IPv4 addresses")
flags.IntVar(&configuration.maximumIntervals, "maximum-intervals", 1, "maximum IPv4 intervals")
flags.DurationVar(&configuration.maximumLifetime, "maximum-lifetime", 24*time.Hour, "maximum entry and bundle lifetime")
flags.DurationVar(&configuration.maximumObservationWindow, "maximum-observation-window", time.Hour, "maximum observation window")
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
func activate(now time.Time, configuration activationConfig, bundlePath string, persist bool, source string) error {
if err := writeActivePolicyState(configuration.stateDirectory, now, "transition", nil); err != nil {
return err
}
validated, err := readBundle(now, configuration, bundlePath)
if err != nil {
return err
}
temporaryDirectory, err := os.MkdirTemp(configuration.stateDirectory, ".activate-")
if err != nil {
return err
}
defer func() { _ = os.RemoveAll(temporaryDirectory) }()
rulesetPath := filepath.Join(temporaryDirectory, "smtp-deny.nft")
ruleset, err := os.OpenFile(rulesetPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err != nil {
return err
}
if err := render(ruleset, now, validated, configuration.publicInterface); err != nil {
_ = ruleset.Close()
return err
}
if err := ruleset.Close(); err != nil {
return err
}
if err := ensureTable(configuration.nft); err != nil {
return err
}
if output, err := exec.Command(configuration.nft, "--check", "--file", rulesetPath).CombinedOutput(); err != nil {
return fmt.Errorf("validate rendered ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
if output, err := exec.Command(configuration.nft, "--file", rulesetPath).CombinedOutput(); err != nil {
return fmt.Errorf("load rendered ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
if persist {
if err := replaceFile(bundlePath, filepath.Join(configuration.stateDirectory, "last-good.json.gz")); err != nil {
return err
}
}
if err := writeActivePolicyState(configuration.stateDirectory, now, source, &validated); err != nil {
return err
}
fmt.Printf("denied_intervals=%d denied_addresses=%d expires_at=%s\n", len(validated.entries), validated.DeniedAddressCount, validated.ExpiresAt.Format(time.RFC3339))
return nil
}
func restore(now time.Time, configuration activationConfig) error {
if configuration.publicInterface == "" {
return errors.New("public-interface is required")
}
return activate(now, configuration, filepath.Join(configuration.stateDirectory, "last-good.json.gz"), false, "cache")
}
func failOpen(now time.Time, configuration activationConfig) error {
if err := writeActivePolicyState(configuration.stateDirectory, now, "transition", nil); err != nil {
return err
}
if err := ensureTable(configuration.nft); err != nil {
return err
}
ruleset, err := os.CreateTemp("", "oci-l4-relay-smtp-deny-fail-open-*.nft")
if err != nil {
return err
}
path := ruleset.Name()
defer func() { _ = os.Remove(path) }()
if err := renderFailOpen(ruleset, configuration.publicInterface); err != nil {
_ = ruleset.Close()
return err
}
if err := ruleset.Close(); err != nil {
return err
}
if output, err := exec.Command(configuration.nft, "--check", "--file", path).CombinedOutput(); err != nil {
return fmt.Errorf("validate fail-open ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
if output, err := exec.Command(configuration.nft, "--file", path).CombinedOutput(); err != nil {
return fmt.Errorf("load fail-open ruleset: %w: %s", err, strings.TrimSpace(string(output)))
}
if err := writeActivePolicyState(configuration.stateDirectory, now, "fail-open", nil); err != nil {
return err
}
fmt.Println("smtp_deny_fail_open=true")
return nil
}
func writeActivePolicyState(directory string, now time.Time, source string, active *validatedBundle) error {
state := activePolicyState{ActivatedAt: now, Source: source}
path := filepath.Join(directory, activePolicyStateFile)
if previous, err := readActivePolicyState(path); err == nil {
state.LastRemoteCurrentAt = previous.LastRemoteCurrentAt
}
if active != nil {
state.BundleGeneratedAt = active.GeneratedAt
state.BundleExpiresAt = active.ExpiresAt
}
if source == "current" {
state.LastRemoteCurrentAt = now
}
contents, err := json.Marshal(state)
if err != nil {
return err
}
if err := os.MkdirAll(directory, 0700); err != nil {
return err
}
temporary, err := os.CreateTemp(directory, ".active-state-")
if err != nil {
return err
}
temporaryPath := temporary.Name()
defer func() { _ = os.Remove(temporaryPath) }()
if err := temporary.Chmod(0600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(append(contents, '\n')); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(temporaryPath, path)
}
func readActivePolicyState(path string) (activePolicyState, error) {
contents, err := os.ReadFile(path)
if err != nil {
return activePolicyState{}, err
}
decoder := json.NewDecoder(strings.NewReader(string(contents)))
decoder.DisallowUnknownFields()
state := activePolicyState{}
if err := decoder.Decode(&state); err != nil {
return activePolicyState{}, err
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return activePolicyState{}, errors.New("active policy state contains trailing data")
}
return state, nil
}
func ensureTable(nft string) error {
if err := exec.Command(nft, "list", "table", "inet", tableName).Run(); err == nil {
return nil
}
if output, err := exec.Command(nft, "add", "table", "inet", tableName).CombinedOutput(); err != nil {
return fmt.Errorf("create SMTP deny table: %w: %s", err, strings.TrimSpace(string(output)))
}
return nil
}
func replaceFile(source, destination string) error {
contents, err := os.ReadFile(source)
if err != nil {
return err
}
temporary, err := os.CreateTemp(filepath.Dir(destination), ".last-good-")
if err != nil {
return err
}
name := temporary.Name()
defer func() { _ = os.Remove(name) }()
if err := temporary.Chmod(0600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(contents); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(name, destination)
}
func readBundle(now time.Time, configuration activationConfig, path string) (validatedBundle, error) {
file, err := os.Open(path)
if err != nil {
return validatedBundle{}, err
}
defer func() { _ = file.Close() }()
reader := io.LimitReader(file, maximumBundleBytes+1)
if strings.HasSuffix(path, ".gz") {
compressed, err := gzip.NewReader(reader)
if err != nil {
return validatedBundle{}, fmt.Errorf("open bundle gzip: %w", err)
}
defer func() { _ = compressed.Close() }()
reader = io.LimitReader(compressed, maximumBundleBytes+1)
}
decoder := json.NewDecoder(reader)
decoder.DisallowUnknownFields()
document := bundle{}
if err := decoder.Decode(&document); err != nil {
return validatedBundle{}, fmt.Errorf("decode bundle: %w", err)
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return validatedBundle{}, errors.New("bundle contains trailing data")
}
return validateBundle(now, configuration, document)
}
func validateBundle(now time.Time, configuration activationConfig, document bundle) (validatedBundle, error) {
if configuration.minimumObservations == 0 ||
configuration.maximumAddresses == 0 ||
configuration.maximumIntervals < 0 ||
configuration.maximumIntervals > absoluteMaxEntries ||
configuration.maximumLifetime <= 0 ||
configuration.maximumLifetime > absoluteMaxLife ||
configuration.maximumObservationWindow <= 0 ||
configuration.maximumObservationWindow > configuration.maximumLifetime {
return validatedBundle{}, errors.New("invalid policy bounds")
}
if document.SchemaVersion != 1 {
return validatedBundle{}, fmt.Errorf("unsupported schema version %d", document.SchemaVersion)
}
if document.GeneratedAt.After(now.Add(5 * time.Minute)) {
return validatedBundle{}, errors.New("bundle generation time is in the future")
}
if !document.ExpiresAt.After(now) ||
!document.ExpiresAt.After(document.GeneratedAt) ||
document.ExpiresAt.Sub(document.GeneratedAt) > configuration.maximumLifetime ||
document.GeneratedAt.Before(now.Add(-configuration.maximumLifetime)) {
return validatedBundle{}, errors.New("bundle lifetime or generation age exceeds bounds")
}
if len(document.DeniedIPv4) > configuration.maximumIntervals {
return validatedBundle{}, errors.New("denied interval count exceeds configured bound")
}
entries := make([]parsedEntry, 0, len(document.DeniedIPv4))
var addressCount uint64
var previousEnd uint32
for index, entry := range document.DeniedIPv4 {
start, err := parseIPv4(entry.Start)
if err != nil {
return validatedBundle{}, fmt.Errorf("entry %d start: %w", index, err)
}
end, err := parseIPv4(entry.End)
if err != nil {
return validatedBundle{}, fmt.Errorf("entry %d end: %w", index, err)
}
if start > end {
return validatedBundle{}, fmt.Errorf("entry %d is reversed", index)
}
if index > 0 && start <= previousEnd {
return validatedBundle{}, fmt.Errorf("entry %d overlaps or is unsorted", index)
}
if _, exists := transportReasons[entry.Reason]; !exists {
return validatedBundle{}, fmt.Errorf("entry %d has invalid transport reason", index)
}
if entry.ObservationCount < configuration.minimumObservations {
return validatedBundle{}, fmt.Errorf("entry %d is below the observation threshold", index)
}
if entry.ObservationStarted.After(entry.ObservationEnded) ||
entry.ObservationEnded.After(document.GeneratedAt.Add(5*time.Minute)) ||
entry.ObservationEnded.Sub(entry.ObservationStarted) > configuration.maximumObservationWindow {
return validatedBundle{}, fmt.Errorf("entry %d has invalid observation window", index)
}
if !entry.ExpiresAt.After(entry.ObservationEnded) ||
entry.ExpiresAt.After(document.ExpiresAt) ||
entry.ExpiresAt.Sub(entry.ObservationEnded) > configuration.maximumLifetime {
return validatedBundle{}, fmt.Errorf("entry %d has invalid expiry", index)
}
addressCount += uint64(end) - uint64(start) + 1
if addressCount > configuration.maximumAddresses {
return validatedBundle{}, errors.New("denied address count exceeds configured bound")
}
entries = append(entries, parsedEntry{start: start, end: end, expiresAt: entry.ExpiresAt})
previousEnd = end
}
if document.DeniedIntervalCount != len(entries) || document.DeniedAddressCount != addressCount {
return validatedBundle{}, errors.New("denied statistics do not match payload")
}
return validatedBundle{bundle: document, entries: entries}, nil
}
func parseIPv4(value string) (uint32, error) {
address, err := netip.ParseAddr(value)
if err != nil || !address.Is4() || address.String() != value {
return 0, fmt.Errorf("invalid canonical IPv4 address %q", value)
}
bytes := address.As4()
return uint32(bytes[0])<<24 | uint32(bytes[1])<<16 | uint32(bytes[2])<<8 | uint32(bytes[3]), nil
}
func ipv4String(value uint32) string {
return netip.AddrFrom4([4]byte{byte(value >> 24), byte(value >> 16), byte(value >> 8), byte(value)}).String()
}
func render(output io.Writer, now time.Time, document validatedBundle, publicInterface string) error {
if err := validateInterface(publicInterface); err != nil {
return err
}
writer := bufio.NewWriter(output)
if _, err := fmt.Fprintf(writer, "delete table inet %s\ntable inet %s {\n", tableName, tableName); err != nil {
return err
}
if _, err := fmt.Fprintln(writer, " set denied_v4 {\n type ipv4_addr\n flags interval, timeout"); err != nil {
return err
}
active := make([]parsedEntry, 0, len(document.entries))
for _, entry := range document.entries {
if entry.expiresAt.Sub(now) >= time.Second {
active = append(active, entry)
}
}
if len(active) > 0 {
if _, err := fmt.Fprintln(writer, " elements = {"); err != nil {
return err
}
for index, entry := range active {
address := ipv4String(entry.start)
if entry.start != entry.end {
address += "-" + ipv4String(entry.end)
}
separator := ","
if index == len(active)-1 {
separator = ""
}
if _, err := fmt.Fprintf(writer, " %s timeout %ds%s\n", address, int64(entry.expiresAt.Sub(now)/time.Second), separator); err != nil {
return err
}
}
if _, err := fmt.Fprintln(writer, " }"); err != nil {
return err
}
}
if _, err := fmt.Fprintf(writer, " }\n counter smtp_denied {}\n chain ingress {\n type filter hook prerouting priority raw - 19; policy accept;\n iifname %s tcp dport 25 ip saddr @denied_v4 counter name smtp_denied drop\n }\n}\n", strconv.Quote(publicInterface)); err != nil {
return err
}
return writer.Flush()
}
func renderFailOpen(output io.Writer, publicInterface string) error {
if err := validateInterface(publicInterface); err != nil {
return err
}
_, err := fmt.Fprintf(output, "delete table inet %s\ntable inet %s {\n set denied_v4 {\n type ipv4_addr\n flags interval, timeout\n }\n counter smtp_denied {}\n chain ingress {\n type filter hook prerouting priority raw - 19; policy accept;\n iifname %s tcp dport 25 ip saddr @denied_v4 counter name smtp_denied drop\n }\n}\n", tableName, tableName, strconv.Quote(publicInterface))
return err
}
func validateInterface(value string) error {
if value == "" || strings.ContainsAny(value, "\"\\\n\r\t ") {
return errors.New("invalid public interface")
}
return nil
}

400
services/oci-l4-relay-guest/cmd/smtp-deny/main_test.go Normal file
View file

@ -0,0 +1,400 @@
package main
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func policy() activationConfig {
return activationConfig{
publicInterface: "ens3",
minimumObservations: 20,
maximumAddresses: 4096,
maximumIntervals: 1024,
maximumLifetime: 24 * time.Hour,
maximumObservationWindow: 10 * time.Minute,
}
}
func validBundle(now time.Time) bundle {
return bundle{
SchemaVersion: 1,
GeneratedAt: now,
ExpiresAt: now.Add(24 * time.Hour),
DeniedIPv4: []denyEntry{{
Start: "192.0.2.1",
End: "192.0.2.1",
Reason: "connection-rate",
ObservationStarted: now.Add(-10 * time.Minute),
ObservationEnded: now,
ObservationCount: 20,
ExpiresAt: now.Add(24 * time.Hour),
}},
DeniedIntervalCount: 1,
DeniedAddressCount: 1,
}
}
func TestWriteAndReadEmptyBundle(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
path := filepath.Join(t.TempDir(), "empty", "current.json.gz")
document := bundle{
SchemaVersion: 1,
GeneratedAt: now,
ExpiresAt: now.Add(24 * time.Hour),
DeniedIPv4: []denyEntry{},
}
if err := writeBundle(path, document); err != nil {
t.Fatal(err)
}
validated, err := readBundle(now, policy(), path)
if err != nil {
t.Fatal(err)
}
if validated.DeniedIntervalCount != 0 || validated.DeniedAddressCount != 0 || validated.DeniedIPv4 == nil {
t.Fatal("empty bundle did not retain an explicit empty deny list")
}
}
func TestRemovedIntervalCount(t *testing.T) {
now := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC)
previous, err := validateBundle(now, policy(), validBundle(now))
if err != nil {
t.Fatal(err)
}
currentDocument := validBundle(now)
currentDocument.DeniedIPv4 = []denyEntry{}
currentDocument.DeniedIntervalCount = 0
currentDocument.DeniedAddressCount = 0
current, err := validateBundle(now, policy(), currentDocument)
if err != nil {
t.Fatal(err)
}
if removedIntervalCount(previous, current) != 1 {
t.Fatal("omitted range was not recorded as removed")
}
}
func TestWriteJSONFileCreatesParentDirectory(t *testing.T) {
path := filepath.Join(t.TempDir(), "audit", "summary.json")
if err := writeJSONFile(path, map[string]int{"removed_interval_count": 1}); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(contents) != "{\"removed_interval_count\":1}\n" {
t.Fatalf("unexpected summary %q", contents)
}
}
func TestWriteActivePolicyStatePreservesRemoteCurrentActivation(t *testing.T) {
now := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)
directory := t.TempDir()
validated, err := validateBundle(now, policy(), validBundle(now))
if err != nil {
t.Fatal(err)
}
if err := writeActivePolicyState(directory, now, "current", &validated); err != nil {
t.Fatal(err)
}
cacheActivation := now.Add(time.Minute)
if err := writeActivePolicyState(directory, cacheActivation, "cache", &validated); err != nil {
t.Fatal(err)
}
state, err := readActivePolicyState(filepath.Join(directory, activePolicyStateFile))
if err != nil {
t.Fatal(err)
}
if state.Source != "cache" || !state.ActivatedAt.Equal(cacheActivation) || !state.LastRemoteCurrentAt.Equal(now) {
t.Fatalf("unexpected active state %#v", state)
}
if !state.BundleGeneratedAt.Equal(validated.GeneratedAt) || !state.BundleExpiresAt.Equal(validated.ExpiresAt) {
t.Fatalf("unexpected bundle state %#v", state)
}
}
func TestWriteActivePolicyStateRecordsFailOpen(t *testing.T) {
now := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)
directory := t.TempDir()
if err := writeActivePolicyState(directory, now, "fail-open", nil); err != nil {
t.Fatal(err)
}
state, err := readActivePolicyState(filepath.Join(directory, activePolicyStateFile))
if err != nil {
t.Fatal(err)
}
if state.Source != "fail-open" || !state.BundleGeneratedAt.IsZero() || !state.BundleExpiresAt.IsZero() {
t.Fatalf("unexpected fail-open state %#v", state)
}
}
func TestFailOpenRecordsFinalStateAfterRulesLoad(t *testing.T) {
now := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)
directory := t.TempDir()
observedPath := filepath.Join(directory, "observed-state.json")
nftPath := filepath.Join(directory, "nft")
script := "#!/bin/sh\nif [ \"$1\" = \"--file\" ]; then cp \"$STATE_PATH\" \"$OBSERVED_PATH\"; exit \"${NFT_LOAD_EXIT:-0}\"; fi\nexit 0\n"
if err := os.WriteFile(nftPath, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("STATE_PATH", filepath.Join(directory, activePolicyStateFile))
t.Setenv("OBSERVED_PATH", observedPath)
configuration := policy()
configuration.stateDirectory = directory
configuration.nft = nftPath
if err := failOpen(now, configuration); err != nil {
t.Fatal(err)
}
observed, err := readActivePolicyState(observedPath)
if err != nil {
t.Fatal(err)
}
if observed.Source != "transition" {
t.Fatalf("rules loaded after source %q was reported", observed.Source)
}
final, err := readActivePolicyState(filepath.Join(directory, activePolicyStateFile))
if err != nil {
t.Fatal(err)
}
if final.Source != "fail-open" {
t.Fatalf("unexpected final source %q", final.Source)
}
}
func TestActivationRecordsFinalStateAfterRulesLoad(t *testing.T) {
now := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)
directory := t.TempDir()
bundlePath := filepath.Join(directory, "current.json.gz")
if err := writeBundle(bundlePath, validBundle(now)); err != nil {
t.Fatal(err)
}
observedPath := filepath.Join(directory, "observed-state.json")
nftPath := filepath.Join(directory, "nft")
script := "#!/bin/sh\nif [ \"$1\" = \"--file\" ]; then cp \"$STATE_PATH\" \"$OBSERVED_PATH\"; fi\nexit 0\n"
if err := os.WriteFile(nftPath, []byte(script), 0700); err != nil {
t.Fatal(err)
}
t.Setenv("STATE_PATH", filepath.Join(directory, activePolicyStateFile))
t.Setenv("OBSERVED_PATH", observedPath)
configuration := policy()
configuration.stateDirectory = directory
configuration.nft = nftPath
if err := activate(now, configuration, bundlePath, true, "current"); err != nil {
t.Fatal(err)
}
observed, err := readActivePolicyState(observedPath)
if err != nil {
t.Fatal(err)
}
if observed.Source != "transition" {
t.Fatalf("rules loaded after source %q was reported", observed.Source)
}
final, err := readActivePolicyState(filepath.Join(directory, activePolicyStateFile))
if err != nil {
t.Fatal(err)
}
if final.Source != "current" {
t.Fatalf("unexpected final source %q", final.Source)
}
}
func TestFailOpenLoadFailureRemainsTransitional(t *testing.T) {
now := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC)
directory := t.TempDir()
nftPath := filepath.Join(directory, "nft")
script := "#!/bin/sh\nif [ \"$1\" = \"--file\" ]; then exit 1; fi\nexit 0\n"
if err := os.WriteFile(nftPath, []byte(script), 0700); err != nil {
t.Fatal(err)
}
configuration := policy()
configuration.stateDirectory = directory
configuration.nft = nftPath
if err := failOpen(now, configuration); err == nil {
t.Fatal("expected nftables load failure")
}
state, err := readActivePolicyState(filepath.Join(directory, activePolicyStateFile))
if err != nil {
t.Fatal(err)
}
if state.Source != "transition" {
t.Fatalf("failed rules load reported source %q", state.Source)
}
}
func TestValidateAndRender(t *testing.T) {
now := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC)
validated, err := validateBundle(now, policy(), validBundle(now))
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
if err := render(&output, now, validated, "ens3"); err != nil {
t.Fatal(err)
}
for _, expected := range []string{
"table inet oci_l4_relay_smtp_deny",
"flags interval, timeout",
"192.0.2.1 timeout 86400s",
`iifname "ens3" tcp dport 25`,
"counter name smtp_denied drop",
} {
if !strings.Contains(output.String(), expected) {
t.Errorf("rendered ruleset does not contain %q", expected)
}
}
for _, excluded := range []string{"dport 465", "dport 587", "dport 993", "dport { 80, 443 }"} {
if strings.Contains(output.String(), excluded) {
t.Errorf("rendered SMTP deny ruleset contains %q", excluded)
}
}
}
func TestEmptyBundleIsValidAndFailOpenIsEmpty(t *testing.T) {
now := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC)
document := validBundle(now)
document.DeniedIPv4 = nil
document.DeniedIntervalCount = 0
document.DeniedAddressCount = 0
validated, err := validateBundle(now, policy(), document)
if err != nil {
t.Fatal(err)
}
var rendered bytes.Buffer
if err := render(&rendered, now, validated, "ens3"); err != nil {
t.Fatal(err)
}
var failOpen bytes.Buffer
if err := renderFailOpen(&failOpen, "ens3"); err != nil {
t.Fatal(err)
}
for name, value := range map[string]string{"empty bundle": rendered.String(), "fail open": failOpen.String()} {
if strings.Contains(value, "elements =") {
t.Errorf("%s unexpectedly contains deny elements", name)
}
if !strings.Contains(value, "ip saddr @denied_v4") {
t.Errorf("%s does not retain the empty deny lookup", name)
}
}
}
func TestExpiredEntryDecaysWithoutInvalidatingBundle(t *testing.T) {
now := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC)
document := validBundle(now)
document.DeniedIPv4[0].ObservationEnded = now.Add(-2 * time.Minute)
document.DeniedIPv4[0].ExpiresAt = now.Add(-time.Minute)
validated, err := validateBundle(now, policy(), document)
if err != nil {
t.Fatal(err)
}
var output bytes.Buffer
if err := render(&output, now, validated, "ens3"); err != nil {
t.Fatal(err)
}
if strings.Contains(output.String(), "192.0.2.1") {
t.Fatal("expired entry remained in the rendered deny set")
}
}
func TestPolicyBoundaries(t *testing.T) {
now := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC)
maximum := validBundle(now)
maximum.DeniedIPv4[0].Start = "192.0.2.0"
maximum.DeniedIPv4[0].End = "192.0.17.255"
maximum.DeniedAddressCount = 4096
if _, err := validateBundle(now, policy(), maximum); err != nil {
t.Fatal(err)
}
tests := map[string]func(*bundle){
"observation threshold": func(value *bundle) {
value.DeniedIPv4[0].ObservationCount = 19
},
"address cap": func(value *bundle) {
value.DeniedIPv4[0].Start = "192.0.2.0"
value.DeniedIPv4[0].End = "192.0.18.0"
value.DeniedAddressCount = 4097
},
"lifetime": func(value *bundle) {
value.ExpiresAt = now.Add(24*time.Hour + time.Second)
value.DeniedIPv4[0].ExpiresAt = value.ExpiresAt
},
"observation window": func(value *bundle) {
value.DeniedIPv4[0].ObservationStarted = now.Add(-10*time.Minute - time.Second)
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
value := validBundle(now)
mutate(&value)
if _, err := validateBundle(now, policy(), value); err == nil {
t.Fatal("expected policy-bound failure")
}
})
}
intervals := validBundle(now)
intervals.DeniedIPv4 = make([]denyEntry, 1025)
for index := range intervals.DeniedIPv4 {
entry := validBundle(now).DeniedIPv4[0]
entry.Start = ipv4String(uint32(index * 2))
entry.End = entry.Start
intervals.DeniedIPv4[index] = entry
}
intervals.DeniedIntervalCount = len(intervals.DeniedIPv4)
intervals.DeniedAddressCount = uint64(len(intervals.DeniedIPv4))
if _, err := validateBundle(now, policy(), intervals); err == nil {
t.Fatal("expected interval-cap failure")
}
}
func TestValidationRejectsNonTransportOrMalformedEvidence(t *testing.T) {
now := time.Date(2026, 7, 26, 0, 0, 0, 0, time.UTC)
tests := map[string]func(*bundle){
"schema": func(value *bundle) { value.SchemaVersion = 2 },
"expired": func(value *bundle) { value.ExpiresAt = now },
"ipv6": func(value *bundle) { value.DeniedIPv4[0].Start = "2001:db8::1" },
"reason": func(value *bundle) { value.DeniedIPv4[0].Reason = "spam-score" },
"future window": func(value *bundle) {
value.DeniedIPv4[0].ObservationEnded = now.Add(6 * time.Minute)
},
}
for name, mutate := range tests {
t.Run(name, func(t *testing.T) {
value := validBundle(now)
mutate(&value)
if _, err := validateBundle(now, policy(), value); err == nil {
t.Fatal("expected validation failure")
}
})
}
}
func TestReadBundleRejectsContentEvidence(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
contents, err := json.Marshal(validBundle(now))
if err != nil {
t.Fatal(err)
}
contents = bytes.Replace(contents, []byte(`"reason":"connection-rate"`), []byte(`"reason":"connection-rate","spam_score":9`), 1)
path := filepath.Join(t.TempDir(), "bundle.json")
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := readBundle(now, policy(), path); err == nil {
t.Fatal("expected content evidence to be rejected")
}
}
func TestRejectsUnsafeInterface(t *testing.T) {
if err := renderFailOpen(&bytes.Buffer{}, "ens3\nflush ruleset"); err == nil {
t.Fatal("expected unsafe interface rejection")
}
}

213
services/oci-l4-relay-guest/cmd/smtp-evidence/conntrack.go Normal file
View file

@ -0,0 +1,213 @@
package main
import (
"context"
"encoding/binary"
"encoding/json"
"fmt"
"hash/fnv"
"io"
"net/netip"
"sync/atomic"
"syscall"
"time"
"github.com/ti-mo/conntrack"
"github.com/ti-mo/netfilter"
)
type conntrackMetrics struct {
Emitted atomic.Uint64
Eligible atomic.Uint64
RateDropped atomic.Uint64
SampledOut atomic.Uint64
Up atomic.Bool
}
type conntrackRecord struct {
DurationMilliseconds uint64 `json:"duration_milliseconds"`
EndedAt time.Time `json:"ended_at"`
Original conntrackTuple `json:"original"`
OriginalBytes uint64 `json:"original_bytes"`
OriginalPackets uint64 `json:"original_packets"`
Reply conntrackTuple `json:"reply"`
ReplyBytes uint64 `json:"reply_bytes"`
ReplyPackets uint64 `json:"reply_packets"`
SchemaVersion int `json:"schema_version"`
Service string `json:"service"`
StartedAt time.Time `json:"started_at"`
TCPState string `json:"tcp_state"`
Type string `json:"type"`
}
type conntrackTuple struct {
DestinationAddress string `json:"destination_address"`
DestinationPort uint16 `json:"destination_port"`
SourceAddress string `json:"source_address"`
SourcePort uint16 `json:"source_port"`
}
type tokenBucket struct {
burst float64
last time.Time
rate float64
remaining float64
}
func newTokenBucket(rate, burst int, now time.Time) tokenBucket {
return tokenBucket{burst: float64(burst), last: now, rate: float64(rate), remaining: float64(burst)}
}
func (bucket *tokenBucket) allow(now time.Time) bool {
if now.After(bucket.last) {
bucket.remaining = min(bucket.burst, bucket.remaining+now.Sub(bucket.last).Seconds()*bucket.rate)
bucket.last = now
}
if bucket.remaining < 1 {
return false
}
bucket.remaining--
return true
}
func observeConntrack(ctx context.Context, configuration relayConfig, metrics *conntrackMetrics, output io.Writer) error {
connection, err := conntrack.Dial(nil)
if err != nil {
return err
}
defer connection.Close()
events := make(chan conntrack.Event, configuration.ConntrackEventBuffer)
errors, err := connection.Listen(events, 1, []netfilter.NetlinkGroup{netfilter.GroupCTDestroy})
if err != nil {
return err
}
metrics.Up.Store(true)
defer metrics.Up.Store(false)
go func() {
<-ctx.Done()
connection.Close()
}()
encoder := json.NewEncoder(output)
limiter := newTokenBucket(configuration.ConntrackEmissionRatePerSecond, configuration.ConntrackEmissionBurst, time.Now())
for {
select {
case <-ctx.Done():
return ctx.Err()
case err := <-errors:
return err
case event := <-events:
record, eligible := conntrackRecordForFlow(event.Flow, configuration.ConntrackServices)
if !eligible {
continue
}
metrics.Eligible.Add(1)
if !sampleConntrackFlow(event.Flow, configuration.ConntrackSampleDenominator) {
metrics.SampledOut.Add(1)
continue
}
if !limiter.allow(time.Now()) {
metrics.RateDropped.Add(1)
continue
}
if err := encoder.Encode(record); err != nil {
return err
}
metrics.Emitted.Add(1)
}
}
}
func conntrackRecordForFlow(flow *conntrack.Flow, services []conntrackService) (conntrackRecord, bool) {
if flow == nil || flow.TupleOrig.Proto.Protocol != syscall.IPPROTO_TCP || flow.TupleReply.Proto.Protocol != syscall.IPPROTO_TCP {
return conntrackRecord{}, false
}
service := ""
for _, candidate := range services {
if flow.TupleReply.IP.SourceAddress != candidate.TargetAddress {
continue
}
for _, port := range candidate.Ports {
if flow.TupleOrig.Proto.DestinationPort == port {
service = candidate.Name
break
}
}
}
if service == "" || !validTuple(flow.TupleOrig) || !validTuple(flow.TupleReply) || flow.Timestamp.Start.IsZero() {
return conntrackRecord{}, false
}
endedAt := flow.Timestamp.Stop
if endedAt.IsZero() {
endedAt = time.Now()
}
duration := endedAt.Sub(flow.Timestamp.Start)
if duration < 0 {
return conntrackRecord{}, false
}
state := "unknown"
if flow.ProtoInfo.TCP != nil {
state = tcpState(flow.ProtoInfo.TCP.State)
}
return conntrackRecord{
DurationMilliseconds: uint64(duration.Milliseconds()),
EndedAt: endedAt.UTC(),
Original: recordTuple(flow.TupleOrig),
OriginalBytes: flow.CountersOrig.Bytes,
OriginalPackets: flow.CountersOrig.Packets,
Reply: recordTuple(flow.TupleReply),
ReplyBytes: flow.CountersReply.Bytes,
ReplyPackets: flow.CountersReply.Packets,
SchemaVersion: 1,
Service: service,
StartedAt: flow.Timestamp.Start.UTC(),
TCPState: state,
Type: "oci_l4_relay_conntrack",
}, true
}
func validTuple(tuple conntrack.Tuple) bool {
return tuple.IP.SourceAddress.IsValid() && tuple.IP.DestinationAddress.IsValid() && tuple.Proto.SourcePort != 0 && tuple.Proto.DestinationPort != 0
}
func recordTuple(tuple conntrack.Tuple) conntrackTuple {
return conntrackTuple{
DestinationAddress: tuple.IP.DestinationAddress.String(),
DestinationPort: tuple.Proto.DestinationPort,
SourceAddress: tuple.IP.SourceAddress.String(),
SourcePort: tuple.Proto.SourcePort,
}
}
func sampleConntrackFlow(flow *conntrack.Flow, denominator uint64) bool {
if denominator == 1 {
return true
}
hash := fnv.New64a()
for _, address := range []netip.Addr{
flow.TupleOrig.IP.SourceAddress,
flow.TupleOrig.IP.DestinationAddress,
flow.TupleReply.IP.SourceAddress,
flow.TupleReply.IP.DestinationAddress,
} {
_, _ = hash.Write(address.AsSlice())
}
var values [8]byte
for _, port := range []uint16{
flow.TupleOrig.Proto.SourcePort,
flow.TupleOrig.Proto.DestinationPort,
flow.TupleReply.Proto.SourcePort,
flow.TupleReply.Proto.DestinationPort,
} {
binary.BigEndian.PutUint16(values[:2], port)
_, _ = hash.Write(values[:2])
}
return hash.Sum64()%denominator == 0
}
func tcpState(value uint8) string {
states := []string{"none", "syn_sent", "syn_recv", "established", "fin_wait", "close_wait", "last_ack", "time_wait", "close", "syn_sent2"}
if int(value) >= len(states) {
return fmt.Sprintf("unknown_%d", value)
}
return states[value]
}

63
services/oci-l4-relay-guest/cmd/smtp-evidence/conntrack_test.go Normal file
View file

@ -0,0 +1,63 @@
package main
import (
"net/netip"
"testing"
"time"
"github.com/ti-mo/conntrack"
)
func TestConntrackRecordRequiresDeclaredPortAndReplyTarget(t *testing.T) {
flow := acceptedFlow()
record, ok := conntrackRecordForFlow(&flow, []conntrackService{{
Name: "web",
Ports: []uint16{80, 443},
TargetAddress: netip.MustParseAddr("10.104.40.127"),
}})
if !ok {
t.Fatal("accepted relay flow was not classified")
}
if record.Service != "web" || record.Original.SourceAddress != "198.18.0.1" || record.Reply.SourceAddress != "10.104.40.127" || record.DurationMilliseconds != 1500 || record.TCPState != "time_wait" {
t.Fatalf("unexpected record %#v", record)
}
flow.TupleReply.IP.SourceAddress = netip.MustParseAddr("203.0.113.10")
if _, ok := conntrackRecordForFlow(&flow, []conntrackService{{Name: "web", Ports: []uint16{443}, TargetAddress: netip.MustParseAddr("10.104.40.127")}}); ok {
t.Fatal("outbound connection was classified as accepted relay traffic")
}
}
func TestConntrackSamplingIsDeterministic(t *testing.T) {
flow := acceptedFlow()
first := sampleConntrackFlow(&flow, 10)
for attempt := 0; attempt < 100; attempt++ {
if sampleConntrackFlow(&flow, 10) != first {
t.Fatal("sampling decision changed for an unchanged tuple")
}
}
}
func TestTokenBucketBoundsBurstAndRefills(t *testing.T) {
now := time.Date(2026, 7, 31, 0, 0, 0, 0, time.UTC)
bucket := newTokenBucket(1, 2, now)
if !bucket.allow(now) || !bucket.allow(now) || bucket.allow(now) {
t.Fatal("initial burst was not bounded")
}
if !bucket.allow(now.Add(time.Second)) || bucket.allow(now.Add(time.Second)) {
t.Fatal("sustained rate was not bounded")
}
}
func acceptedFlow() conntrack.Flow {
startedAt := time.Date(2026, 7, 31, 0, 0, 0, 0, time.UTC)
flow := conntrack.NewFlow(6, 0, netip.MustParseAddr("198.18.0.1"), netip.MustParseAddr("10.49.127.43"), 50000, 443, 60, 0)
flow.TupleReply.IP.SourceAddress = netip.MustParseAddr("10.104.40.127")
flow.TupleReply.IP.DestinationAddress = netip.MustParseAddr("198.18.0.1")
flow.TupleReply.Proto.SourcePort = 443
flow.TupleReply.Proto.DestinationPort = 50000
flow.Timestamp = conntrack.Timestamp{Start: startedAt, Stop: startedAt.Add(1500 * time.Millisecond)}
flow.CountersOrig = conntrack.Counter{Packets: 3, Bytes: 180}
flow.CountersReply = conntrack.Counter{Direction: true, Packets: 4, Bytes: 240}
flow.ProtoInfo.TCP = &conntrack.ProtoInfoTCP{State: 7}
return flow
}

618
services/oci-l4-relay-guest/cmd/smtp-evidence/main.go Normal file
View file

@ -0,0 +1,618 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/netip"
"os"
"os/exec"
"path/filepath"
"sort"
"sync/atomic"
"time"
)
const (
defaultConfigPath = "/etc/oci-l4-relay-smtp-evidence.json"
defaultNftPath = "/usr/sbin/nft"
defaultSnapshotPath = "/var/lib/oci-l4-relay/smtp-evidence/current.json"
defaultTCPath = "/usr/sbin/tc"
defaultTelemetryPath = "/run/oci-l4-relay-telemetry/current.json"
nftablesFamily = "inet"
nftablesTable = "oci_l4_relay"
reason = "syn-rate"
)
var evidenceSets = []evidenceSet{
{Service: "web", Name: "syn_quarantine_v4"},
{Service: "smtp", Name: "smtp_syn_quarantine_v4"},
}
var startConntrackObserver = observeConntrack
var dropMetricSpecs = []dropMetricSpec{
{Name: "web_allow_denied", Service: "web", Reason: "admission"},
{Name: "web_syn_source_dropped", Service: "web", Reason: "source_rate"},
{Name: "web_syn_quarantine_dropped", Service: "web", Reason: "quarantine"},
{Name: "web_syn_aggregate_dropped", Service: "web", Reason: "aggregate_syn"},
{Name: "web_packet_aggregate_dropped", Service: "web", Reason: "aggregate_packet"},
{Name: "smtp_syn_source_dropped", Service: "smtp", Reason: "source_rate"},
{Name: "smtp_syn_quarantine_dropped", Service: "smtp", Reason: "quarantine"},
{Name: "smtp_syn_aggregate_dropped", Service: "smtp", Reason: "aggregate_syn"},
{Name: "smtp_packet_aggregate_dropped", Service: "smtp", Reason: "aggregate_packet"},
{Name: "smtp_denied", Service: "smtp", Reason: "deny_policy", MailOnly: true},
{Name: "auth_syn_source_dropped", Service: "authenticated_mail", Reason: "source_rate"},
{Name: "auth_syn_quarantine_dropped", Service: "authenticated_mail", Reason: "quarantine"},
{Name: "auth_syn_aggregate_dropped", Service: "authenticated_mail", Reason: "aggregate_syn"},
{Name: "auth_packet_aggregate_dropped", Service: "authenticated_mail", Reason: "aggregate_packet"},
}
type relayConfig struct {
ConntrackEmissionBurst int `json:"conntrack_emission_burst"`
ConntrackEmissionRatePerSecond int `json:"conntrack_emission_rate_per_second"`
ConntrackEventBuffer int `json:"conntrack_event_buffer"`
ConntrackSampleDenominator uint64 `json:"conntrack_sample_denominator"`
ConntrackServices []conntrackService `json:"conntrack_services"`
MaximumRecords uint64 `json:"maximum_records"`
MetricsListenAddress string `json:"metrics_listen_address"`
ObservationWindowSeconds int `json:"observation_window_seconds"`
SnapshotIntervalSeconds int `json:"snapshot_interval_seconds"`
}
type conntrackService struct {
Name string `json:"name"`
Ports []uint16 `json:"ports"`
TargetAddress netip.Addr `json:"target_address"`
}
type evidenceSet struct {
Name string
Service string
}
type evidenceRecord struct {
Address string `json:"address"`
Count uint64 `json:"count"`
ExpiresAt time.Time `json:"expires_at"`
}
type snapshot struct {
SchemaVersion int `json:"schema_version"`
GeneratedAt time.Time `json:"generated_at"`
Reason string `json:"reason"`
ObservationWindowSeconds int `json:"observation_window_seconds"`
Records []evidenceRecord `json:"records"`
}
type serviceMetrics struct {
Records []evidenceRecord
Service string
Total uint64
}
type dropMetric struct {
Packets uint64
Reason string
Service string
}
type dropMetricSpec struct {
MailOnly bool
Name string
Reason string
Service string
}
type metricsSnapshot struct {
Drops []dropMetric
GeneratedAt time.Time
Health relayHealth
QdiscDrops []qdiscMetric
Services []serviceMetrics
TelemetryAt time.Time
}
type qdiscMetric struct {
Channel string
Packets uint64
}
type relayHealth struct {
ConntrackCount int64 `json:"conntrack_count"`
ConntrackMaximum int64 `json:"conntrack_maximum"`
HandshakeAgeSeconds int64 `json:"handshake_age_seconds"`
NftablesActive bool `json:"nftables_active"`
SMTPDenyActive bool `json:"smtp_deny_active"`
SMTPDenyActiveSource string `json:"smtp_deny_active_source,omitempty"`
SMTPDenyBundleAgeSeconds int64 `json:"smtp_deny_bundle_age_seconds"`
SMTPDenyBundleRemainingSeconds int64 `json:"smtp_deny_bundle_remaining_seconds"`
SMTPDenyCached bool `json:"smtp_deny_cached"`
SMTPDenyPolicyValid bool `json:"smtp_deny_policy_valid"`
SMTPDenyRefreshCompletedAgeSeconds int64 `json:"smtp_deny_refresh_completed_age_seconds"`
SMTPDenyRefreshSuccessful bool `json:"smtp_deny_refresh_successful"`
SMTPDenyRemoteCurrentAgeSeconds int64 `json:"smtp_deny_remote_current_age_seconds"`
WebAllowActive bool `json:"web_allow_active"`
WebAllowCached bool `json:"web_allow_cached"`
WireGuardActive bool `json:"wireguard_active"`
}
type telemetrySnapshot struct {
GeneratedAt time.Time `json:"generated_at"`
Status relayHealth `json:"status"`
}
type qdiscDocumentItem struct {
Drops uint64 `json:"drops"`
Parent string `json:"parent"`
}
type nftablesSetDocument struct {
Nftables []struct {
Set *struct {
Family string `json:"family"`
Name string `json:"name"`
Table string `json:"table"`
Elem []struct {
Element struct {
Value string `json:"val"`
Expires int64 `json:"expires"`
Counter *struct {
Packets uint64 `json:"packets"`
} `json:"counter"`
} `json:"elem"`
} `json:"elem"`
} `json:"set,omitempty"`
} `json:"nftables"`
}
type nftablesCounterDocument struct {
Nftables []struct {
Counter *struct {
Name string `json:"name"`
Packets uint64 `json:"packets"`
} `json:"counter"`
} `json:"nftables"`
}
func main() {
configPath := flag.String("config", defaultConfigPath, "relay configuration path")
nftPath := flag.String("nft", defaultNftPath, "nft path")
snapshotPath := flag.String("snapshot", defaultSnapshotPath, "snapshot path")
tcPath := flag.String("tc", defaultTCPath, "tc path")
telemetryPath := flag.String("telemetry", defaultTelemetryPath, "telemetry snapshot path")
once := flag.Bool("once", false, "write one snapshot and exit")
flag.Parse()
configuration, err := loadConfig(*configPath)
if err != nil {
fatal(err)
}
if err := run(context.Background(), configuration, *nftPath, *tcPath, *telemetryPath, *snapshotPath, *once); err != nil {
fatal(err)
}
}
func loadConfig(path string) (relayConfig, error) {
contents, err := os.ReadFile(path)
if err != nil {
return relayConfig{}, err
}
var configuration relayConfig
decoder := json.NewDecoder(bytes.NewReader(contents))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&configuration); err != nil {
return relayConfig{}, err
}
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
return relayConfig{}, errors.New("SMTP evidence configuration has trailing data")
}
if configuration.MaximumRecords == 0 || configuration.ObservationWindowSeconds <= 0 {
return relayConfig{}, errors.New("SMTP evidence bounds must be positive")
}
if configuration.ConntrackEmissionBurst <= 0 || configuration.ConntrackEmissionRatePerSecond <= 0 || configuration.ConntrackEventBuffer <= 0 || configuration.ConntrackSampleDenominator == 0 {
return relayConfig{}, errors.New("conntrack observation bounds must be positive")
}
if configuration.ConntrackEventBuffer > 65536 || configuration.ConntrackEmissionBurst > 1000 || configuration.ConntrackEmissionRatePerSecond > 100 {
return relayConfig{}, errors.New("conntrack observation bounds exceed tested limits")
}
if len(configuration.ConntrackServices) == 0 {
return relayConfig{}, errors.New("at least one conntrack service is required")
}
serviceNames := make(map[string]bool)
servicePorts := make(map[uint16]bool)
for _, service := range configuration.ConntrackServices {
if service.Name == "" || serviceNames[service.Name] || !service.TargetAddress.Is4() || service.TargetAddress.IsUnspecified() || len(service.Ports) == 0 {
return relayConfig{}, errors.New("conntrack service is invalid")
}
serviceNames[service.Name] = true
for _, port := range service.Ports {
if port == 0 || servicePorts[port] {
return relayConfig{}, errors.New("conntrack service ports must be nonzero and unique")
}
servicePorts[port] = true
}
}
interval := configuration.SnapshotIntervalSeconds
if interval != 30 && interval != 60 {
return relayConfig{}, errors.New("SMTP evidence snapshot interval must be 30 or 60 seconds")
}
listenAddress, err := netip.ParseAddrPort(configuration.MetricsListenAddress)
if err != nil || !listenAddress.Addr().Is4() || listenAddress.Addr().IsUnspecified() || listenAddress.Port() == 0 || listenAddress.String() != configuration.MetricsListenAddress {
return relayConfig{}, errors.New("metrics listen address must contain a canonical IPv4 address and port")
}
return configuration, nil
}
func run(ctx context.Context, configuration relayConfig, nftPath, tcPath, telemetryPath, snapshotPath string, once bool) error {
var latest atomic.Pointer[metricsSnapshot]
if err := collect(ctx, configuration, nftPath, tcPath, telemetryPath, snapshotPath, &latest); err != nil {
return err
}
if once {
return nil
}
listener, err := net.Listen("tcp4", configuration.MetricsListenAddress)
if err != nil {
return err
}
var conntrackState conntrackMetrics
server := &http.Server{Handler: metricsHandler{Conntrack: &conntrackState, Latest: &latest, SMTPConfigured: conntrackServiceConfigured(configuration.ConntrackServices, "smtp")}, ReadHeaderTimeout: 5 * time.Second}
serverErrors := make(chan error, 1)
observerErrors := make(chan error, 1)
go func() {
serverErrors <- server.Serve(listener)
}()
go func() {
observerErrors <- startConntrackObserver(ctx, configuration, &conntrackState, os.Stdout)
}()
defer server.Close()
for {
timer := time.NewTimer(time.Duration(configuration.SnapshotIntervalSeconds) * time.Second)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case err := <-serverErrors:
timer.Stop()
return err
case err := <-observerErrors:
timer.Stop()
return err
case <-timer.C:
if err := collect(ctx, configuration, nftPath, tcPath, telemetryPath, snapshotPath, &latest); err != nil {
return err
}
}
}
}
func collect(ctx context.Context, configuration relayConfig, nftPath, tcPath, telemetryPath, snapshotPath string, latest *atomic.Pointer[metricsSnapshot]) error {
now := time.Now().UTC()
services := make([]serviceMetrics, 0, len(evidenceSets))
var smtpRecords []evidenceRecord
for _, set := range evidenceSets {
output, err := exec.CommandContext(ctx, nftPath, "-j", "list", "set", nftablesFamily, nftablesTable, set.Name).Output()
if err != nil {
return err
}
records, total, err := parseRecords(now, output, set.Name, configuration.MaximumRecords)
if err != nil {
return err
}
services = append(services, serviceMetrics{Records: records, Service: set.Service, Total: total})
if set.Service == "smtp" {
smtpRecords = records
}
}
counterOutput, err := exec.CommandContext(ctx, nftPath, "-j", "list", "counters").Output()
if err != nil {
return err
}
drops, err := parseDropMetrics(counterOutput, conntrackServiceConfigured(configuration.ConntrackServices, "smtp"))
if err != nil {
return err
}
if err := writeSnapshot(snapshotPath, snapshot{
SchemaVersion: 1,
GeneratedAt: now,
Reason: reason,
ObservationWindowSeconds: configuration.ObservationWindowSeconds,
Records: smtpRecords,
}); err != nil {
return err
}
qdiscOutput, err := exec.CommandContext(ctx, tcPath, "-s", "-j", "qdisc", "show", "dev", "wg-oci0").Output()
if err != nil {
return err
}
qdiscDrops, err := parseQdiscMetrics(qdiscOutput)
if err != nil {
return err
}
telemetryContents, err := os.ReadFile(telemetryPath)
if err != nil {
return err
}
telemetry, err := parseTelemetrySnapshot(telemetryContents)
if err != nil {
return err
}
latest.Store(&metricsSnapshot{Drops: drops, GeneratedAt: now, Health: telemetry.Status, QdiscDrops: qdiscDrops, Services: services, TelemetryAt: telemetry.GeneratedAt})
return nil
}
func parseRecords(now time.Time, contents []byte, expectedSet string, maximum uint64) ([]evidenceRecord, uint64, error) {
var document nftablesSetDocument
decoder := json.NewDecoder(bytes.NewReader(contents))
if err := decoder.Decode(&document); err != nil {
return nil, 0, err
}
records := make([]evidenceRecord, 0)
found := false
seen := make(map[netip.Addr]bool)
for _, object := range document.Nftables {
if object.Set == nil {
continue
}
if object.Set.Family != nftablesFamily || object.Set.Table != nftablesTable || object.Set.Name != expectedSet {
return nil, 0, fmt.Errorf("nftables returned the wrong quarantine set for %q", expectedSet)
}
found = true
for _, item := range object.Set.Elem {
if item.Element.Counter == nil {
return nil, 0, fmt.Errorf("evidence counter is absent for address %q", item.Element.Value)
}
address, err := netip.ParseAddr(item.Element.Value)
if err != nil || !address.Is4() || address.String() != item.Element.Value || seen[address] {
return nil, 0, fmt.Errorf("invalid or duplicate evidence address %q", item.Element.Value)
}
if item.Element.Expires <= 0 {
return nil, 0, fmt.Errorf("invalid expiry for evidence address %q", item.Element.Value)
}
seen[address] = true
records = append(records, evidenceRecord{
Address: address.String(),
Count: item.Element.Counter.Packets,
ExpiresAt: now.Add(time.Duration(item.Element.Expires) * time.Second),
})
}
}
if !found {
return nil, 0, fmt.Errorf("quarantine set %q is absent from nftables output", expectedSet)
}
sort.Slice(records, func(left, right int) bool {
leftAddress := netip.MustParseAddr(records[left].Address)
rightAddress := netip.MustParseAddr(records[right].Address)
return leftAddress.Less(rightAddress)
})
total := uint64(len(records))
if total > maximum {
records = records[:maximum]
}
return records, total, nil
}
func parseDropMetrics(contents []byte, mailConfigured bool) ([]dropMetric, error) {
var document nftablesCounterDocument
if err := json.Unmarshal(contents, &document); err != nil {
return nil, err
}
values := make(map[string]uint64)
for _, object := range document.Nftables {
if object.Counter != nil {
values[object.Counter.Name] = object.Counter.Packets
}
}
metrics := make([]dropMetric, 0, len(dropMetricSpecs))
for _, spec := range dropMetricSpecs {
packets, found := values[spec.Name]
if spec.MailOnly && !mailConfigured && !found {
packets = 0
found = true
}
if !found {
return nil, fmt.Errorf("nftables counter %q is absent", spec.Name)
}
metrics = append(metrics, dropMetric{Packets: packets, Reason: spec.Reason, Service: spec.Service})
}
return metrics, nil
}
func parseQdiscMetrics(contents []byte) ([]qdiscMetric, error) {
var document []qdiscDocumentItem
if err := json.Unmarshal(contents, &document); err != nil {
return nil, err
}
channels := map[string]string{
"1:10": "web",
"1:20": "smtp",
"1:30": "authenticated_mail",
"1:40": "default",
}
values := map[string]uint64{
"authenticated_mail": 0,
"default": 0,
"smtp": 0,
"total": 0,
"web": 0,
}
for _, qdisc := range document {
values["total"] += qdisc.Drops
if channel, ok := channels[qdisc.Parent]; ok {
values[channel] += qdisc.Drops
}
}
metrics := make([]qdiscMetric, 0, len(values))
for _, channel := range []string{"web", "smtp", "authenticated_mail", "default", "total"} {
metrics = append(metrics, qdiscMetric{Channel: channel, Packets: values[channel]})
}
return metrics, nil
}
func parseTelemetrySnapshot(contents []byte) (telemetrySnapshot, error) {
var value telemetrySnapshot
decoder := json.NewDecoder(bytes.NewReader(contents))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&value); err != nil {
return telemetrySnapshot{}, err
}
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
return telemetrySnapshot{}, errors.New("telemetry snapshot has trailing data")
}
if value.GeneratedAt.IsZero() || value.GeneratedAt.After(time.Now().Add(5*time.Minute)) {
return telemetrySnapshot{}, errors.New("telemetry snapshot timestamp is invalid")
}
return value, nil
}
type metricsHandler struct {
Conntrack *conntrackMetrics
Latest *atomic.Pointer[metricsSnapshot]
SMTPConfigured bool
}
func (handler metricsHandler) ServeHTTP(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/metrics" {
http.NotFound(response, request)
return
}
latest := handler.Latest.Load()
if latest == nil {
http.Error(response, "metrics are not ready", http.StatusServiceUnavailable)
return
}
response.Header().Set("Content-Type", "text/plain; version=0.0.4")
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_quarantine_packets gauge")
for _, service := range latest.Services {
for _, record := range service.Records {
fmt.Fprintf(response, "oci_l4_relay_tcp_quarantine_packets{service=%q,address=%q} %d\n", service.Service, record.Address, record.Count)
}
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_quarantine_expires_timestamp_seconds gauge")
for _, service := range latest.Services {
for _, record := range service.Records {
fmt.Fprintf(response, "oci_l4_relay_tcp_quarantine_expires_timestamp_seconds{service=%q,address=%q} %d\n", service.Service, record.Address, record.ExpiresAt.Unix())
}
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_evidence_records gauge")
for _, service := range latest.Services {
fmt.Fprintf(response, "oci_l4_relay_tcp_evidence_records{service=%q} %d\n", service.Service, len(service.Records))
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_evidence_truncated gauge")
for _, service := range latest.Services {
truncated := 0
if service.Total > uint64(len(service.Records)) {
truncated = 1
}
fmt.Fprintf(response, "oci_l4_relay_tcp_evidence_truncated{service=%q} %d\n", service.Service, truncated)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_tcp_evidence_generated_timestamp_seconds gauge")
fmt.Fprintf(response, "oci_l4_relay_tcp_evidence_generated_timestamp_seconds %d\n", latest.GeneratedAt.Unix())
fmt.Fprintln(response, "# TYPE oci_l4_relay_nftables_dropped_packets_total counter")
for _, metric := range latest.Drops {
fmt.Fprintf(response, "oci_l4_relay_nftables_dropped_packets_total{service=%q,reason=%q} %d\n", metric.Service, metric.Reason, metric.Packets)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_qdisc_dropped_packets_total counter")
for _, metric := range latest.QdiscDrops {
fmt.Fprintf(response, "oci_l4_relay_qdisc_dropped_packets_total{channel=%q} %d\n", metric.Channel, metric.Packets)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_component_healthy gauge")
for component, healthy := range map[string]bool{
"nftables": latest.Health.NftablesActive,
"smtp_deny": !handler.SMTPConfigured || latest.Health.SMTPDenyActive && latest.Health.SMTPDenyPolicyValid,
"web_allow": latest.Health.WebAllowActive && latest.Health.WebAllowCached,
"wireguard": latest.Health.WireGuardActive,
} {
fmt.Fprintf(response, "oci_l4_relay_component_healthy{component=%q} %d\n", component, booleanValue(healthy))
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_admission_cached gauge")
fmt.Fprintf(response, "oci_l4_relay_admission_cached{service=%q} %d\n", "web", booleanValue(latest.Health.WebAllowCached))
fmt.Fprintf(response, "oci_l4_relay_admission_cached{service=%q} %d\n", "smtp", booleanValue(latest.Health.SMTPDenyCached))
fmt.Fprintln(response, "# TYPE oci_l4_relay_smtp_deny_refresh_successful gauge")
fmt.Fprintf(response, "oci_l4_relay_smtp_deny_refresh_successful %d\n", booleanValue(latest.Health.SMTPDenyRefreshSuccessful))
fmt.Fprintln(response, "# TYPE oci_l4_relay_smtp_deny_configured gauge")
fmt.Fprintf(response, "oci_l4_relay_smtp_deny_configured %d\n", booleanValue(handler.SMTPConfigured))
for name, value := range map[string]int64{
"oci_l4_relay_conntrack_entries": latest.Health.ConntrackCount,
"oci_l4_relay_conntrack_maximum": latest.Health.ConntrackMaximum,
"oci_l4_relay_smtp_deny_bundle_age_seconds": latest.Health.SMTPDenyBundleAgeSeconds,
"oci_l4_relay_smtp_deny_bundle_remaining_seconds": latest.Health.SMTPDenyBundleRemainingSeconds,
"oci_l4_relay_smtp_deny_refresh_completed_age_seconds": latest.Health.SMTPDenyRefreshCompletedAgeSeconds,
"oci_l4_relay_smtp_deny_remote_current_age_seconds": latest.Health.SMTPDenyRemoteCurrentAgeSeconds,
"oci_l4_relay_wireguard_handshake_age_seconds": latest.Health.HandshakeAgeSeconds,
"oci_l4_relay_telemetry_generated_timestamp_seconds": latest.TelemetryAt.Unix(),
} {
fmt.Fprintf(response, "# TYPE %s gauge\n%s %d\n", name, name, value)
}
if handler.Conntrack != nil {
fmt.Fprintln(response, "# TYPE oci_l4_relay_conntrack_observer_events_total counter")
for outcome, value := range map[string]uint64{
"eligible": handler.Conntrack.Eligible.Load(),
"emitted": handler.Conntrack.Emitted.Load(),
"rate_dropped": handler.Conntrack.RateDropped.Load(),
"sampled_out": handler.Conntrack.SampledOut.Load(),
} {
fmt.Fprintf(response, "oci_l4_relay_conntrack_observer_events_total{outcome=%q} %d\n", outcome, value)
}
fmt.Fprintln(response, "# TYPE oci_l4_relay_conntrack_observer_up gauge")
fmt.Fprintf(response, "oci_l4_relay_conntrack_observer_up %d\n", booleanValue(handler.Conntrack.Up.Load()))
}
}
func booleanValue(value bool) int {
if value {
return 1
}
return 0
}
func conntrackServiceConfigured(services []conntrackService, name string) bool {
for _, service := range services {
if service.Name == name {
return true
}
}
return false
}
func writeSnapshot(path string, value snapshot) error {
contents, err := json.Marshal(value)
if err != nil {
return err
}
directory := filepath.Dir(path)
temporary, err := os.CreateTemp(directory, ".smtp-evidence-*.tmp")
if err != nil {
return err
}
temporaryPath := temporary.Name()
defer os.Remove(temporaryPath)
if err := temporary.Chmod(0600); err != nil {
temporary.Close()
return err
}
if _, err := temporary.Write(append(contents, '\n')); err != nil {
temporary.Close()
return err
}
if err := temporary.Sync(); err != nil {
temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(temporaryPath, path)
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}

270
services/oci-l4-relay-guest/cmd/smtp-evidence/main_test.go Normal file
View file

@ -0,0 +1,270 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
)
func TestParseRecordsSortsAndBoundsEvidence(t *testing.T) {
now := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC)
contents := []byte(`{"nftables":[{"metainfo":{"json_schema_version":1}},{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.2","expires":598,"counter":{"packets":4,"bytes":160}}},{"elem":{"val":"198.18.0.1","expires":599,"counter":{"packets":1,"bytes":40}}}]}}]}`)
records, total, err := parseRecords(now, contents, "smtp_syn_quarantine_v4", 1)
if err != nil {
t.Fatal(err)
}
if total != 2 || len(records) != 1 || records[0].Address != "198.18.0.1" || records[0].Count != 1 || !records[0].ExpiresAt.Equal(now.Add(599*time.Second)) {
t.Fatalf("unexpected records %#v", records)
}
}
func TestParseRecordsRequiresCanonicalIPv4(t *testing.T) {
contents := []byte(`{"nftables":[{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"2001:db8::1","expires":598,"counter":{"packets":1}}}]}}]}`)
if _, _, err := parseRecords(time.Now(), contents, "smtp_syn_quarantine_v4", 1); err == nil {
t.Fatal("IPv6 evidence was accepted")
}
}
func TestParseDropMetricsRequiresEveryBoundedCounter(t *testing.T) {
items := make([]string, 0, len(dropMetricSpecs))
for index, spec := range dropMetricSpecs {
items = append(items, fmt.Sprintf(`{"counter":{"name":%q,"packets":%d}}`, spec.Name, index+1))
}
metrics, err := parseDropMetrics([]byte(`{"nftables":[`+strings.Join(items, ",")+`]}`), true)
if err != nil {
t.Fatal(err)
}
if len(metrics) != len(dropMetricSpecs) || metrics[0].Service != "web" || metrics[0].Reason != "admission" || metrics[0].Packets != 1 {
t.Fatalf("unexpected metrics %#v", metrics)
}
if _, err := parseDropMetrics([]byte(`{"nftables":[]}`), true); err == nil {
t.Fatal("missing counters were accepted")
}
}
func TestMetricsHandlerExposesBoundedEvidenceAndDrops(t *testing.T) {
generated := time.Date(2026, 7, 30, 1, 0, 0, 0, time.UTC)
var latest atomic.Pointer[metricsSnapshot]
latest.Store(&metricsSnapshot{
GeneratedAt: generated,
Services: []serviceMetrics{
{Service: "web", Records: []evidenceRecord{{Address: "198.18.0.1", Count: 2, ExpiresAt: generated.Add(time.Minute)}}, Total: 2},
{Service: "smtp", Records: []evidenceRecord{{Address: "198.18.0.2", Count: 4, ExpiresAt: generated.Add(10 * time.Minute)}}, Total: 1},
},
Drops: []dropMetric{{Service: "web", Reason: "source_rate", Packets: 7}},
Health: relayHealth{ConntrackCount: 12, ConntrackMaximum: 262144, NftablesActive: true, WebAllowActive: true, WebAllowCached: true, WireGuardActive: true},
QdiscDrops: []qdiscMetric{{Channel: "web", Packets: 3}},
TelemetryAt: generated,
})
var conntrackState conntrackMetrics
conntrackState.Up.Store(true)
conntrackState.Emitted.Store(2)
request := httptest.NewRequest(http.MethodGet, "/metrics", nil)
response := httptest.NewRecorder()
metricsHandler{Conntrack: &conntrackState, Latest: &latest, SMTPConfigured: true}.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d", response.Code)
}
for _, expected := range []string{
`oci_l4_relay_tcp_quarantine_packets{service="web",address="198.18.0.1"} 2`,
`oci_l4_relay_tcp_quarantine_expires_timestamp_seconds{service="smtp",address="198.18.0.2"} 1785373800`,
`oci_l4_relay_tcp_evidence_records{service="smtp"} 1`,
`oci_l4_relay_tcp_evidence_truncated{service="web"} 1`,
`oci_l4_relay_tcp_evidence_generated_timestamp_seconds 1785373200`,
`oci_l4_relay_nftables_dropped_packets_total{service="web",reason="source_rate"} 7`,
`oci_l4_relay_qdisc_dropped_packets_total{channel="web"} 3`,
`oci_l4_relay_component_healthy{component="web_allow"} 1`,
`oci_l4_relay_conntrack_entries 12`,
`oci_l4_relay_conntrack_observer_events_total{outcome="emitted"} 2`,
`oci_l4_relay_conntrack_observer_up 1`,
`oci_l4_relay_smtp_deny_configured 1`,
} {
if !strings.Contains(response.Body.String(), expected) {
t.Errorf("metrics do not contain %q", expected)
}
}
}
func TestRunCollectsAndServesMetrics(t *testing.T) {
listener, err := net.Listen("tcp4", "127.0.0.1:0")
if err != nil {
if errors.Is(err, syscall.EPERM) {
t.Skip("network namespace does not permit listeners")
}
t.Fatal(err)
}
listenAddress := listener.Addr().String()
listener.Close()
items := make([]string, 0, len(dropMetricSpecs))
for index, spec := range dropMetricSpecs {
items = append(items, fmt.Sprintf(`{"counter":{"name":%q,"packets":%d}}`, spec.Name, index+1))
}
script := fmt.Sprintf(`#!/bin/sh
case "$*" in
*smtp_syn_quarantine_v4*) printf '%%s' '{"nftables":[{"set":{"family":"inet","name":"smtp_syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.2","expires":599,"counter":{"packets":4}}}]}}]}' ;;
*syn_quarantine_v4*) printf '%%s' '{"nftables":[{"set":{"family":"inet","name":"syn_quarantine_v4","table":"oci_l4_relay","elem":[{"elem":{"val":"198.18.0.1","expires":59,"counter":{"packets":2}}}]}}]}' ;;
*list\ counters*) printf '%%s' '{"nftables":[%s]}' ;;
*) exit 1 ;;
esac
`, strings.Join(items, ","))
nftPath := filepath.Join(t.TempDir(), "nft")
if err := os.WriteFile(nftPath, []byte(script), 0700); err != nil {
t.Fatal(err)
}
snapshotPath := filepath.Join(t.TempDir(), "current.json")
tcPath := filepath.Join(t.TempDir(), "tc")
if err := os.WriteFile(tcPath, []byte("#!/bin/sh\nprintf '%s' '[{\"parent\":\"1:10\",\"drops\":3}]'\n"), 0700); err != nil {
t.Fatal(err)
}
telemetryPath := filepath.Join(t.TempDir(), "telemetry.json")
if err := os.WriteFile(telemetryPath, []byte(`{"generated_at":"2026-07-30T01:00:00Z","status":{"conntrack_count":12,"conntrack_maximum":262144,"handshake_age_seconds":10,"nftables_active":true,"smtp_deny_active":true,"smtp_deny_active_source":"current","smtp_deny_bundle_age_seconds":60,"smtp_deny_bundle_remaining_seconds":600,"smtp_deny_cached":true,"smtp_deny_policy_valid":true,"smtp_deny_refresh_completed_age_seconds":30,"smtp_deny_refresh_successful":true,"smtp_deny_remote_current_age_seconds":30,"web_allow_active":true,"web_allow_cached":true,"wireguard_active":true}}`), 0600); err != nil {
t.Fatal(err)
}
previousObserver := startConntrackObserver
startConntrackObserver = func(ctx context.Context, _ relayConfig, _ *conntrackMetrics, _ io.Writer) error {
<-ctx.Done()
return ctx.Err()
}
defer func() { startConntrackObserver = previousObserver }()
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
done <- run(ctx, validRelayConfig(listenAddress), nftPath, tcPath, telemetryPath, snapshotPath, false)
}()
var body []byte
client := http.Client{Timeout: time.Second}
for attempt := 0; attempt < 50; attempt++ {
response, err := client.Get("http://" + listenAddress + "/metrics")
if err == nil {
body, err = io.ReadAll(response.Body)
response.Body.Close()
if err == nil && response.StatusCode == http.StatusOK {
break
}
}
time.Sleep(10 * time.Millisecond)
}
cancel()
if err := <-done; !errors.Is(err, context.Canceled) {
t.Fatal(err)
}
for _, expected := range []string{
`oci_l4_relay_tcp_quarantine_packets{service="web",address="198.18.0.1"} 2`,
`oci_l4_relay_tcp_quarantine_packets{service="smtp",address="198.18.0.2"} 4`,
`oci_l4_relay_nftables_dropped_packets_total{service="authenticated_mail",reason="aggregate_packet"} 14`,
} {
if !strings.Contains(string(body), expected) {
t.Errorf("metrics do not contain %q", expected)
}
}
contents, err := os.ReadFile(snapshotPath)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(contents), `"address":"198.18.0.2"`) || strings.Contains(string(contents), `"address":"198.18.0.1"`) {
t.Fatalf("unexpected SMTP snapshot %s", contents)
}
}
func TestWriteSnapshotReplacesCurrentFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "current.json")
if err := os.WriteFile(path, []byte("old"), 0600); err != nil {
t.Fatal(err)
}
value := snapshot{SchemaVersion: 1, Records: []evidenceRecord{}}
if err := writeSnapshot(path, value); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var actual snapshot
if err := json.Unmarshal(contents, &actual); err != nil {
t.Fatal(err)
}
if actual.SchemaVersion != 1 || actual.Records == nil {
t.Fatalf("unexpected snapshot %s", contents)
}
}
func TestLoadConfigAcceptsTestedCadences(t *testing.T) {
for _, interval := range []int{30, 60} {
path := filepath.Join(t.TempDir(), "config.json")
contents := relayConfigJSON(interval)
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err != nil {
t.Fatalf("interval %d: %v", interval, err)
}
}
}
func TestLoadConfigRejectsUntestedCadence(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
contents := relayConfigJSON(45)
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err == nil {
t.Fatal("untested cadence was accepted")
}
}
func TestLoadConfigRejectsUnknownField(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
contents := []byte(strings.TrimSuffix(string(relayConfigJSON(60)), "}") + `,"upload":true}`)
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err == nil {
t.Fatal("unknown field was accepted")
}
}
func TestLoadConfigRejectsNonSpecificMetricsAddress(t *testing.T) {
path := filepath.Join(t.TempDir(), "config.json")
contents := []byte(strings.Replace(string(relayConfigJSON(60)), "10.40.68.115:19100", "0.0.0.0:19100", 1))
if err := os.WriteFile(path, contents, 0600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(path); err == nil {
t.Fatal("wildcard metrics address was accepted")
}
}
func validRelayConfig(listenAddress string) relayConfig {
return relayConfig{
ConntrackEmissionBurst: 10,
ConntrackEmissionRatePerSecond: 1,
ConntrackEventBuffer: 4096,
ConntrackSampleDenominator: 10,
ConntrackServices: []conntrackService{{
Name: "web",
Ports: []uint16{80, 443},
TargetAddress: netip.MustParseAddr("10.104.40.127"),
}},
MaximumRecords: 4096,
MetricsListenAddress: listenAddress,
ObservationWindowSeconds: 600,
SnapshotIntervalSeconds: 60,
}
}
func relayConfigJSON(interval int) []byte {
return []byte(fmt.Sprintf(`{"conntrack_emission_burst":10,"conntrack_emission_rate_per_second":1,"conntrack_event_buffer":4096,"conntrack_sample_denominator":10,"conntrack_services":[{"name":"web","ports":[80,443],"target_address":"10.104.40.127"}],"maximum_records":4096,"metrics_listen_address":"10.40.68.115:19100","observation_window_seconds":600,"snapshot_interval_seconds":%d}`, interval))
}