ms2k/quorate
SHA256
Watch
0
Code Releases Activity

ce7cb702218693d08ea6d83464d2457c9250027c

Browse source
This commit is contained in:
ms2k 2026-11-12 20:46:28 +09:00
commit 34c5f8963c
10 changed files with 145336 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

107
packages/oci-l4-relay-publisher/cmd/fetch-ip2proxy/main.go Normal file
View file

@ -0,0 +1,107 @@
package main
import (
"archive/zip"
"context"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
)
func downloadToken(path string) (string, error) {
contents, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read IP2Location credential")
}
token := strings.TrimSpace(string(contents))
if token == "" {
return "", fmt.Errorf("IP2Location credential is empty")
}
return token, nil
}
func fetch(ctx context.Context, client *http.Client, token, code, output string) error {
query := url.Values{"token": {token}, "file": {code}}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.ip2location.com/download?"+query.Encode(), nil)
if err != nil {
return fmt.Errorf("create IP2Proxy request")
}
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
response, err := client.Do(request)
if err != nil {
return fmt.Errorf("download IP2Proxy package")
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("download IP2Proxy package: HTTP %d", response.StatusCode)
}
if err := os.MkdirAll(filepath.Dir(output), 0o700); err != nil {
return err
}
temporary, err := os.CreateTemp(filepath.Dir(output), ".ip2proxy-*")
if err != nil {
return err
}
temporaryPath := temporary.Name()
defer func() { _ = os.Remove(temporaryPath) }()
if err := temporary.Chmod(0o600); err != nil {
_ = temporary.Close()
return err
}
written, copyErr := io.Copy(temporary, io.LimitReader(response.Body, (512<<20)+1))
closeErr := temporary.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
if written > 512<<20 {
return fmt.Errorf("IP2Proxy package exceeds size limit")
}
archive, err := zip.OpenReader(temporaryPath)
if err != nil {
return fmt.Errorf("validate IP2Proxy archive")
}
defer func() { _ = archive.Close() }()
found := false
for _, file := range archive.File {
if strings.HasSuffix(file.Name, "IP2PROXY-LITE-PX12.CSV") {
found = true
break
}
}
if !found {
return fmt.Errorf("IP2Proxy PX12 archive member is missing")
}
if err := os.Rename(temporaryPath, output); err != nil {
return err
}
fmt.Printf("source=ip2proxy bytes=%d\n", written)
return nil
}
func main() {
downloadTokenFile := flag.String("download-token-file", "/credentials/ip2location/download-token", "IP2Location download-token file")
packageCode := flag.String("package-code", "PX12LITE", "entitled IP2Proxy package code")
dataDirectory := flag.String("data-directory", "/work/data", "validated source cache")
flag.Parse()
token, err := downloadToken(*downloadTokenFile)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
client := &http.Client{Timeout: 15 * time.Minute}
if err := fetch(ctx, client, token, *packageCode, filepath.Join(*dataDirectory, "ip2proxy.zip")); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}

401
packages/oci-l4-relay-publisher/cmd/fetch/main.go Normal file
View file

@ -0,0 +1,401 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
)
const maximumSourceBytes = 512 << 20
type source struct {
name string
url string
minimumInterval time.Duration
headBeforeGet bool
redactURL bool
}
type sourceState struct {
CheckedAt time.Time `json:"checked_at"`
UpdatedAt time.Time `json:"updated_at"`
NotBefore time.Time `json:"not_before"`
ETag string `json:"etag,omitempty"`
LastModified string `json:"last_modified,omitempty"`
SHA256 string `json:"sha256"`
Bytes int64 `json:"bytes"`
}
type sourceResult struct {
Name string `json:"name"`
Changed bool `json:"changed"`
State sourceState `json:"state"`
}
func credential(path, name string) (string, error) {
contents, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read %s credential", name)
}
value := strings.TrimSpace(string(contents))
if value == "" {
return "", fmt.Errorf("%s credential is empty", name)
}
return value, nil
}
func azureURL(ctx context.Context, client *http.Client) (string, error) {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519", nil)
if err != nil {
return "", err
}
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
response, err := client.Do(request)
if err != nil {
return "", fmt.Errorf("fetch Azure release page")
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return "", fmt.Errorf("fetch Azure release page: HTTP %d", response.StatusCode)
}
contents, err := io.ReadAll(io.LimitReader(response.Body, 8<<20))
if err != nil {
return "", fmt.Errorf("read Azure release page")
}
pattern := regexp.MustCompile(`https://download\.microsoft\.com/download/[^"']+ServiceTags_Public_[0-9]+\.json`)
match := pattern.FindString(string(contents))
if match == "" {
return "", errors.New("discover Azure service tag release")
}
return strings.ReplaceAll(match, "&amp;", "&"), nil
}
func sourceCatalog(ctx context.Context, client *http.Client, selected map[string]bool, maxMindKeyPath string) (map[string]source, error) {
catalog := map[string]source{
"iplocate-country": {name: "iplocate-country.zip", url: "https://github.com/iplocate/ip-address-databases/raw/refs/heads/main/ip-to-country/ip-to-country.csv.zip", minimumInterval: 20 * time.Hour},
"iplocate-asn": {name: "iplocate-asn.zip", url: "https://github.com/iplocate/ip-address-databases/raw/refs/heads/main/ip-to-asn/ip-to-asn.csv.zip", minimumInterval: 20 * time.Hour},
"spamhaus-drop": {name: "spamhaus-drop.json", url: "https://www.spamhaus.org/drop/drop_v4.json", minimumInterval: 24 * time.Hour},
"tor-exits": {name: "tor-exits.txt", url: "https://check.torproject.org/torbulkexitlist", minimumInterval: 6 * 24 * time.Hour},
"aws": {name: "aws.json", url: "https://ip-ranges.amazonaws.com/ip-ranges.json", minimumInterval: 6 * 24 * time.Hour},
"gcp": {name: "gcp.json", url: "https://www.gstatic.com/ipranges/cloud.json", minimumInterval: 6 * 24 * time.Hour},
"oci": {name: "oci.json", url: "https://docs.oracle.com/iaas/tools/public_ip_ranges.json", minimumInterval: 24 * time.Hour},
"cloudflare": {name: "cloudflare.txt", url: "https://www.cloudflare.com/ips-v4", minimumInterval: 6 * 24 * time.Hour},
"fastly": {name: "fastly.json", url: "https://api.fastly.com/public-ip-list", minimumInterval: 6 * 24 * time.Hour},
"github": {name: "github.json", url: "https://api.github.com/meta", minimumInterval: 6 * 24 * time.Hour},
"atlassian": {name: "atlassian.json", url: "https://ip-ranges.atlassian.com/", minimumInterval: 6 * 24 * time.Hour},
"digitalocean": {name: "digitalocean.csv", url: "https://digitalocean.com/geo/google.csv", minimumInterval: 6 * 24 * time.Hour},
"linode": {name: "linode.csv", url: "https://geoip.linode.com/", minimumInterval: 6 * 24 * time.Hour},
"vultr": {name: "vultr.csv", url: "https://geofeed.constant.com/?text", minimumInterval: 6 * 24 * time.Hour},
}
if selected["maxmind-country"] {
key, err := credential(maxMindKeyPath, "MaxMind")
if err != nil {
return nil, err
}
query := url.Values{"edition_id": {"GeoLite2-Country-CSV"}, "license_key": {key}, "suffix": {"zip"}}
catalog["maxmind-country"] = source{name: "maxmind-country.zip", url: "https://download.maxmind.com/app/geoip_download?" + query.Encode(), minimumInterval: 12 * time.Hour, headBeforeGet: true, redactURL: true}
}
if selected["azure"] {
downloadURL, err := azureURL(ctx, client)
if err != nil {
return nil, err
}
catalog["azure"] = source{name: "azure.json", url: downloadURL, minimumInterval: 6 * 24 * time.Hour}
}
return catalog, nil
}
func readState(path string) (sourceState, error) {
contents, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return sourceState{}, nil
}
if err != nil {
return sourceState{}, err
}
var state sourceState
if err := json.Unmarshal(contents, &state); err != nil {
return sourceState{}, err
}
return state, nil
}
func writeState(path string, state sourceState) error {
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
contents, err := json.Marshal(state)
if err != nil {
return err
}
temporary, err := os.CreateTemp(filepath.Dir(path), ".state-*")
if err != nil {
return err
}
temporaryPath := temporary.Name()
defer func() { _ = os.Remove(temporaryPath) }()
if err := temporary.Chmod(0o600); err != nil {
_ = temporary.Close()
return err
}
if _, err := temporary.Write(append(contents, '\n')); err != nil {
_ = temporary.Close()
return err
}
if err := temporary.Close(); err != nil {
return err
}
return os.Rename(temporaryPath, path)
}
func writeJSON(path string, value any) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
contents, err := json.Marshal(value)
if err != nil {
return err
}
return os.WriteFile(path, append(contents, '\n'), 0o644)
}
func retryAfter(response *http.Response, now time.Time) time.Time {
value := response.Header.Get("Retry-After")
if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 {
return now.Add(time.Duration(seconds) * time.Second)
}
if parsed, err := http.ParseTime(value); err == nil && parsed.After(now) {
return parsed
}
return now.Add(time.Hour)
}
func request(ctx context.Context, client *http.Client, method string, item source, state sourceState) (*http.Response, error) {
request, err := http.NewRequestWithContext(ctx, method, item.url, nil)
if err != nil {
return nil, fmt.Errorf("create %s request", item.name)
}
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
if state.ETag != "" {
request.Header.Set("If-None-Match", state.ETag)
}
if state.LastModified != "" {
request.Header.Set("If-Modified-Since", state.LastModified)
}
response, err := client.Do(request)
if err != nil {
if !item.redactURL {
return nil, fmt.Errorf("fetch %s: %w", item.name, err)
}
return nil, fmt.Errorf("fetch %s", item.name)
}
return response, nil
}
func fetchSource(ctx context.Context, client *http.Client, key string, item source, dataDirectory, stateDirectory string, now time.Time) (bool, error) {
statePath := filepath.Join(stateDirectory, key+".json")
state, err := readState(statePath)
if err != nil {
return false, fmt.Errorf("read %s state", key)
}
dataPath := filepath.Join(dataDirectory, item.name)
if now.Before(state.NotBefore) {
if _, err := os.Stat(dataPath); err != nil || state.SHA256 == "" {
return false, fmt.Errorf("%s is not due and has no cached generation", key)
}
fmt.Printf("source=%s due=false\n", key)
return false, nil
}
method := http.MethodGet
if item.headBeforeGet && state.SHA256 != "" {
method = http.MethodHead
}
response, err := request(ctx, client, method, item, state)
if err != nil {
return false, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode == http.StatusTooManyRequests || response.StatusCode >= 500 {
state.NotBefore = retryAfter(response, now)
if err := writeState(statePath, state); err != nil {
return false, err
}
return false, fmt.Errorf("fetch %s: HTTP %d", key, response.StatusCode)
}
if response.StatusCode == http.StatusNotModified {
state.CheckedAt = now
state.NotBefore = now.Add(item.minimumInterval)
if err := writeState(statePath, state); err != nil {
return false, err
}
fmt.Printf("source=%s changed=false\n", key)
return false, nil
}
if response.StatusCode != http.StatusOK {
return false, fmt.Errorf("fetch %s: HTTP %d", key, response.StatusCode)
}
if method == http.MethodHead {
unchanged := response.Header.Get("ETag") == state.ETag && response.Header.Get("Last-Modified") == state.LastModified
if unchanged {
state.CheckedAt = now
state.NotBefore = now.Add(item.minimumInterval)
if err := writeState(statePath, state); err != nil {
return false, err
}
fmt.Printf("source=%s changed=false\n", key)
return false, nil
}
_ = response.Body.Close()
response, err = request(ctx, client, http.MethodGet, item, sourceState{})
if err != nil {
return false, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return false, fmt.Errorf("download %s: HTTP %d", key, response.StatusCode)
}
}
if err := os.MkdirAll(dataDirectory, 0o700); err != nil {
return false, err
}
temporary, err := os.CreateTemp(dataDirectory, ".download-*")
if err != nil {
return false, err
}
temporaryPath := temporary.Name()
defer func() { _ = os.Remove(temporaryPath) }()
if err := temporary.Chmod(0o600); err != nil {
_ = temporary.Close()
return false, err
}
hash := sha256.New()
written, copyErr := io.Copy(io.MultiWriter(temporary, hash), io.LimitReader(response.Body, maximumSourceBytes+1))
closeErr := temporary.Close()
if copyErr != nil {
return false, copyErr
}
if closeErr != nil {
return false, closeErr
}
if written > maximumSourceBytes {
return false, fmt.Errorf("%s exceeds size limit", key)
}
digest := hex.EncodeToString(hash.Sum(nil))
changed := digest != state.SHA256
if changed {
if err := os.Rename(temporaryPath, dataPath); err != nil {
return false, err
}
}
state = sourceState{
CheckedAt: now,
UpdatedAt: state.UpdatedAt,
NotBefore: now.Add(item.minimumInterval),
ETag: response.Header.Get("ETag"),
LastModified: response.Header.Get("Last-Modified"),
SHA256: digest,
Bytes: written,
}
if changed {
state.UpdatedAt = now
}
if err := writeState(statePath, state); err != nil {
return false, err
}
fmt.Printf("source=%s changed=%t bytes=%d sha256=%s\n", key, changed, written, digest)
return changed, nil
}
func selectedSources(value string) (map[string]bool, []string, error) {
selected := make(map[string]bool)
var order []string
for _, item := range strings.Split(value, ",") {
item = strings.TrimSpace(item)
if item == "" || selected[item] {
continue
}
selected[item] = true
order = append(order, item)
}
if len(order) == 0 {
return nil, nil, errors.New("at least one source is required")
}
return selected, order, nil
}
func main() {
sourceNames := flag.String("sources", "", "comma-separated source names")
dataDirectory := flag.String("data-directory", "/work/data", "validated source cache")
stateDirectory := flag.String("state-directory", "/work/state", "source fetch state")
changedFile := flag.String("changed-file", "/work/fetch-changed", "aggregate changed result")
summaryFile := flag.String("summary-file", "/work/fetch-summary.json", "sanitized fetch summary")
maxMindKeyPath := flag.String("maxmind-license-key-file", "/credentials/maxmind/license-key", "MaxMind license-key file")
flag.Parse()
selected, order, err := selectedSources(*sourceNames)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
client := &http.Client{Timeout: 15 * time.Minute}
catalog, err := sourceCatalog(ctx, client, selected, *maxMindKeyPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
now := time.Now().UTC()
changed := false
results := make([]sourceResult, 0, len(order))
if contents, err := os.ReadFile(*changedFile); err == nil {
changed, _ = strconv.ParseBool(strings.TrimSpace(string(contents)))
}
for _, name := range order {
item, ok := catalog[name]
if !ok {
fmt.Fprintf(os.Stderr, "unknown source %q\n", name)
os.Exit(2)
}
currentChanged, err := fetchSource(ctx, client, name, item, *dataDirectory, *stateDirectory, now)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
changed = changed || currentChanged
state, err := readState(filepath.Join(*stateDirectory, name+".json"))
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
results = append(results, sourceResult{Name: name, Changed: currentChanged, State: state})
if err := os.MkdirAll(filepath.Dir(*changedFile), 0o755); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err := os.WriteFile(*changedFile, []byte(strconv.FormatBool(changed)+"\n"), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
if err := os.WriteFile(*changedFile, []byte(strconv.FormatBool(changed)+"\n"), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if err := writeJSON(*summaryFile, results); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}

89
packages/oci-l4-relay-publisher/cmd/fetch/main_test.go Normal file
View file

@ -0,0 +1,89 @@
package main
import (
"context"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
type roundTripFunc func(*http.Request) (*http.Response, error)
func (function roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
return function(request)
}
func TestFetchSourceUsesValidatorsAndNotBefore(t *testing.T) {
requests := 0
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
requests++
if request.Header.Get("If-None-Match") == `"generation-1"` {
return &http.Response{StatusCode: http.StatusNotModified, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(""))}, nil
}
header := make(http.Header)
header.Set("ETag", `"generation-1"`)
return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(strings.NewReader("source generation"))}, nil
})}
directory := t.TempDir()
dataDirectory := filepath.Join(directory, "data")
stateDirectory := filepath.Join(directory, "state")
item := source{name: "source.txt", url: "https://source.example.invalid/feed", minimumInterval: time.Hour}
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
changed, err := fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now)
if err != nil || !changed {
t.Fatalf("initial fetch changed=%t err=%v", changed, err)
}
changed, err = fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now.Add(30*time.Minute))
if err != nil || changed {
t.Fatalf("not-before fetch changed=%t err=%v", changed, err)
}
changed, err = fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now.Add(2*time.Hour))
if err != nil || changed {
t.Fatalf("conditional fetch changed=%t err=%v", changed, err)
}
if requests != 2 {
t.Fatalf("requests=%d, want 2", requests)
}
contents, err := os.ReadFile(filepath.Join(dataDirectory, "source.txt"))
if err != nil || string(contents) != "source generation" {
t.Fatalf("cached source=%q err=%v", contents, err)
}
}
func TestFetchSourcePersistsRetryFloor(t *testing.T) {
requests := 0
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
requests++
header := make(http.Header)
header.Set("Retry-After", "3600")
return &http.Response{StatusCode: http.StatusTooManyRequests, Header: header, Body: io.NopCloser(strings.NewReader(""))}, nil
})}
directory := t.TempDir()
dataDirectory := filepath.Join(directory, "data")
stateDirectory := filepath.Join(directory, "state")
if err := os.MkdirAll(dataDirectory, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dataDirectory, "source.txt"), []byte("cached"), 0o600); err != nil {
t.Fatal(err)
}
if err := writeState(filepath.Join(stateDirectory, "source.json"), sourceState{SHA256: "cached-generation"}); err != nil {
t.Fatal(err)
}
item := source{name: "source.txt", url: "https://source.example.invalid/feed", minimumInterval: time.Hour}
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
if _, err := fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now); err == nil {
t.Fatal("expected rate-limit error")
}
changed, err := fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now.Add(time.Minute))
if err != nil || changed {
t.Fatalf("retry-floor fetch changed=%t err=%v", changed, err)
}
if requests != 1 {
t.Fatalf("requests=%d, want 1", requests)
}
}

445
packages/oci-l4-relay-publisher/cmd/publish/main.go Normal file
View file

@ -0,0 +1,445 @@
package main
import (
"bytes"
"context"
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
)
const maximumBundleBytes = 64 << 20
var objectPrefixPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*/v[1-9][0-9]*$`)
type credentials struct {
tenancy string
user string
fingerprint string
privateKey *rsa.PrivateKey
}
type stringValues []string
func (values *stringValues) String() string {
return strings.Join(*values, ",")
}
func (values *stringValues) Set(value string) error {
*values = append(*values, value)
return nil
}
type publication struct {
Prefix string `json:"prefix"`
VersionObject string `json:"version_object"`
VersionCreated bool `json:"version_created"`
CurrentObject string `json:"current_object"`
CurrentPromoted bool `json:"current_promoted"`
}
type smtpDenySummary struct {
GeneratedAt time.Time `json:"generated_at"`
ExpiresAt time.Time `json:"expires_at"`
DeniedIntervalCount int `json:"denied_interval_count"`
DeniedAddressCount uint64 `json:"denied_address_count"`
RemovedIntervalCount int `json:"removed_interval_count"`
}
func readSMTPDenySummary(path string) (*smtpDenySummary, error) {
if path == "" {
return nil, nil
}
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
decoder := json.NewDecoder(io.LimitReader(file, 1<<20))
decoder.DisallowUnknownFields()
summary := smtpDenySummary{}
if err := decoder.Decode(&summary); err != nil {
return nil, err
}
if err := decoder.Decode(&struct{}{}); err != io.EOF {
return nil, errors.New("SMTP deny summary contains trailing data")
}
if summary.GeneratedAt.IsZero() ||
!summary.ExpiresAt.After(summary.GeneratedAt) ||
summary.DeniedIntervalCount < 0 ||
summary.RemovedIntervalCount < 0 {
return nil, errors.New("SMTP deny summary is invalid")
}
return &summary, nil
}
func publicationPrefixes(primary string, additional []string) ([]string, error) {
values := append([]string{primary}, additional...)
seen := make(map[string]struct{}, len(values))
result := make([]string, 0, len(values))
for _, value := range values {
if !objectPrefixPattern.MatchString(value) {
return nil, fmt.Errorf("invalid object prefix %q", value)
}
if _, exists := seen[value]; exists {
return nil, fmt.Errorf("duplicate object prefix %q", value)
}
seen[value] = struct{}{}
result = append(result, value)
}
return result, nil
}
func publicationObjects(prefix, digest string) (string, string) {
return prefix + "/versions/" + digest + ".json.gz", prefix + "/current.json.gz"
}
func publishBundle(ctx context.Context, client *http.Client, endpoint, namespace, bucket string, prefixes []string, body []byte, identity credentials, promoteCurrent bool) (string, []publication, error) {
digest := sha256.Sum256(body)
digestText := hex.EncodeToString(digest[:])
publications := make([]publication, 0, len(prefixes))
for _, prefix := range prefixes {
versionObject, currentObject := publicationObjects(prefix, digestText)
created, err := upload(ctx, client, endpoint, namespace, bucket, versionObject, body, identity, true)
if err != nil {
return "", nil, err
}
if err := verifyObject(ctx, client, endpoint, namespace, bucket, versionObject, body, identity); err != nil {
return "", nil, err
}
if promoteCurrent {
if _, err := upload(ctx, client, endpoint, namespace, bucket, currentObject, body, identity, false); err != nil {
return "", nil, err
}
if err := verifyObject(ctx, client, endpoint, namespace, bucket, currentObject, body, identity); err != nil {
return "", nil, err
}
}
publications = append(publications, publication{
Prefix: prefix,
VersionObject: versionObject,
VersionCreated: created,
CurrentObject: currentObject,
CurrentPromoted: promoteCurrent,
})
}
return digestText, publications, nil
}
func writeJSONFile(path string, value any) error {
contents, err := json.Marshal(value)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
return os.WriteFile(path, append(contents, '\n'), 0o644)
}
func readCredential(directory, name string) (string, error) {
contents, err := os.ReadFile(filepath.Join(directory, name))
if err != nil {
return "", fmt.Errorf("read OCI %s", strings.ReplaceAll(name, "_", " "))
}
value := strings.TrimSpace(string(contents))
if value == "" {
return "", fmt.Errorf("OCI %s is empty", strings.ReplaceAll(name, "_", " "))
}
return value, nil
}
func optionalCredential(directory, name, fallback string) string {
value, err := readCredential(directory, name)
if err != nil {
return fallback
}
return value
}
func parsePrivateKey(value string) (*rsa.PrivateKey, error) {
block, _ := pem.Decode([]byte(value))
if block == nil {
return nil, errors.New("decode OCI private key")
}
if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil {
return key, nil
}
parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err != nil {
return nil, errors.New("parse OCI private key")
}
key, ok := parsed.(*rsa.PrivateKey)
if !ok {
return nil, errors.New("OCI private key is not RSA")
}
return key, nil
}
func readCredentials(directory string) (credentials, error) {
tenancy, err := readCredential(directory, "tenancy_ocid")
if err != nil {
return credentials{}, err
}
user, err := readCredential(directory, "user_ocid")
if err != nil {
return credentials{}, err
}
fingerprint, err := readCredential(directory, "fingerprint")
if err != nil {
return credentials{}, err
}
privateKeyPEM, err := readCredential(directory, "private_key")
if err != nil {
return credentials{}, err
}
privateKey, err := parsePrivateKey(privateKeyPEM)
if err != nil {
return credentials{}, err
}
return credentials{tenancy: tenancy, user: user, fingerprint: fingerprint, privateKey: privateKey}, nil
}
func objectURL(endpoint, namespace, bucket, object string) (string, error) {
parsed, err := url.Parse(endpoint)
if err != nil {
return "", err
}
if parsed.Scheme != "https" || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" {
return "", errors.New("invalid OCI Object Storage endpoint")
}
parsed.Path = "/n/" + namespace + "/b/" + bucket + "/o/" + object
parsed.RawPath = "/n/" + url.PathEscape(namespace) + "/b/" + url.PathEscape(bucket) + "/o/" + url.PathEscape(object)
return parsed.String(), nil
}
func signRequest(request *http.Request, body []byte, identity credentials, now time.Time) error {
digest := sha256.Sum256(body)
request.Header.Set("Date", now.UTC().Format(http.TimeFormat))
request.Header.Set("Content-Type", "application/gzip")
request.Header.Set("Content-Length", strconv.Itoa(len(body)))
request.Header.Set("X-Content-SHA256", base64.StdEncoding.EncodeToString(digest[:]))
request.ContentLength = int64(len(body))
headerNames := "(request-target) host date x-content-sha256 content-type content-length"
signingString := strings.Join([]string{
"(request-target): " + strings.ToLower(request.Method) + " " + request.URL.EscapedPath(),
"host: " + request.URL.Host,
"date: " + request.Header.Get("Date"),
"x-content-sha256: " + request.Header.Get("X-Content-SHA256"),
"content-type: " + request.Header.Get("Content-Type"),
"content-length: " + request.Header.Get("Content-Length"),
}, "\n")
signingDigest := sha256.Sum256([]byte(signingString))
signature, err := rsa.SignPKCS1v15(rand.Reader, identity.privateKey, crypto.SHA256, signingDigest[:])
if err != nil {
return errors.New("sign OCI request")
}
keyID := identity.tenancy + "/" + identity.user + "/" + identity.fingerprint
request.Header.Set("Authorization", fmt.Sprintf(`Signature version="1",keyId="%s",algorithm="rsa-sha256",headers="%s",signature="%s"`, keyID, headerNames, base64.StdEncoding.EncodeToString(signature)))
return nil
}
func signDownloadRequest(request *http.Request, identity credentials, now time.Time) error {
request.Header.Set("Date", now.UTC().Format(http.TimeFormat))
headerNames := "(request-target) host date"
signingString := strings.Join([]string{
"(request-target): " + strings.ToLower(request.Method) + " " + request.URL.EscapedPath(),
"host: " + request.URL.Host,
"date: " + request.Header.Get("Date"),
}, "\n")
signingDigest := sha256.Sum256([]byte(signingString))
signature, err := rsa.SignPKCS1v15(rand.Reader, identity.privateKey, crypto.SHA256, signingDigest[:])
if err != nil {
return errors.New("sign OCI request")
}
keyID := identity.tenancy + "/" + identity.user + "/" + identity.fingerprint
request.Header.Set("Authorization", fmt.Sprintf(`Signature version="1",keyId="%s",algorithm="rsa-sha256",headers="%s",signature="%s"`, keyID, headerNames, base64.StdEncoding.EncodeToString(signature)))
return nil
}
func upload(ctx context.Context, client *http.Client, endpoint, namespace, bucket, object string, body []byte, identity credentials, immutable bool) (bool, error) {
target, err := objectURL(endpoint, namespace, bucket, object)
if err != nil {
return false, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodPut, target, bytes.NewReader(body))
if err != nil {
return false, errors.New("create OCI upload request")
}
if immutable {
request.Header.Set("If-None-Match", "*")
}
if err := signRequest(request, body, identity, time.Now()); err != nil {
return false, err
}
response, err := client.Do(request)
if err != nil {
return false, errors.New("upload OCI object")
}
defer func() { _ = response.Body.Close() }()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
if immutable && response.StatusCode == http.StatusPreconditionFailed {
return false, nil
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
return false, fmt.Errorf("upload OCI object: HTTP %d", response.StatusCode)
}
return true, nil
}
func download(ctx context.Context, client *http.Client, endpoint, namespace, bucket, object string, identity credentials) ([]byte, error) {
target, err := objectURL(endpoint, namespace, bucket, object)
if err != nil {
return nil, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
if err != nil {
return nil, errors.New("create OCI download request")
}
if err := signDownloadRequest(request, identity, time.Now()); err != nil {
return nil, err
}
response, err := client.Do(request)
if err != nil {
return nil, errors.New("download OCI object")
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
return nil, fmt.Errorf("download OCI object: HTTP %d", response.StatusCode)
}
contents, err := io.ReadAll(io.LimitReader(response.Body, maximumBundleBytes+1))
if err != nil {
return nil, errors.New("read OCI object")
}
if len(contents) > maximumBundleBytes {
return nil, errors.New("OCI object exceeds maximum bundle size")
}
return contents, nil
}
func verifyObject(ctx context.Context, client *http.Client, endpoint, namespace, bucket, object string, expected []byte, identity credentials) error {
contents, err := download(ctx, client, endpoint, namespace, bucket, object, identity)
if err != nil {
return err
}
expectedDigest := sha256.Sum256(expected)
actualDigest := sha256.Sum256(contents)
if actualDigest != expectedDigest {
return fmt.Errorf("verify OCI object %q: SHA-256 mismatch", object)
}
return nil
}
func main() {
bundlePath := flag.String("bundle", "/work/output/current.json.gz", "validated admission bundle")
credentialsDirectory := flag.String("credentials-directory", "/credentials/oci", "OCI credential directory")
region := flag.String("region", "", "OCI region")
namespace := flag.String("namespace", "", "OCI Object Storage namespace")
bucket := flag.String("bucket", "", "OCI Object Storage bucket")
objectPrefix := flag.String("object-prefix", "admission/v1", "Object Storage prefix for versions and current")
additionalObjectPrefixes := stringValues{}
flag.Var(&additionalObjectPrefixes, "additional-object-prefix", "additional Object Storage prefix for the same bundle")
promoteCurrent := flag.Bool("promote-current", true, "promote the validated bundle after immutable upload")
endpoint := flag.String("endpoint", "", "OCI Object Storage endpoint")
resultFile := flag.String("result-file", "", "write sanitized publication result")
smtpDenySummaryPath := flag.String("smtp-deny-summary", "", "validated SMTP deny summary")
restoreVersionSHA256 := flag.String("restore-version-sha256", "", "download an immutable version by SHA-256")
restoreOutput := flag.String("restore-output", "/work/output/restore.json.gz", "write the immutable version for validation")
flag.Parse()
*region = optionalCredential(*credentialsDirectory, "region", *region)
*namespace = optionalCredential(*credentialsDirectory, "object_storage_namespace", *namespace)
*bucket = optionalCredential(*credentialsDirectory, "bucket_name", *bucket)
if *region == "" || *namespace == "" || *bucket == "" {
fmt.Fprintln(os.Stderr, "region, namespace, and bucket are required")
os.Exit(2)
}
if *endpoint == "" {
*endpoint = "https://objectstorage." + *region + ".oraclecloud.com"
}
prefixes, err := publicationPrefixes(*objectPrefix, additionalObjectPrefixes)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
smtpSummary, err := readSMTPDenySummary(*smtpDenySummaryPath)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
identity, err := readCredentials(*credentialsDirectory)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
client := &http.Client{Timeout: 2 * time.Minute}
if *restoreVersionSHA256 != "" {
if !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(*restoreVersionSHA256) {
fmt.Fprintln(os.Stderr, "restore version SHA-256 must be 64 lowercase hexadecimal characters")
os.Exit(2)
}
versionObject, _ := publicationObjects(prefixes[0], *restoreVersionSHA256)
bundle, err := download(ctx, client, *endpoint, *namespace, *bucket, versionObject, identity)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
digest := sha256.Sum256(bundle)
if hex.EncodeToString(digest[:]) != *restoreVersionSHA256 {
fmt.Fprintln(os.Stderr, "downloaded immutable version does not match requested SHA-256")
os.Exit(1)
}
if err := os.MkdirAll(filepath.Dir(*restoreOutput), 0o700); err != nil {
fmt.Fprintln(os.Stderr, "create restore output directory")
os.Exit(1)
}
if err := os.WriteFile(*restoreOutput, bundle, 0o600); err != nil {
fmt.Fprintln(os.Stderr, "write restored immutable version")
os.Exit(1)
}
fmt.Printf("restored_bundle_sha256=%s\n", *restoreVersionSHA256)
return
}
bundle, err := os.ReadFile(*bundlePath)
if err != nil {
fmt.Fprintln(os.Stderr, "read validated admission bundle")
os.Exit(1)
}
digestText, publications, err := publishBundle(ctx, client, *endpoint, *namespace, *bucket, prefixes, bundle, identity, *promoteCurrent)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if *resultFile != "" {
result := struct {
BundleSHA256 string `json:"bundle_sha256"`
Publications []publication `json:"publications"`
SMTPDenySummary *smtpDenySummary `json:"smtp_deny_summary,omitempty"`
}{BundleSHA256: digestText, Publications: publications, SMTPDenySummary: smtpSummary}
if err := writeJSONFile(*resultFile, result); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
fmt.Printf("bundle_sha256=%s publication_count=%d current_promoted=%t\n", digestText, len(publications), *promoteCurrent)
}

358
packages/oci-l4-relay-publisher/cmd/publish/main_test.go Normal file
View file

@ -0,0 +1,358 @@
package main
import (
"bytes"
"context"
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestWriteJSONFileCreatesParentDirectory(t *testing.T) {
path := filepath.Join(t.TempDir(), "workflow", "publication-result.json")
if err := writeJSONFile(path, map[string]bool{"published": true}); err != nil {
t.Fatal(err)
}
contents, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(contents) != "{\"published\":true}\n" {
t.Fatalf("unexpected contents %q", contents)
}
}
func TestObjectURL(t *testing.T) {
result, err := objectURL("https://objectstorage.example.invalid", "namespace", "bucket", "admission/v1/current.json.gz")
if err != nil {
t.Fatal(err)
}
if result != "https://objectstorage.example.invalid/n/namespace/b/bucket/o/admission%2Fv1%2Fcurrent.json.gz" {
t.Fatalf("unexpected object URL %q", result)
}
}
func TestPublicationPrefixes(t *testing.T) {
prefixes, err := publicationPrefixes("admission/v1", []string{"web-allow/v1"})
if err != nil {
t.Fatal(err)
}
if strings.Join(prefixes, ",") != "admission/v1,web-allow/v1" {
t.Fatalf("unexpected prefixes %q", prefixes)
}
version, current := publicationObjects(prefixes[1], strings.Repeat("a", 64))
if version != "web-allow/v1/versions/"+strings.Repeat("a", 64)+".json.gz" {
t.Fatalf("unexpected version object %q", version)
}
if current != "web-allow/v1/current.json.gz" {
t.Fatalf("unexpected current object %q", current)
}
}
func TestPublicationPrefixesRejectInvalidOrDuplicateValues(t *testing.T) {
for name, values := range map[string][]string{
"empty": {""},
"traversal": {"../web-allow/v1"},
"nested": {"mail/smtp-deny/v1"},
"duplicate": {"web-allow/v1", "web-allow/v1"},
} {
t.Run(name, func(t *testing.T) {
if _, err := publicationPrefixes(values[0], values[1:]); err == nil {
t.Fatal("expected prefix validation failure")
}
})
}
}
func TestPublishBundleUsesIdenticalContentForBothPrefixes(t *testing.T) {
type uploadRequest struct {
body string
immutable bool
escapedPath string
}
requests := make([]uploadRequest, 0, 4)
objects := map[string][]byte{}
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodGet {
body, exists := objects[request.URL.EscapedPath()]
if !exists {
response.WriteHeader(http.StatusNotFound)
return
}
_, _ = response.Write(body)
return
}
body, err := io.ReadAll(request.Body)
if err != nil {
t.Error(err)
}
objects[request.URL.EscapedPath()] = body
requests = append(requests, uploadRequest{
body: string(body),
immutable: request.Header.Get("If-None-Match") == "*",
escapedPath: request.URL.EscapedPath(),
})
response.WriteHeader(http.StatusOK)
}))
defer server.Close()
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
body := []byte("validated bundle")
digest, publications, err := publishBundle(
context.Background(),
server.Client(),
server.URL,
"namespace",
"bucket",
[]string{"admission/v1", "web-allow/v1"},
body,
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
true,
)
if err != nil {
t.Fatal(err)
}
if len(publications) != 2 || len(requests) != 4 {
t.Fatalf("publications=%d requests=%d", len(publications), len(requests))
}
expectedPaths := []string{
"/n/namespace/b/bucket/o/admission%2Fv1%2Fversions%2F" + digest + ".json.gz",
"/n/namespace/b/bucket/o/admission%2Fv1%2Fcurrent.json.gz",
"/n/namespace/b/bucket/o/web-allow%2Fv1%2Fversions%2F" + digest + ".json.gz",
"/n/namespace/b/bucket/o/web-allow%2Fv1%2Fcurrent.json.gz",
}
for index, request := range requests {
if request.body != string(body) {
t.Errorf("request %d changed bundle content", index)
}
if request.escapedPath != expectedPaths[index] {
t.Errorf("request %d path %q", index, request.escapedPath)
}
if request.immutable != (index%2 == 0) {
t.Errorf("request %d immutable=%t", index, request.immutable)
}
}
}
func TestPublishBundleVerifiesExistingImmutableObject(t *testing.T) {
body := []byte("validated bundle")
digest := sha256.Sum256(body)
versionPath := "/n/namespace/b/bucket/o/smtp-deny%2Fv1%2Fversions%2F" + fmt.Sprintf("%x", digest) + ".json.gz"
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodPut {
response.WriteHeader(http.StatusPreconditionFailed)
return
}
if request.URL.EscapedPath() != versionPath {
t.Errorf("unexpected path %q", request.URL.EscapedPath())
}
_, _ = response.Write(body)
}))
defer server.Close()
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
_, publications, err := publishBundle(
context.Background(),
server.Client(),
server.URL,
"namespace",
"bucket",
[]string{"smtp-deny/v1"},
body,
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
false,
)
if err != nil {
t.Fatal(err)
}
if len(publications) != 1 || publications[0].VersionCreated {
t.Fatalf("unexpected publications %#v", publications)
}
}
func TestPublishBundleRejectsMismatchedExistingImmutableObject(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
if request.Method == http.MethodPut {
response.WriteHeader(http.StatusPreconditionFailed)
return
}
_, _ = response.Write([]byte("different bundle"))
}))
defer server.Close()
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
_, _, err = publishBundle(
context.Background(),
server.Client(),
server.URL,
"namespace",
"bucket",
[]string{"smtp-deny/v1"},
[]byte("validated bundle"),
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
false,
)
if err == nil || !strings.Contains(err.Error(), "SHA-256 mismatch") {
t.Fatalf("unexpected error %v", err)
}
}
func TestPublishBundleRejectsMismatchedCurrentObject(t *testing.T) {
objects := map[string][]byte{}
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
path := request.URL.EscapedPath()
if request.Method == http.MethodPut {
body, err := io.ReadAll(request.Body)
if err != nil {
t.Error(err)
}
objects[path] = body
response.WriteHeader(http.StatusOK)
return
}
if strings.HasSuffix(path, "current.json.gz") {
_, _ = response.Write([]byte("different bundle"))
return
}
_, _ = response.Write(objects[path])
}))
defer server.Close()
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
_, _, err = publishBundle(
context.Background(),
server.Client(),
server.URL,
"namespace",
"bucket",
[]string{"smtp-deny/v1"},
[]byte("validated bundle"),
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
true,
)
if err == nil || !strings.Contains(err.Error(), "current.json.gz") || !strings.Contains(err.Error(), "SHA-256 mismatch") {
t.Fatalf("unexpected error %v", err)
}
}
func TestReadSMTPDenySummary(t *testing.T) {
path := filepath.Join(t.TempDir(), "summary.json")
contents := `{"generated_at":"2026-07-26T00:00:00Z","expires_at":"2026-07-27T00:00:00Z","denied_interval_count":2,"denied_address_count":17,"removed_interval_count":1}`
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
summary, err := readSMTPDenySummary(path)
if err != nil {
t.Fatal(err)
}
if summary.DeniedIntervalCount != 2 || summary.DeniedAddressCount != 17 || summary.RemovedIntervalCount != 1 {
t.Fatalf("unexpected summary %#v", summary)
}
}
func TestReadSMTPDenySummaryRejectsContentEvidence(t *testing.T) {
path := filepath.Join(t.TempDir(), "summary.json")
contents := `{"generated_at":"2026-07-26T00:00:00Z","expires_at":"2026-07-27T00:00:00Z","denied_interval_count":1,"denied_address_count":1,"removed_interval_count":0,"recipient":"[email protected]"}`
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
if _, err := readSMTPDenySummary(path); err == nil {
t.Fatal("expected content-bearing summary rejection")
}
}
func TestSignRequest(t *testing.T) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
body := []byte("bundle")
request, err := http.NewRequest(http.MethodPut, "https://objectstorage.example.invalid/n/ns/b/b/o/current.json.gz", bytes.NewReader(body))
if err != nil {
t.Fatal(err)
}
identity := credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key}
if err := signRequest(request, body, identity, time.Unix(0, 0)); err != nil {
t.Fatal(err)
}
authorization := request.Header.Get("Authorization")
marker := `signature="`
start := strings.Index(authorization, marker)
if start < 0 {
t.Fatal("missing signature")
}
encoded := strings.TrimSuffix(authorization[start+len(marker):], `"`)
signature, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
t.Fatal(err)
}
signingString := strings.Join([]string{
"(request-target): put /n/ns/b/b/o/current.json.gz",
"host: objectstorage.example.invalid",
"date: " + request.Header.Get("Date"),
"x-content-sha256: " + request.Header.Get("X-Content-SHA256"),
"content-type: application/gzip",
"content-length: 6",
}, "\n")
digest := sha256.Sum256([]byte(signingString))
if err := rsa.VerifyPKCS1v15(&key.PublicKey, 0, digest[:], signature); err == nil {
t.Fatal("signature unexpectedly verified without SHA-256")
}
if err := rsa.VerifyPKCS1v15(&key.PublicKey, crypto.SHA256, digest[:], signature); err != nil {
t.Fatal(err)
}
}
func TestSignDownloadRequest(t *testing.T) {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
request, err := http.NewRequest(http.MethodGet, "https://objectstorage.example.invalid/n/ns/b/b/o/admission%2Fv1%2Fversions%2Fbundle.json.gz", nil)
if err != nil {
t.Fatal(err)
}
identity := credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key}
if err := signDownloadRequest(request, identity, time.Unix(0, 0)); err != nil {
t.Fatal(err)
}
authorization := request.Header.Get("Authorization")
marker := `signature="`
start := strings.Index(authorization, marker)
if start < 0 {
t.Fatal("missing signature")
}
encoded := strings.TrimSuffix(authorization[start+len(marker):], `"`)
signature, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
t.Fatal(err)
}
signingString := strings.Join([]string{
"(request-target): get /n/ns/b/b/o/admission%2Fv1%2Fversions%2Fbundle.json.gz",
"host: objectstorage.example.invalid",
"date: " + request.Header.Get("Date"),
}, "\n")
digest := sha256.Sum256([]byte(signingString))
if err := rsa.VerifyPKCS1v15(&key.PublicKey, crypto.SHA256, digest[:], signature); err != nil {
t.Fatal(err)
}
}

3
packages/oci-l4-relay-publisher/go.mod Normal file
View file

@ -0,0 +1,3 @@
module quorate.dev/packages/oci-l4-relay-publisher
go 1.26.4

900
packages/oci-l4-relay-publisher/main.go Normal file
View file

@ -0,0 +1,900 @@
package main
import (
"archive/zip"
"bufio"
"compress/gzip"
"encoding/csv"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
func headerIndexes(header []string) map[string]int {
result := make(map[string]int, len(header))
for index, name := range header {
result[name] = index
}
return result
}
func fileWithSuffix(archive *zip.ReadCloser, suffix string) (*zip.File, error) {
for _, file := range archive.File {
if strings.HasSuffix(file.Name, suffix) {
return file, nil
}
}
return nil, fmt.Errorf("missing %s", suffix)
}
func openCSV(file *zip.File) (*csv.Reader, io.Closer, error) {
contents, err := file.Open()
if err != nil {
return nil, nil, err
}
return csv.NewReader(contents), contents, nil
}
func selectedCountry(country, continent string) bool {
return country == "JP" || country == "KR" || country == "SG" || country == "US" || continent == "EU"
}
func vpnExceptionCountry(country string) bool {
return country == "JP" || country == "KR"
}
type geographySets struct {
admission []addressRange
vpnException []addressRange
}
type countrySelection struct {
admission bool
vpnException bool
}
func maxMindGeography(path string) (geographySets, error) {
archive, err := zip.OpenReader(path)
if err != nil {
return geographySets{}, err
}
defer func() { _ = archive.Close() }()
locationFile, err := fileWithSuffix(archive, "Country-Locations-en.csv")
if err != nil {
return geographySets{}, err
}
locationReader, locationContents, err := openCSV(locationFile)
if err != nil {
return geographySets{}, err
}
locationHeader, err := locationReader.Read()
if err != nil {
_ = locationContents.Close()
return geographySets{}, err
}
locationFields := headerIndexes(locationHeader)
locations := make(map[int]countrySelection)
for {
row, err := locationReader.Read()
if err == io.EOF {
break
}
if err != nil {
_ = locationContents.Close()
return geographySets{}, err
}
id, err := strconv.Atoi(row[locationFields["geoname_id"]])
if err == nil {
country := row[locationFields["country_iso_code"]]
locations[id] = countrySelection{
admission: selectedCountry(country, row[locationFields["continent_code"]]),
vpnException: vpnExceptionCountry(country),
}
}
}
if err := locationContents.Close(); err != nil {
return geographySets{}, err
}
blocksFile, err := fileWithSuffix(archive, "Country-Blocks-IPv4.csv")
if err != nil {
return geographySets{}, err
}
blocksReader, blocksContents, err := openCSV(blocksFile)
if err != nil {
return geographySets{}, err
}
defer func() { _ = blocksContents.Close() }()
blocksHeader, err := blocksReader.Read()
if err != nil {
return geographySets{}, err
}
blockFields := headerIndexes(blocksHeader)
result := geographySets{}
for {
row, err := blocksReader.Read()
if err == io.EOF {
break
}
if err != nil {
return geographySets{}, err
}
idText := row[blockFields["geoname_id"]]
if idText == "" {
idText = row[blockFields["registered_country_geoname_id"]]
}
id, err := strconv.Atoi(idText)
selection, found := locations[id]
if err != nil || !found {
continue
}
if network, ok := rangeFromPrefix(row[blockFields["network"]]); ok {
if selection.admission {
result.admission = append(result.admission, network)
}
if selection.vpnException {
result.vpnException = append(result.vpnException, network)
}
}
}
result.admission = mergeRanges(result.admission)
result.vpnException = mergeRanges(result.vpnException)
return result, nil
}
func ipLocateGeography(path string) (geographySets, error) {
archive, err := zip.OpenReader(path)
if err != nil {
return geographySets{}, err
}
defer func() { _ = archive.Close() }()
file, err := fileWithSuffix(archive, ".csv")
if err != nil {
return geographySets{}, err
}
reader, contents, err := openCSV(file)
if err != nil {
return geographySets{}, err
}
defer func() { _ = contents.Close() }()
header, err := reader.Read()
if err != nil {
return geographySets{}, err
}
fields := headerIndexes(header)
result := geographySets{}
for {
row, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return geographySets{}, err
}
country := row[fields["country_code"]]
if network, ok := rangeFromPrefix(row[fields["network"]]); ok {
if selectedCountry(country, row[fields["continent_code"]]) {
result.admission = append(result.admission, network)
}
if vpnExceptionCountry(country) {
result.vpnException = append(result.vpnException, network)
}
}
}
result.admission = mergeRanges(result.admission)
result.vpnException = mergeRanges(result.vpnException)
return result, nil
}
func parseASN(value string) (uint32, error) {
value = strings.TrimSpace(value)
if len(value) > 2 && strings.EqualFold(value[:2], "AS") {
value = value[2:]
}
parsed, err := strconv.ParseUint(value, 10, 32)
if err != nil || parsed == 0 {
return 0, fmt.Errorf("invalid ASN %q", value)
}
return uint32(parsed), nil
}
func ipLocateASNRanges(path string, selected map[uint32]struct{}) ([]addressRange, error) {
if len(selected) == 0 {
return nil, nil
}
archive, err := zip.OpenReader(path)
if err != nil {
return nil, err
}
defer func() { _ = archive.Close() }()
file, err := fileWithSuffix(archive, ".csv")
if err != nil {
return nil, err
}
reader, contents, err := openCSV(file)
if err != nil {
return nil, err
}
defer func() { _ = contents.Close() }()
header, err := reader.Read()
if err != nil {
return nil, err
}
fields := headerIndexes(header)
var result []addressRange
for {
row, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
asn, err := parseASN(row[fields["asn"]])
if err != nil {
continue
}
if _, found := selected[asn]; !found {
continue
}
if network, ok := rangeFromPrefix(row[fields["network"]]); ok {
result = append(result, network)
}
}
return mergeRanges(result), nil
}
func jsonContents(path string, target any) error {
contents, err := os.ReadFile(path)
if err != nil {
return err
}
return json.Unmarshal(contents, target)
}
func prefixRanges(prefixes []string) []addressRange {
result := make([]addressRange, 0, len(prefixes))
for _, prefix := range prefixes {
if network, ok := rangeFromPrefix(prefix); ok {
result = append(result, network)
}
}
return mergeRanges(result)
}
func awsRanges(path string) ([]addressRange, error) {
var document struct {
Prefixes []struct {
Prefix string `json:"ip_prefix"`
} `json:"prefixes"`
}
if err := jsonContents(path, &document); err != nil {
return nil, err
}
prefixes := make([]string, 0, len(document.Prefixes))
for _, item := range document.Prefixes {
prefixes = append(prefixes, item.Prefix)
}
return prefixRanges(prefixes), nil
}
func gcpRanges(path string) ([]addressRange, error) {
var document struct {
Prefixes []struct {
Prefix string `json:"ipv4Prefix"`
} `json:"prefixes"`
}
if err := jsonContents(path, &document); err != nil {
return nil, err
}
prefixes := make([]string, 0, len(document.Prefixes))
for _, item := range document.Prefixes {
prefixes = append(prefixes, item.Prefix)
}
return prefixRanges(prefixes), nil
}
func azureRanges(path string) ([]addressRange, error) {
var document struct {
Values []struct {
Properties struct {
Prefixes []string `json:"addressPrefixes"`
} `json:"properties"`
} `json:"values"`
}
if err := jsonContents(path, &document); err != nil {
return nil, err
}
var prefixes []string
for _, value := range document.Values {
prefixes = append(prefixes, value.Properties.Prefixes...)
}
return prefixRanges(prefixes), nil
}
func ociRanges(path string) ([]addressRange, error) {
var document struct {
Regions []struct {
CIDRs []struct {
CIDR string `json:"cidr"`
} `json:"cidrs"`
} `json:"regions"`
}
if err := jsonContents(path, &document); err != nil {
return nil, err
}
var prefixes []string
for _, region := range document.Regions {
for _, cidr := range region.CIDRs {
prefixes = append(prefixes, cidr.CIDR)
}
}
return prefixRanges(prefixes), nil
}
func fastlyRanges(path string) ([]addressRange, error) {
var document struct {
Addresses []string `json:"addresses"`
}
if err := jsonContents(path, &document); err != nil {
return nil, err
}
return prefixRanges(document.Addresses), nil
}
func githubRanges(path string) ([]addressRange, error) {
var document map[string]json.RawMessage
if err := jsonContents(path, &document); err != nil {
return nil, err
}
var prefixes []string
for _, contents := range document {
var values []string
if err := json.Unmarshal(contents, &values); err == nil {
prefixes = append(prefixes, values...)
}
}
return prefixRanges(prefixes), nil
}
func atlassianRanges(path string) ([]addressRange, error) {
var document struct {
Items []struct {
CIDR string `json:"cidr"`
} `json:"items"`
}
if err := jsonContents(path, &document); err != nil {
return nil, err
}
prefixes := make([]string, 0, len(document.Items))
for _, item := range document.Items {
prefixes = append(prefixes, item.CIDR)
}
return prefixRanges(prefixes), nil
}
func geofeedRanges(path string) ([]addressRange, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
reader := csv.NewReader(file)
reader.Comment = '#'
reader.FieldsPerRecord = -1
var result []addressRange
for {
row, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if len(row) == 0 {
continue
}
if network, ok := rangeFromPrefix(strings.TrimSpace(row[0])); ok {
result = append(result, network)
}
}
return mergeRanges(result), nil
}
func lineRanges(path string) ([]addressRange, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
var result []addressRange
scanner := bufio.NewScanner(file)
for scanner.Scan() {
value := strings.TrimSpace(scanner.Text())
if value == "" {
continue
}
if !strings.Contains(value, "/") {
value += "/32"
}
if network, ok := rangeFromPrefix(value); ok {
result = append(result, network)
}
}
return mergeRanges(result), scanner.Err()
}
func spamhausRanges(path string) ([]addressRange, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
decoder := json.NewDecoder(file)
var result []addressRange
for {
var record struct {
CIDR string `json:"cidr"`
}
if err := decoder.Decode(&record); err == io.EOF {
break
} else if err != nil {
return nil, err
}
if network, ok := rangeFromPrefix(record.CIDR); ok {
result = append(result, network)
}
}
return mergeRanges(result), nil
}
func unionRanges(sets ...[]addressRange) []addressRange {
var result []addressRange
for _, set := range sets {
result = append(result, set...)
}
return mergeRanges(result)
}
func percent(part, whole uint64) float64 {
if whole == 0 {
return 0
}
return float64(part) * 100 / float64(whole)
}
type categoryStat struct {
records int
addresses uint64
}
type ip2ProxyStats struct {
all []addressRange
recentSevenDays []addressRange
threat []addressRange
highFraudScore []addressRange
usageTypes map[string]categoryStat
proxyTypes map[string]categoryStat
threatRecords int
positiveFraudScores int
minimumLastSeen int
maximumLastSeen int
}
func incrementCategory(categories map[string]categoryStat, name string, size uint64) {
current := categories[name]
current.records++
current.addresses += size
categories[name] = current
}
func ip2ProxyClassifications(path string) (ip2ProxyStats, error) {
archive, err := zip.OpenReader(path)
if err != nil {
return ip2ProxyStats{}, err
}
defer func() { _ = archive.Close() }()
file, err := fileWithSuffix(archive, "IP2PROXY-LITE-PX12.CSV")
if err != nil {
return ip2ProxyStats{}, err
}
reader, contents, err := openCSV(file)
if err != nil {
return ip2ProxyStats{}, err
}
defer func() { _ = contents.Close() }()
result := ip2ProxyStats{usageTypes: make(map[string]categoryStat), proxyTypes: make(map[string]categoryStat)}
for {
row, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return ip2ProxyStats{}, err
}
if len(row) != 16 {
return ip2ProxyStats{}, fmt.Errorf("unexpected IP2Proxy field count %d", len(row))
}
start, err := strconv.ParseUint(row[0], 10, 32)
if err != nil {
return ip2ProxyStats{}, err
}
end, err := strconv.ParseUint(row[1], 10, 32)
if err != nil {
return ip2ProxyStats{}, err
}
network := addressRange{start: uint32(start), end: uint32(end)}
size := end - start + 1
proxyType := row[2]
usageType := row[9]
incrementCategory(result.proxyTypes, proxyType, size)
incrementCategory(result.usageTypes, usageType, size)
result.all = append(result.all, network)
if row[13] != "-" {
result.threatRecords++
result.threat = append(result.threat, network)
}
lastSeen, err := strconv.Atoi(row[12])
if err == nil && lastSeen > 0 {
if result.minimumLastSeen == 0 || lastSeen < result.minimumLastSeen {
result.minimumLastSeen = lastSeen
}
if lastSeen > result.maximumLastSeen {
result.maximumLastSeen = lastSeen
}
if lastSeen <= 7 {
result.recentSevenDays = append(result.recentSevenDays, network)
}
}
fraudScore, err := strconv.Atoi(row[15])
if err == nil && fraudScore > 0 {
result.positiveFraudScores++
}
if err == nil && fraudScore >= 80 {
result.highFraudScore = append(result.highFraudScore, network)
}
}
result.all = mergeRanges(result.all)
result.recentSevenDays = mergeRanges(result.recentSevenDays)
result.threat = mergeRanges(result.threat)
result.highFraudScore = mergeRanges(result.highFraudScore)
return result, nil
}
func loadIP2ProxyClassifications(path string, now time.Time, maximumAge time.Duration) (ip2ProxyStats, string, time.Duration, error) {
empty := ip2ProxyStats{usageTypes: make(map[string]categoryStat), proxyTypes: make(map[string]categoryStat)}
if maximumAge <= 0 {
return empty, "", 0, errors.New("maximum IP2Proxy age must be positive")
}
info, err := os.Stat(path)
if errors.Is(err, os.ErrNotExist) {
return empty, "absent", 0, nil
}
if err != nil {
return empty, "", 0, err
}
if info.ModTime().After(now.Add(5 * time.Minute)) {
return empty, "", 0, errors.New("IP2Proxy cache modification time is in the future")
}
age := now.Sub(info.ModTime())
if age > maximumAge {
return empty, "stale", age, nil
}
classifications, err := ip2ProxyClassifications(path)
if err != nil {
return empty, "", age, err
}
return classifications, "fresh", age, nil
}
func printCategories(name string, categories map[string]categoryStat) {
keys := make([]string, 0, len(categories))
for key := range categories {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
label := key
if label == "-" {
label = "unclassified"
}
fmt.Printf("%s_%s_records=%d\n", name, strings.ToLower(label), categories[key].records)
fmt.Printf("%s_%s_addresses=%d\n", name, strings.ToLower(label), categories[key].addresses)
}
}
func printSet(name string, ranges []addressRange) {
fmt.Printf("%s_intervals=%d\n", name, len(ranges))
fmt.Printf("%s_addresses=%d\n", name, addressCount(ranges))
}
func ipv4String(value uint32) string {
return fmt.Sprintf("%d.%d.%d.%d", byte(value>>24), byte(value>>16), byte(value>>8), byte(value))
}
func writeNftSet(writer *bufio.Writer, name string, ranges []addressRange) error {
if _, err := fmt.Fprintf(writer, " set %s {\n type ipv4_addr\n flags interval\n auto-merge\n elements = {\n", name); err != nil {
return err
}
for index, current := range ranges {
separator := ","
if index == len(ranges)-1 {
separator = ""
}
value := ipv4String(current.start)
if current.start != current.end {
value += "-" + ipv4String(current.end)
}
if _, err := fmt.Fprintf(writer, " %s%s\n", value, separator); err != nil {
return err
}
}
_, err := fmt.Fprint(writer, " }\n }\n")
return err
}
func writeNft(path string, geography, denied []addressRange) error {
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
if err != nil {
return err
}
writer := bufio.NewWriter(file)
if _, err := fmt.Fprintln(writer, "table inet relay_spike {"); err != nil {
_ = file.Close()
return err
}
if err := writeNftSet(writer, "geography_v4", geography); err != nil {
_ = file.Close()
return err
}
if err := writeNftSet(writer, "deny_v4", denied); err != nil {
_ = file.Close()
return err
}
if _, err := fmt.Fprintln(writer, "}"); err != nil {
_ = file.Close()
return err
}
if err := writer.Flush(); err != nil {
_ = file.Close()
return err
}
return file.Close()
}
type bundleRange struct {
Start string `json:"start"`
End string `json:"end"`
}
type admissionBundle struct {
SchemaVersion int `json:"schema_version"`
GeneratedAt time.Time `json:"generated_at"`
ExpiresAt time.Time `json:"expires_at"`
AdmittedIPv4 []bundleRange `json:"admitted_ipv4"`
ExceptionIPv4 []bundleRange `json:"exception_ipv4"`
AdmittedIntervalCount int `json:"admitted_interval_count"`
AdmittedAddressCount uint64 `json:"admitted_address_count"`
ExceptionIntervalCount int `json:"exception_interval_count"`
ExceptionAddressCount uint64 `json:"exception_address_count"`
}
func writeBundle(path string, admitted, exceptions []addressRange, now time.Time) error {
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
if err != nil {
return err
}
compressed := gzip.NewWriter(file)
document := admissionBundle{
SchemaVersion: 1,
GeneratedAt: now.UTC(),
ExpiresAt: now.UTC().Add(72 * time.Hour),
AdmittedIPv4: make([]bundleRange, 0, len(admitted)),
ExceptionIPv4: make([]bundleRange, 0, len(exceptions)),
AdmittedIntervalCount: len(admitted),
AdmittedAddressCount: addressCount(admitted),
ExceptionIntervalCount: len(exceptions),
ExceptionAddressCount: addressCount(exceptions),
}
for _, item := range admitted {
document.AdmittedIPv4 = append(document.AdmittedIPv4, bundleRange{Start: ipv4String(item.start), End: ipv4String(item.end)})
}
for _, item := range exceptions {
document.ExceptionIPv4 = append(document.ExceptionIPv4, bundleRange{Start: ipv4String(item.start), End: ipv4String(item.end)})
}
encodeErr := json.NewEncoder(compressed).Encode(document)
closeGzipErr := compressed.Close()
closeFileErr := file.Close()
if encodeErr != nil {
return encodeErr
}
if closeGzipErr != nil {
return closeGzipErr
}
return closeFileErr
}
func main() {
nftOutput := flag.String("nft-output", "", "write selective nftables sets")
nftFullOutput := flag.String("nft-full-output", "", "write complete nftables sets")
bundleOutput := flag.String("bundle-output", "", "write normalized selective admission bundle")
maximumIP2ProxyAge := flag.Duration("maximum-ip2proxy-age", 35*24*time.Hour, "omit IP2Proxy classifications after this cache age")
vpnExceptionASNs := map[uint32]struct{}{}
flag.Func("vpn-exception-asn", "VPN operator ASN admitted only within MaxMind or IPLocate JP/KR ranges", func(value string) error {
asn, err := parseASN(value)
if err != nil {
return err
}
vpnExceptionASNs[asn] = struct{}{}
return nil
})
flag.Parse()
now := time.Now().UTC()
maxMind, err := maxMindGeography("data/maxmind-country.zip")
if err != nil {
panic(err)
}
ipLocate, err := ipLocateGeography("data/iplocate-country.zip")
if err != nil {
panic(err)
}
vpnASN, err := ipLocateASNRanges("data/iplocate-asn.zip", vpnExceptionASNs)
if err != nil {
panic(err)
}
aws, err := awsRanges("data/aws.json")
if err != nil {
panic(err)
}
gcp, err := gcpRanges("data/gcp.json")
if err != nil {
panic(err)
}
azure, err := azureRanges("data/azure.json")
if err != nil {
panic(err)
}
oci, err := ociRanges("data/oci.json")
if err != nil {
panic(err)
}
cloudflare, err := lineRanges("data/cloudflare.txt")
if err != nil {
panic(err)
}
tor, err := lineRanges("data/tor-exits.txt")
if err != nil {
panic(err)
}
spamhaus, err := spamhausRanges("data/spamhaus-drop.json")
if err != nil {
panic(err)
}
fastly, err := fastlyRanges("data/fastly.json")
if err != nil {
panic(err)
}
github, err := githubRanges("data/github.json")
if err != nil {
panic(err)
}
atlassian, err := atlassianRanges("data/atlassian.json")
if err != nil {
panic(err)
}
digitalOcean, err := geofeedRanges("data/digitalocean.csv")
if err != nil {
panic(err)
}
linode, err := geofeedRanges("data/linode.csv")
if err != nil {
panic(err)
}
vultr, err := geofeedRanges("data/vultr.csv")
if err != nil {
panic(err)
}
ip2Proxy, ip2ProxyState, ip2ProxyAge, err := loadIP2ProxyClassifications("data/ip2proxy.zip", now, *maximumIP2ProxyAge)
if err != nil {
panic(err)
}
fmt.Printf("ip2proxy_cache_state=%s\n", ip2ProxyState)
if ip2ProxyState != "absent" {
fmt.Printf("ip2proxy_cache_age_hours=%.2f\n", ip2ProxyAge.Hours())
}
fmt.Printf("ip2proxy_cache_omitted=%t\n", ip2ProxyState != "fresh")
consensus := intersectRanges(maxMind.admission, ipLocate.admission)
geoUnion := unionRanges(maxMind.admission, ipLocate.admission)
geoDisagreement := addressCount(geoUnion) - addressCount(consensus)
vpnExceptionGeography := unionRanges(maxMind.vpnException, ipLocate.vpnException)
vpnExceptionCandidate := subtractRanges(intersectRanges(vpnASN, vpnExceptionGeography), consensus)
cloud := unionRanges(aws, gcp, azure, oci, cloudflare, fastly, github, atlassian, digitalOcean, linode, vultr)
denied := unionRanges(cloud, tor, spamhaus, ip2Proxy.all)
selectiveProxy := unionRanges(ip2Proxy.recentSevenDays, ip2Proxy.threat)
selectiveDenied := unionRanges(cloud, tor, spamhaus, selectiveProxy)
deniedInConsensus := intersectRanges(consensus, denied)
selectiveDeniedInConsensus := intersectRanges(consensus, selectiveDenied)
candidate := subtractRanges(consensus, denied)
selectiveCandidate := subtractRanges(consensus, selectiveDenied)
vpnExceptions := subtractRanges(vpnExceptionCandidate, denied)
selectiveVPNExceptions := subtractRanges(vpnExceptionCandidate, selectiveDenied)
printSet("maxmind_geo", maxMind.admission)
printSet("iplocate_geo", ipLocate.admission)
printSet("geo_consensus", consensus)
fmt.Printf("geo_disagreement_addresses=%d\n", geoDisagreement)
fmt.Printf("geo_disagreement_percent_of_union=%.6f\n", percent(geoDisagreement, addressCount(geoUnion)))
printSet("vpn_exception_geo_union", vpnExceptionGeography)
printSet("vpn_exception_asn", vpnASN)
printSet("vpn_exception_candidate_before_denies", vpnExceptionCandidate)
printSet("vpn_exception_after_denies", vpnExceptions)
printSet("selective_vpn_exception_after_denies", selectiveVPNExceptions)
printSet("aws", aws)
printSet("gcp", gcp)
printSet("azure", azure)
printSet("oci", oci)
printSet("cloudflare", cloudflare)
printSet("fastly", fastly)
printSet("github", github)
printSet("atlassian", atlassian)
printSet("digitalocean", digitalOcean)
printSet("linode", linode)
printSet("vultr", vultr)
printSet("cloud_union", cloud)
printSet("tor", tor)
printSet("spamhaus", spamhaus)
printSet("deny_union", denied)
printSet("deny_in_geo_consensus", deniedInConsensus)
fmt.Printf("deny_percent_of_geo_consensus=%.6f\n", percent(addressCount(deniedInConsensus), addressCount(consensus)))
printSet("candidate_after_denies", candidate)
printSet("selective_proxy", selectiveProxy)
printSet("selective_deny_union", selectiveDenied)
printSet("selective_deny_in_geo_consensus", selectiveDeniedInConsensus)
fmt.Printf("selective_deny_percent_of_geo_consensus=%.6f\n", percent(addressCount(selectiveDeniedInConsensus), addressCount(consensus)))
printSet("selective_candidate_after_denies", selectiveCandidate)
printSet("ip2proxy_open_proxy", ip2Proxy.all)
printSet("ip2proxy_recent_seven_days", ip2Proxy.recentSevenDays)
printSet("ip2proxy_threat", ip2Proxy.threat)
printSet("ip2proxy_high_fraud_score", ip2Proxy.highFraudScore)
printCategories("ip2proxy_usage", ip2Proxy.usageTypes)
printCategories("ip2proxy_proxy", ip2Proxy.proxyTypes)
fmt.Printf("ip2proxy_threat_records=%d\n", ip2Proxy.threatRecords)
fmt.Printf("ip2proxy_positive_fraud_score_records=%d\n", ip2Proxy.positiveFraudScores)
fmt.Printf("ip2proxy_minimum_last_seen_days=%d\n", ip2Proxy.minimumLastSeen)
fmt.Printf("ip2proxy_maximum_last_seen_days=%d\n", ip2Proxy.maximumLastSeen)
fmt.Println("open_proxy_classification=true")
fmt.Println("consumer_classification=false")
fmt.Println("generic_vpn_classification=false")
fmt.Println("residential_proxy_classification=false")
fmt.Printf("explicit_vpn_exception_asns=%d\n", len(vpnExceptionASNs))
if *nftOutput != "" {
if err := writeNft(*nftOutput, unionRanges(consensus, vpnExceptionCandidate), selectiveDenied); err != nil {
panic(err)
}
fmt.Println("nft_output_written=true")
}
if *nftFullOutput != "" {
if err := writeNft(*nftFullOutput, unionRanges(consensus, vpnExceptionCandidate), denied); err != nil {
panic(err)
}
fmt.Println("nft_full_output_written=true")
}
if *bundleOutput != "" {
if err := writeBundle(*bundleOutput, selectiveCandidate, selectiveVPNExceptions, now); err != nil {
panic(err)
}
fmt.Println("bundle_output_written=true")
}
}

184
packages/oci-l4-relay-publisher/main_test.go Normal file
View file

@ -0,0 +1,184 @@
package main
import (
"archive/zip"
"fmt"
"os"
"path/filepath"
"testing"
"time"
)
func writeZipFiles(t *testing.T, files map[string]string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "fixture.zip")
output, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
archive := zip.NewWriter(output)
for name, contents := range files {
entry, err := archive.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := fmt.Fprint(entry, contents); err != nil {
t.Fatal(err)
}
}
if err := archive.Close(); err != nil {
t.Fatal(err)
}
if err := output.Close(); err != nil {
t.Fatal(err)
}
return path
}
func mustAddress(t *testing.T, value string) uint32 {
t.Helper()
address, ok := rangeFromPrefix(value + "/32")
if !ok {
t.Fatalf("invalid test address %q", value)
}
return address.start
}
func rangeContains(ranges []addressRange, address uint32) bool {
for _, candidate := range ranges {
if candidate.start <= address && candidate.end >= address {
return true
}
}
return false
}
func rangesEqual(left, right []addressRange) bool {
left = mergeRanges(left)
right = mergeRanges(right)
if len(left) != len(right) {
return false
}
for index := range left {
if left[index] != right[index] {
return false
}
}
return true
}
func TestGeographySetsSeparateVPNExceptionCountries(t *testing.T) {
path := writeZipFiles(t, map[string]string{
"ip-to-country.csv": "network,country_code,continent_code\n192.0.2.0/28,JP,AS\n192.0.2.16/28,KR,AS\n198.51.100.0/24,US,NA\n203.0.113.0/24,BR,SA\n",
})
sets, err := ipLocateGeography(path)
if err != nil {
t.Fatal(err)
}
if !rangeContains(sets.admission, mustAddress(t, "198.51.100.1")) || rangeContains(sets.admission, mustAddress(t, "203.0.113.1")) {
t.Fatalf("unexpected admission ranges: %#v", sets.admission)
}
if !rangeContains(sets.vpnException, mustAddress(t, "192.0.2.1")) || !rangeContains(sets.vpnException, mustAddress(t, "192.0.2.17")) || rangeContains(sets.vpnException, mustAddress(t, "198.51.100.1")) {
t.Fatalf("unexpected VPN exception ranges: %#v", sets.vpnException)
}
}
func TestMaxMindGeographySetsSeparateVPNExceptionCountries(t *testing.T) {
path := writeZipFiles(t, map[string]string{
"GeoLite2-Country-Locations-en.csv": "geoname_id,locale_code,continent_code,country_iso_code\n1,en,AS,JP\n2,en,AS,KR\n3,en,NA,US\n4,en,SA,BR\n",
"GeoLite2-Country-Blocks-IPv4.csv": "network,geoname_id,registered_country_geoname_id\n192.0.2.0/28,1,1\n192.0.2.16/28,2,2\n198.51.100.0/24,3,3\n203.0.113.0/24,4,4\n",
})
sets, err := maxMindGeography(path)
if err != nil {
t.Fatal(err)
}
if !rangeContains(sets.admission, mustAddress(t, "198.51.100.1")) || rangeContains(sets.admission, mustAddress(t, "203.0.113.1")) {
t.Fatalf("unexpected admission ranges: %#v", sets.admission)
}
if !rangeContains(sets.vpnException, mustAddress(t, "192.0.2.1")) || !rangeContains(sets.vpnException, mustAddress(t, "192.0.2.17")) || rangeContains(sets.vpnException, mustAddress(t, "198.51.100.1")) {
t.Fatalf("unexpected VPN exception ranges: %#v", sets.vpnException)
}
}
func TestIPLocateASNRangesSelectConfiguredOperators(t *testing.T) {
path := writeZipFiles(t, map[string]string{
"ip-to-asn.csv": "network,asn,country_code,name,org,domain\n192.0.2.0/28,147049,AU,selected,selected,example.test\n192.0.2.16/28,64500,JP,other,other,example.test\n198.51.100.0/24,AS147049,AU,selected,selected,example.test\n",
})
ranges, err := ipLocateASNRanges(path, map[uint32]struct{}{147049: {}})
if err != nil {
t.Fatal(err)
}
if !rangeContains(ranges, mustAddress(t, "192.0.2.1")) || !rangeContains(ranges, mustAddress(t, "198.51.100.1")) || rangeContains(ranges, mustAddress(t, "192.0.2.17")) {
t.Fatalf("unexpected ASN ranges: %#v", ranges)
}
}
func TestVPNExceptionCompositionPreservesConsensusAndDenies(t *testing.T) {
consensus := []addressRange{{start: 10, end: 20}}
vpnASN := []addressRange{{start: 15, end: 30}}
vpnGeography := []addressRange{{start: 18, end: 28}}
denied := []addressRange{{start: 24, end: 25}}
candidate := subtractRanges(intersectRanges(vpnASN, vpnGeography), consensus)
exceptions := subtractRanges(candidate, denied)
want := []addressRange{{start: 21, end: 23}, {start: 26, end: 28}}
if !rangesEqual(exceptions, want) {
t.Fatalf("exceptions=%#v want=%#v", exceptions, want)
}
}
func TestParseASN(t *testing.T) {
for _, value := range []string{"147049", "AS147049", "as147049"} {
asn, err := parseASN(value)
if err != nil || asn != 147049 {
t.Fatalf("parseASN(%q)=%d,%v", value, asn, err)
}
}
for _, value := range []string{"", "AS", "0", "-1", "4294967296"} {
if _, err := parseASN(value); err == nil {
t.Fatalf("parseASN(%q) succeeded", value)
}
}
}
func TestLoadIP2ProxyClassificationsOmitsStaleCache(t *testing.T) {
now := time.Unix(2000000000, 0)
path := filepath.Join(t.TempDir(), "ip2proxy.zip")
if err := os.WriteFile(path, []byte("invalid archive"), 0o600); err != nil {
t.Fatal(err)
}
modified := now.Add(-36 * 24 * time.Hour)
if err := os.Chtimes(path, modified, modified); err != nil {
t.Fatal(err)
}
result, state, age, err := loadIP2ProxyClassifications(path, now, 35*24*time.Hour)
if err != nil {
t.Fatal(err)
}
if state != "stale" || age != 36*24*time.Hour || len(result.all) != 0 {
t.Fatalf("state=%s age=%s ranges=%d", state, age, len(result.all))
}
}
func TestLoadIP2ProxyClassificationsRequiresValidFreshCache(t *testing.T) {
now := time.Unix(2000000000, 0)
path := filepath.Join(t.TempDir(), "ip2proxy.zip")
if err := os.WriteFile(path, []byte("invalid archive"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Chtimes(path, now, now); err != nil {
t.Fatal(err)
}
if _, _, _, err := loadIP2ProxyClassifications(path, now, 35*24*time.Hour); err == nil {
t.Fatal("expected fresh invalid cache to fail")
}
}
func TestLoadIP2ProxyClassificationsAllowsAbsentCache(t *testing.T) {
result, state, age, err := loadIP2ProxyClassifications(filepath.Join(t.TempDir(), "missing.zip"), time.Now(), 35*24*time.Hour)
if err != nil {
t.Fatal(err)
}
if state != "absent" || age != 0 || len(result.all) != 0 {
t.Fatalf("state=%s age=%s ranges=%d", state, age, len(result.all))
}
}

110
packages/oci-l4-relay-publisher/ranges.go Normal file
View file

@ -0,0 +1,110 @@
package main
import (
"encoding/binary"
"net/netip"
"sort"
)
type addressRange struct {
start uint32
end uint32
}
func rangeFromPrefix(value string) (addressRange, bool) {
prefix, err := netip.ParsePrefix(value)
if err != nil || !prefix.Addr().Is4() {
return addressRange{}, false
}
address := prefix.Masked().Addr().As4()
start := binary.BigEndian.Uint32(address[:])
hostBits := 32 - prefix.Bits()
end := start
if hostBits == 32 {
end = ^uint32(0)
} else if hostBits > 0 {
end |= (uint32(1) << hostBits) - 1
}
return addressRange{start: start, end: end}, true
}
func mergeRanges(ranges []addressRange) []addressRange {
if len(ranges) == 0 {
return nil
}
sort.Slice(ranges, func(i, j int) bool {
if ranges[i].start == ranges[j].start {
return ranges[i].end < ranges[j].end
}
return ranges[i].start < ranges[j].start
})
merged := []addressRange{ranges[0]}
for _, candidate := range ranges[1:] {
current := &merged[len(merged)-1]
adjacent := current.end != ^uint32(0) && candidate.start == current.end+1
if candidate.start <= current.end || adjacent {
if candidate.end > current.end {
current.end = candidate.end
}
continue
}
merged = append(merged, candidate)
}
return merged
}
func intersectRanges(left, right []addressRange) []addressRange {
left = mergeRanges(left)
right = mergeRanges(right)
var result []addressRange
for leftIndex, rightIndex := 0, 0; leftIndex < len(left) && rightIndex < len(right); {
start := max(left[leftIndex].start, right[rightIndex].start)
end := min(left[leftIndex].end, right[rightIndex].end)
if start <= end {
result = append(result, addressRange{start: start, end: end})
}
if left[leftIndex].end < right[rightIndex].end {
leftIndex++
} else {
rightIndex++
}
}
return mergeRanges(result)
}
func subtractRanges(base, denied []addressRange) []addressRange {
base = mergeRanges(base)
denied = mergeRanges(denied)
var result []addressRange
denyIndex := 0
for _, candidate := range base {
cursor := uint64(candidate.start)
end := uint64(candidate.end)
for denyIndex < len(denied) && uint64(denied[denyIndex].end) < cursor {
denyIndex++
}
for index := denyIndex; index < len(denied) && uint64(denied[index].start) <= end; index++ {
if uint64(denied[index].start) > cursor {
result = append(result, addressRange{start: uint32(cursor), end: denied[index].start - 1})
}
if uint64(denied[index].end)+1 > cursor {
cursor = uint64(denied[index].end) + 1
}
if cursor > end {
break
}
}
if cursor <= end {
result = append(result, addressRange{start: uint32(cursor), end: uint32(end)})
}
}
return mergeRanges(result)
}
func addressCount(ranges []addressRange) uint64 {
var result uint64
for _, current := range mergeRanges(ranges) {
result += uint64(current.end) - uint64(current.start) + 1
}
return result
}

32
packages/oci-l4-relay-publisher/ranges_test.go Normal file
View file

@ -0,0 +1,32 @@
package main
import "testing"
func mustRange(t *testing.T, prefix string) addressRange {
t.Helper()
result, ok := rangeFromPrefix(prefix)
if !ok {
t.Fatalf("invalid prefix %s", prefix)
}
return result
}
func TestRangeSetOperations(t *testing.T) {
left := []addressRange{mustRange(t, "10.63.81.107/24"), mustRange(t, "10.211.128.249/24")}
right := []addressRange{mustRange(t, "10.211.128.249/25"), mustRange(t, "10.104.47.60/24")}
merged := mergeRanges(left)
if len(merged) != 1 || addressCount(merged) != 512 {
t.Fatalf("unexpected merge: %#v", merged)
}
intersection := intersectRanges(left, right)
if len(intersection) != 1 || addressCount(intersection) != 128 {
t.Fatalf("unexpected intersection: %#v", intersection)
}
difference := subtractRanges(left, right)
if len(difference) != 2 || addressCount(difference) != 384 {
t.Fatalf("unexpected difference: %#v", difference)
}
}