ce7cb702218693d08ea6d83464d2457c9250027c
This commit is contained in:
commit
34c5f8963c
5 changed files with 145336 additions and 0 deletions
107
packages/oci-l4-relay-publisher/cmd/fetch-ip2proxy/main.go
Normal file
107
packages/oci-l4-relay-publisher/cmd/fetch-ip2proxy/main.go
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func downloadToken(path string) (string, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read IP2Location credential")
|
||||
}
|
||||
token := strings.TrimSpace(string(contents))
|
||||
if token == "" {
|
||||
return "", fmt.Errorf("IP2Location credential is empty")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func fetch(ctx context.Context, client *http.Client, token, code, output string) error {
|
||||
query := url.Values{"token": {token}, "file": {code}}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.ip2location.com/download?"+query.Encode(), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create IP2Proxy request")
|
||||
}
|
||||
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download IP2Proxy package")
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("download IP2Proxy package: HTTP %d", response.StatusCode)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(output), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
temporary, err := os.CreateTemp(filepath.Dir(output), ".ip2proxy-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryPath) }()
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
written, copyErr := io.Copy(temporary, io.LimitReader(response.Body, (512<<20)+1))
|
||||
closeErr := temporary.Close()
|
||||
if copyErr != nil {
|
||||
return copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if written > 512<<20 {
|
||||
return fmt.Errorf("IP2Proxy package exceeds size limit")
|
||||
}
|
||||
archive, err := zip.OpenReader(temporaryPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("validate IP2Proxy archive")
|
||||
}
|
||||
defer func() { _ = archive.Close() }()
|
||||
found := false
|
||||
for _, file := range archive.File {
|
||||
if strings.HasSuffix(file.Name, "IP2PROXY-LITE-PX12.CSV") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("IP2Proxy PX12 archive member is missing")
|
||||
}
|
||||
if err := os.Rename(temporaryPath, output); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("source=ip2proxy bytes=%d\n", written)
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
downloadTokenFile := flag.String("download-token-file", "/credentials/ip2location/download-token", "IP2Location download-token file")
|
||||
packageCode := flag.String("package-code", "PX12LITE", "entitled IP2Proxy package code")
|
||||
dataDirectory := flag.String("data-directory", "/work/data", "validated source cache")
|
||||
flag.Parse()
|
||||
token, err := downloadToken(*downloadTokenFile)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
client := &http.Client{Timeout: 15 * time.Minute}
|
||||
if err := fetch(ctx, client, token, *packageCode, filepath.Join(*dataDirectory, "ip2proxy.zip")); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
401
packages/oci-l4-relay-publisher/cmd/fetch/main.go
Normal file
401
packages/oci-l4-relay-publisher/cmd/fetch/main.go
Normal file
|
|
@ -0,0 +1,401 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maximumSourceBytes = 512 << 20
|
||||
|
||||
type source struct {
|
||||
name string
|
||||
url string
|
||||
minimumInterval time.Duration
|
||||
headBeforeGet bool
|
||||
redactURL bool
|
||||
}
|
||||
|
||||
type sourceState struct {
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
NotBefore time.Time `json:"not_before"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
type sourceResult struct {
|
||||
Name string `json:"name"`
|
||||
Changed bool `json:"changed"`
|
||||
State sourceState `json:"state"`
|
||||
}
|
||||
|
||||
func credential(path, name string) (string, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s credential", name)
|
||||
}
|
||||
value := strings.TrimSpace(string(contents))
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("%s credential is empty", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func azureURL(ctx context.Context, client *http.Client) (string, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetch Azure release page")
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("fetch Azure release page: HTTP %d", response.StatusCode)
|
||||
}
|
||||
contents, err := io.ReadAll(io.LimitReader(response.Body, 8<<20))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read Azure release page")
|
||||
}
|
||||
pattern := regexp.MustCompile(`https://download\.microsoft\.com/download/[^"']+ServiceTags_Public_[0-9]+\.json`)
|
||||
match := pattern.FindString(string(contents))
|
||||
if match == "" {
|
||||
return "", errors.New("discover Azure service tag release")
|
||||
}
|
||||
return strings.ReplaceAll(match, "&", "&"), nil
|
||||
}
|
||||
|
||||
func sourceCatalog(ctx context.Context, client *http.Client, selected map[string]bool, maxMindKeyPath string) (map[string]source, error) {
|
||||
catalog := map[string]source{
|
||||
"iplocate-country": {name: "iplocate-country.zip", url: "https://github.com/iplocate/ip-address-databases/raw/refs/heads/main/ip-to-country/ip-to-country.csv.zip", minimumInterval: 20 * time.Hour},
|
||||
"iplocate-asn": {name: "iplocate-asn.zip", url: "https://github.com/iplocate/ip-address-databases/raw/refs/heads/main/ip-to-asn/ip-to-asn.csv.zip", minimumInterval: 20 * time.Hour},
|
||||
"spamhaus-drop": {name: "spamhaus-drop.json", url: "https://www.spamhaus.org/drop/drop_v4.json", minimumInterval: 24 * time.Hour},
|
||||
"tor-exits": {name: "tor-exits.txt", url: "https://check.torproject.org/torbulkexitlist", minimumInterval: 6 * 24 * time.Hour},
|
||||
"aws": {name: "aws.json", url: "https://ip-ranges.amazonaws.com/ip-ranges.json", minimumInterval: 6 * 24 * time.Hour},
|
||||
"gcp": {name: "gcp.json", url: "https://www.gstatic.com/ipranges/cloud.json", minimumInterval: 6 * 24 * time.Hour},
|
||||
"oci": {name: "oci.json", url: "https://docs.oracle.com/iaas/tools/public_ip_ranges.json", minimumInterval: 24 * time.Hour},
|
||||
"cloudflare": {name: "cloudflare.txt", url: "https://www.cloudflare.com/ips-v4", minimumInterval: 6 * 24 * time.Hour},
|
||||
"fastly": {name: "fastly.json", url: "https://api.fastly.com/public-ip-list", minimumInterval: 6 * 24 * time.Hour},
|
||||
"github": {name: "github.json", url: "https://api.github.com/meta", minimumInterval: 6 * 24 * time.Hour},
|
||||
"atlassian": {name: "atlassian.json", url: "https://ip-ranges.atlassian.com/", minimumInterval: 6 * 24 * time.Hour},
|
||||
"digitalocean": {name: "digitalocean.csv", url: "https://digitalocean.com/geo/google.csv", minimumInterval: 6 * 24 * time.Hour},
|
||||
"linode": {name: "linode.csv", url: "https://geoip.linode.com/", minimumInterval: 6 * 24 * time.Hour},
|
||||
"vultr": {name: "vultr.csv", url: "https://geofeed.constant.com/?text", minimumInterval: 6 * 24 * time.Hour},
|
||||
}
|
||||
if selected["maxmind-country"] {
|
||||
key, err := credential(maxMindKeyPath, "MaxMind")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
query := url.Values{"edition_id": {"GeoLite2-Country-CSV"}, "license_key": {key}, "suffix": {"zip"}}
|
||||
catalog["maxmind-country"] = source{name: "maxmind-country.zip", url: "https://download.maxmind.com/app/geoip_download?" + query.Encode(), minimumInterval: 12 * time.Hour, headBeforeGet: true, redactURL: true}
|
||||
}
|
||||
if selected["azure"] {
|
||||
downloadURL, err := azureURL(ctx, client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
catalog["azure"] = source{name: "azure.json", url: downloadURL, minimumInterval: 6 * 24 * time.Hour}
|
||||
}
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
func readState(path string) (sourceState, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return sourceState{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return sourceState{}, err
|
||||
}
|
||||
var state sourceState
|
||||
if err := json.Unmarshal(contents, &state); err != nil {
|
||||
return sourceState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func writeState(path string, state sourceState) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
contents, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporary, err := os.CreateTemp(filepath.Dir(path), ".state-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryPath) }()
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(append(contents, '\n')); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temporaryPath, path)
|
||||
}
|
||||
|
||||
func writeJSON(path string, value any) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
contents, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(contents, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func retryAfter(response *http.Response, now time.Time) time.Time {
|
||||
value := response.Header.Get("Retry-After")
|
||||
if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 {
|
||||
return now.Add(time.Duration(seconds) * time.Second)
|
||||
}
|
||||
if parsed, err := http.ParseTime(value); err == nil && parsed.After(now) {
|
||||
return parsed
|
||||
}
|
||||
return now.Add(time.Hour)
|
||||
}
|
||||
|
||||
func request(ctx context.Context, client *http.Client, method string, item source, state sourceState) (*http.Response, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, method, item.url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create %s request", item.name)
|
||||
}
|
||||
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
|
||||
if state.ETag != "" {
|
||||
request.Header.Set("If-None-Match", state.ETag)
|
||||
}
|
||||
if state.LastModified != "" {
|
||||
request.Header.Set("If-Modified-Since", state.LastModified)
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
if !item.redactURL {
|
||||
return nil, fmt.Errorf("fetch %s: %w", item.name, err)
|
||||
}
|
||||
return nil, fmt.Errorf("fetch %s", item.name)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func fetchSource(ctx context.Context, client *http.Client, key string, item source, dataDirectory, stateDirectory string, now time.Time) (bool, error) {
|
||||
statePath := filepath.Join(stateDirectory, key+".json")
|
||||
state, err := readState(statePath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read %s state", key)
|
||||
}
|
||||
dataPath := filepath.Join(dataDirectory, item.name)
|
||||
if now.Before(state.NotBefore) {
|
||||
if _, err := os.Stat(dataPath); err != nil || state.SHA256 == "" {
|
||||
return false, fmt.Errorf("%s is not due and has no cached generation", key)
|
||||
}
|
||||
fmt.Printf("source=%s due=false\n", key)
|
||||
return false, nil
|
||||
}
|
||||
method := http.MethodGet
|
||||
if item.headBeforeGet && state.SHA256 != "" {
|
||||
method = http.MethodHead
|
||||
}
|
||||
response, err := request(ctx, client, method, item, state)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode == http.StatusTooManyRequests || response.StatusCode >= 500 {
|
||||
state.NotBefore = retryAfter(response, now)
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, fmt.Errorf("fetch %s: HTTP %d", key, response.StatusCode)
|
||||
}
|
||||
if response.StatusCode == http.StatusNotModified {
|
||||
state.CheckedAt = now
|
||||
state.NotBefore = now.Add(item.minimumInterval)
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
fmt.Printf("source=%s changed=false\n", key)
|
||||
return false, nil
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return false, fmt.Errorf("fetch %s: HTTP %d", key, response.StatusCode)
|
||||
}
|
||||
if method == http.MethodHead {
|
||||
unchanged := response.Header.Get("ETag") == state.ETag && response.Header.Get("Last-Modified") == state.LastModified
|
||||
if unchanged {
|
||||
state.CheckedAt = now
|
||||
state.NotBefore = now.Add(item.minimumInterval)
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
fmt.Printf("source=%s changed=false\n", key)
|
||||
return false, nil
|
||||
}
|
||||
_ = response.Body.Close()
|
||||
response, err = request(ctx, client, http.MethodGet, item, sourceState{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return false, fmt.Errorf("download %s: HTTP %d", key, response.StatusCode)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(dataDirectory, 0o700); err != nil {
|
||||
return false, err
|
||||
}
|
||||
temporary, err := os.CreateTemp(dataDirectory, ".download-*")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryPath) }()
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return false, err
|
||||
}
|
||||
hash := sha256.New()
|
||||
written, copyErr := io.Copy(io.MultiWriter(temporary, hash), io.LimitReader(response.Body, maximumSourceBytes+1))
|
||||
closeErr := temporary.Close()
|
||||
if copyErr != nil {
|
||||
return false, copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return false, closeErr
|
||||
}
|
||||
if written > maximumSourceBytes {
|
||||
return false, fmt.Errorf("%s exceeds size limit", key)
|
||||
}
|
||||
digest := hex.EncodeToString(hash.Sum(nil))
|
||||
changed := digest != state.SHA256
|
||||
if changed {
|
||||
if err := os.Rename(temporaryPath, dataPath); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
state = sourceState{
|
||||
CheckedAt: now,
|
||||
UpdatedAt: state.UpdatedAt,
|
||||
NotBefore: now.Add(item.minimumInterval),
|
||||
ETag: response.Header.Get("ETag"),
|
||||
LastModified: response.Header.Get("Last-Modified"),
|
||||
SHA256: digest,
|
||||
Bytes: written,
|
||||
}
|
||||
if changed {
|
||||
state.UpdatedAt = now
|
||||
}
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
fmt.Printf("source=%s changed=%t bytes=%d sha256=%s\n", key, changed, written, digest)
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func selectedSources(value string) (map[string]bool, []string, error) {
|
||||
selected := make(map[string]bool)
|
||||
var order []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item == "" || selected[item] {
|
||||
continue
|
||||
}
|
||||
selected[item] = true
|
||||
order = append(order, item)
|
||||
}
|
||||
if len(order) == 0 {
|
||||
return nil, nil, errors.New("at least one source is required")
|
||||
}
|
||||
return selected, order, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
sourceNames := flag.String("sources", "", "comma-separated source names")
|
||||
dataDirectory := flag.String("data-directory", "/work/data", "validated source cache")
|
||||
stateDirectory := flag.String("state-directory", "/work/state", "source fetch state")
|
||||
changedFile := flag.String("changed-file", "/work/fetch-changed", "aggregate changed result")
|
||||
summaryFile := flag.String("summary-file", "/work/fetch-summary.json", "sanitized fetch summary")
|
||||
maxMindKeyPath := flag.String("maxmind-license-key-file", "/credentials/maxmind/license-key", "MaxMind license-key file")
|
||||
flag.Parse()
|
||||
selected, order, err := selectedSources(*sourceNames)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
client := &http.Client{Timeout: 15 * time.Minute}
|
||||
catalog, err := sourceCatalog(ctx, client, selected, *maxMindKeyPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
changed := false
|
||||
results := make([]sourceResult, 0, len(order))
|
||||
if contents, err := os.ReadFile(*changedFile); err == nil {
|
||||
changed, _ = strconv.ParseBool(strings.TrimSpace(string(contents)))
|
||||
}
|
||||
for _, name := range order {
|
||||
item, ok := catalog[name]
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown source %q\n", name)
|
||||
os.Exit(2)
|
||||
}
|
||||
currentChanged, err := fetchSource(ctx, client, name, item, *dataDirectory, *stateDirectory, now)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
changed = changed || currentChanged
|
||||
state, err := readState(filepath.Join(*stateDirectory, name+".json"))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
results = append(results, sourceResult{Name: name, Changed: currentChanged, State: state})
|
||||
if err := os.MkdirAll(filepath.Dir(*changedFile), 0o755); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(*changedFile, []byte(strconv.FormatBool(changed)+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(*changedFile, []byte(strconv.FormatBool(changed)+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := writeJSON(*summaryFile, results); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
89
packages/oci-l4-relay-publisher/cmd/fetch/main_test.go
Normal file
89
packages/oci-l4-relay-publisher/cmd/fetch/main_test.go
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (function roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return function(request)
|
||||
}
|
||||
|
||||
func TestFetchSourceUsesValidatorsAndNotBefore(t *testing.T) {
|
||||
requests := 0
|
||||
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
requests++
|
||||
if request.Header.Get("If-None-Match") == `"generation-1"` {
|
||||
return &http.Response{StatusCode: http.StatusNotModified, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(""))}, nil
|
||||
}
|
||||
header := make(http.Header)
|
||||
header.Set("ETag", `"generation-1"`)
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: header, Body: io.NopCloser(strings.NewReader("source generation"))}, nil
|
||||
})}
|
||||
directory := t.TempDir()
|
||||
dataDirectory := filepath.Join(directory, "data")
|
||||
stateDirectory := filepath.Join(directory, "state")
|
||||
item := source{name: "source.txt", url: "https://source.example.invalid/feed", minimumInterval: time.Hour}
|
||||
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
|
||||
changed, err := fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("initial fetch changed=%t err=%v", changed, err)
|
||||
}
|
||||
changed, err = fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now.Add(30*time.Minute))
|
||||
if err != nil || changed {
|
||||
t.Fatalf("not-before fetch changed=%t err=%v", changed, err)
|
||||
}
|
||||
changed, err = fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now.Add(2*time.Hour))
|
||||
if err != nil || changed {
|
||||
t.Fatalf("conditional fetch changed=%t err=%v", changed, err)
|
||||
}
|
||||
if requests != 2 {
|
||||
t.Fatalf("requests=%d, want 2", requests)
|
||||
}
|
||||
contents, err := os.ReadFile(filepath.Join(dataDirectory, "source.txt"))
|
||||
if err != nil || string(contents) != "source generation" {
|
||||
t.Fatalf("cached source=%q err=%v", contents, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchSourcePersistsRetryFloor(t *testing.T) {
|
||||
requests := 0
|
||||
client := &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
requests++
|
||||
header := make(http.Header)
|
||||
header.Set("Retry-After", "3600")
|
||||
return &http.Response{StatusCode: http.StatusTooManyRequests, Header: header, Body: io.NopCloser(strings.NewReader(""))}, nil
|
||||
})}
|
||||
directory := t.TempDir()
|
||||
dataDirectory := filepath.Join(directory, "data")
|
||||
stateDirectory := filepath.Join(directory, "state")
|
||||
if err := os.MkdirAll(dataDirectory, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dataDirectory, "source.txt"), []byte("cached"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writeState(filepath.Join(stateDirectory, "source.json"), sourceState{SHA256: "cached-generation"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := source{name: "source.txt", url: "https://source.example.invalid/feed", minimumInterval: time.Hour}
|
||||
now := time.Date(2026, 7, 12, 0, 0, 0, 0, time.UTC)
|
||||
if _, err := fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now); err == nil {
|
||||
t.Fatal("expected rate-limit error")
|
||||
}
|
||||
changed, err := fetchSource(context.Background(), client, "source", item, dataDirectory, stateDirectory, now.Add(time.Minute))
|
||||
if err != nil || changed {
|
||||
t.Fatalf("retry-floor fetch changed=%t err=%v", changed, err)
|
||||
}
|
||||
if requests != 1 {
|
||||
t.Fatalf("requests=%d, want 1", requests)
|
||||
}
|
||||
}
|
||||
445
packages/oci-l4-relay-publisher/cmd/publish/main.go
Normal file
445
packages/oci-l4-relay-publisher/cmd/publish/main.go
Normal file
|
|
@ -0,0 +1,445 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maximumBundleBytes = 64 << 20
|
||||
|
||||
var objectPrefixPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*/v[1-9][0-9]*$`)
|
||||
|
||||
type credentials struct {
|
||||
tenancy string
|
||||
user string
|
||||
fingerprint string
|
||||
privateKey *rsa.PrivateKey
|
||||
}
|
||||
|
||||
type stringValues []string
|
||||
|
||||
func (values *stringValues) String() string {
|
||||
return strings.Join(*values, ",")
|
||||
}
|
||||
|
||||
func (values *stringValues) Set(value string) error {
|
||||
*values = append(*values, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
type publication struct {
|
||||
Prefix string `json:"prefix"`
|
||||
VersionObject string `json:"version_object"`
|
||||
VersionCreated bool `json:"version_created"`
|
||||
CurrentObject string `json:"current_object"`
|
||||
CurrentPromoted bool `json:"current_promoted"`
|
||||
}
|
||||
|
||||
type smtpDenySummary struct {
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
DeniedIntervalCount int `json:"denied_interval_count"`
|
||||
DeniedAddressCount uint64 `json:"denied_address_count"`
|
||||
RemovedIntervalCount int `json:"removed_interval_count"`
|
||||
}
|
||||
|
||||
func readSMTPDenySummary(path string) (*smtpDenySummary, error) {
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
decoder := json.NewDecoder(io.LimitReader(file, 1<<20))
|
||||
decoder.DisallowUnknownFields()
|
||||
summary := smtpDenySummary{}
|
||||
if err := decoder.Decode(&summary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := decoder.Decode(&struct{}{}); err != io.EOF {
|
||||
return nil, errors.New("SMTP deny summary contains trailing data")
|
||||
}
|
||||
if summary.GeneratedAt.IsZero() ||
|
||||
!summary.ExpiresAt.After(summary.GeneratedAt) ||
|
||||
summary.DeniedIntervalCount < 0 ||
|
||||
summary.RemovedIntervalCount < 0 {
|
||||
return nil, errors.New("SMTP deny summary is invalid")
|
||||
}
|
||||
return &summary, nil
|
||||
}
|
||||
|
||||
func publicationPrefixes(primary string, additional []string) ([]string, error) {
|
||||
values := append([]string{primary}, additional...)
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
result := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
if !objectPrefixPattern.MatchString(value) {
|
||||
return nil, fmt.Errorf("invalid object prefix %q", value)
|
||||
}
|
||||
if _, exists := seen[value]; exists {
|
||||
return nil, fmt.Errorf("duplicate object prefix %q", value)
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
result = append(result, value)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func publicationObjects(prefix, digest string) (string, string) {
|
||||
return prefix + "/versions/" + digest + ".json.gz", prefix + "/current.json.gz"
|
||||
}
|
||||
|
||||
func publishBundle(ctx context.Context, client *http.Client, endpoint, namespace, bucket string, prefixes []string, body []byte, identity credentials, promoteCurrent bool) (string, []publication, error) {
|
||||
digest := sha256.Sum256(body)
|
||||
digestText := hex.EncodeToString(digest[:])
|
||||
publications := make([]publication, 0, len(prefixes))
|
||||
for _, prefix := range prefixes {
|
||||
versionObject, currentObject := publicationObjects(prefix, digestText)
|
||||
created, err := upload(ctx, client, endpoint, namespace, bucket, versionObject, body, identity, true)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := verifyObject(ctx, client, endpoint, namespace, bucket, versionObject, body, identity); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if promoteCurrent {
|
||||
if _, err := upload(ctx, client, endpoint, namespace, bucket, currentObject, body, identity, false); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if err := verifyObject(ctx, client, endpoint, namespace, bucket, currentObject, body, identity); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
publications = append(publications, publication{
|
||||
Prefix: prefix,
|
||||
VersionObject: versionObject,
|
||||
VersionCreated: created,
|
||||
CurrentObject: currentObject,
|
||||
CurrentPromoted: promoteCurrent,
|
||||
})
|
||||
}
|
||||
return digestText, publications, nil
|
||||
}
|
||||
|
||||
func writeJSONFile(path string, value any) error {
|
||||
contents, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(contents, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func readCredential(directory, name string) (string, error) {
|
||||
contents, err := os.ReadFile(filepath.Join(directory, name))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read OCI %s", strings.ReplaceAll(name, "_", " "))
|
||||
}
|
||||
value := strings.TrimSpace(string(contents))
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("OCI %s is empty", strings.ReplaceAll(name, "_", " "))
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func optionalCredential(directory, name, fallback string) string {
|
||||
value, err := readCredential(directory, name)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func parsePrivateKey(value string) (*rsa.PrivateKey, error) {
|
||||
block, _ := pem.Decode([]byte(value))
|
||||
if block == nil {
|
||||
return nil, errors.New("decode OCI private key")
|
||||
}
|
||||
if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil {
|
||||
return key, nil
|
||||
}
|
||||
parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, errors.New("parse OCI private key")
|
||||
}
|
||||
key, ok := parsed.(*rsa.PrivateKey)
|
||||
if !ok {
|
||||
return nil, errors.New("OCI private key is not RSA")
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func readCredentials(directory string) (credentials, error) {
|
||||
tenancy, err := readCredential(directory, "tenancy_ocid")
|
||||
if err != nil {
|
||||
return credentials{}, err
|
||||
}
|
||||
user, err := readCredential(directory, "user_ocid")
|
||||
if err != nil {
|
||||
return credentials{}, err
|
||||
}
|
||||
fingerprint, err := readCredential(directory, "fingerprint")
|
||||
if err != nil {
|
||||
return credentials{}, err
|
||||
}
|
||||
privateKeyPEM, err := readCredential(directory, "private_key")
|
||||
if err != nil {
|
||||
return credentials{}, err
|
||||
}
|
||||
privateKey, err := parsePrivateKey(privateKeyPEM)
|
||||
if err != nil {
|
||||
return credentials{}, err
|
||||
}
|
||||
return credentials{tenancy: tenancy, user: user, fingerprint: fingerprint, privateKey: privateKey}, nil
|
||||
}
|
||||
|
||||
func objectURL(endpoint, namespace, bucket, object string) (string, error) {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if parsed.Scheme != "https" || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" {
|
||||
return "", errors.New("invalid OCI Object Storage endpoint")
|
||||
}
|
||||
parsed.Path = "/n/" + namespace + "/b/" + bucket + "/o/" + object
|
||||
parsed.RawPath = "/n/" + url.PathEscape(namespace) + "/b/" + url.PathEscape(bucket) + "/o/" + url.PathEscape(object)
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
func signRequest(request *http.Request, body []byte, identity credentials, now time.Time) error {
|
||||
digest := sha256.Sum256(body)
|
||||
request.Header.Set("Date", now.UTC().Format(http.TimeFormat))
|
||||
request.Header.Set("Content-Type", "application/gzip")
|
||||
request.Header.Set("Content-Length", strconv.Itoa(len(body)))
|
||||
request.Header.Set("X-Content-SHA256", base64.StdEncoding.EncodeToString(digest[:]))
|
||||
request.ContentLength = int64(len(body))
|
||||
headerNames := "(request-target) host date x-content-sha256 content-type content-length"
|
||||
signingString := strings.Join([]string{
|
||||
"(request-target): " + strings.ToLower(request.Method) + " " + request.URL.EscapedPath(),
|
||||
"host: " + request.URL.Host,
|
||||
"date: " + request.Header.Get("Date"),
|
||||
"x-content-sha256: " + request.Header.Get("X-Content-SHA256"),
|
||||
"content-type: " + request.Header.Get("Content-Type"),
|
||||
"content-length: " + request.Header.Get("Content-Length"),
|
||||
}, "\n")
|
||||
signingDigest := sha256.Sum256([]byte(signingString))
|
||||
signature, err := rsa.SignPKCS1v15(rand.Reader, identity.privateKey, crypto.SHA256, signingDigest[:])
|
||||
if err != nil {
|
||||
return errors.New("sign OCI request")
|
||||
}
|
||||
keyID := identity.tenancy + "/" + identity.user + "/" + identity.fingerprint
|
||||
request.Header.Set("Authorization", fmt.Sprintf(`Signature version="1",keyId="%s",algorithm="rsa-sha256",headers="%s",signature="%s"`, keyID, headerNames, base64.StdEncoding.EncodeToString(signature)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func signDownloadRequest(request *http.Request, identity credentials, now time.Time) error {
|
||||
request.Header.Set("Date", now.UTC().Format(http.TimeFormat))
|
||||
headerNames := "(request-target) host date"
|
||||
signingString := strings.Join([]string{
|
||||
"(request-target): " + strings.ToLower(request.Method) + " " + request.URL.EscapedPath(),
|
||||
"host: " + request.URL.Host,
|
||||
"date: " + request.Header.Get("Date"),
|
||||
}, "\n")
|
||||
signingDigest := sha256.Sum256([]byte(signingString))
|
||||
signature, err := rsa.SignPKCS1v15(rand.Reader, identity.privateKey, crypto.SHA256, signingDigest[:])
|
||||
if err != nil {
|
||||
return errors.New("sign OCI request")
|
||||
}
|
||||
keyID := identity.tenancy + "/" + identity.user + "/" + identity.fingerprint
|
||||
request.Header.Set("Authorization", fmt.Sprintf(`Signature version="1",keyId="%s",algorithm="rsa-sha256",headers="%s",signature="%s"`, keyID, headerNames, base64.StdEncoding.EncodeToString(signature)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func upload(ctx context.Context, client *http.Client, endpoint, namespace, bucket, object string, body []byte, identity credentials, immutable bool) (bool, error) {
|
||||
target, err := objectURL(endpoint, namespace, bucket, object)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut, target, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, errors.New("create OCI upload request")
|
||||
}
|
||||
if immutable {
|
||||
request.Header.Set("If-None-Match", "*")
|
||||
}
|
||||
if err := signRequest(request, body, identity, time.Now()); err != nil {
|
||||
return false, err
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return false, errors.New("upload OCI object")
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
||||
if immutable && response.StatusCode == http.StatusPreconditionFailed {
|
||||
return false, nil
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return false, fmt.Errorf("upload OCI object: HTTP %d", response.StatusCode)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func download(ctx context.Context, client *http.Client, endpoint, namespace, bucket, object string, identity credentials) ([]byte, error) {
|
||||
target, err := objectURL(endpoint, namespace, bucket, object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
|
||||
if err != nil {
|
||||
return nil, errors.New("create OCI download request")
|
||||
}
|
||||
if err := signDownloadRequest(request, identity, time.Now()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, errors.New("download OCI object")
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
|
||||
return nil, fmt.Errorf("download OCI object: HTTP %d", response.StatusCode)
|
||||
}
|
||||
contents, err := io.ReadAll(io.LimitReader(response.Body, maximumBundleBytes+1))
|
||||
if err != nil {
|
||||
return nil, errors.New("read OCI object")
|
||||
}
|
||||
if len(contents) > maximumBundleBytes {
|
||||
return nil, errors.New("OCI object exceeds maximum bundle size")
|
||||
}
|
||||
return contents, nil
|
||||
}
|
||||
|
||||
func verifyObject(ctx context.Context, client *http.Client, endpoint, namespace, bucket, object string, expected []byte, identity credentials) error {
|
||||
contents, err := download(ctx, client, endpoint, namespace, bucket, object, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expectedDigest := sha256.Sum256(expected)
|
||||
actualDigest := sha256.Sum256(contents)
|
||||
if actualDigest != expectedDigest {
|
||||
return fmt.Errorf("verify OCI object %q: SHA-256 mismatch", object)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
bundlePath := flag.String("bundle", "/work/output/current.json.gz", "validated admission bundle")
|
||||
credentialsDirectory := flag.String("credentials-directory", "/credentials/oci", "OCI credential directory")
|
||||
region := flag.String("region", "", "OCI region")
|
||||
namespace := flag.String("namespace", "", "OCI Object Storage namespace")
|
||||
bucket := flag.String("bucket", "", "OCI Object Storage bucket")
|
||||
objectPrefix := flag.String("object-prefix", "admission/v1", "Object Storage prefix for versions and current")
|
||||
additionalObjectPrefixes := stringValues{}
|
||||
flag.Var(&additionalObjectPrefixes, "additional-object-prefix", "additional Object Storage prefix for the same bundle")
|
||||
promoteCurrent := flag.Bool("promote-current", true, "promote the validated bundle after immutable upload")
|
||||
endpoint := flag.String("endpoint", "", "OCI Object Storage endpoint")
|
||||
resultFile := flag.String("result-file", "", "write sanitized publication result")
|
||||
smtpDenySummaryPath := flag.String("smtp-deny-summary", "", "validated SMTP deny summary")
|
||||
restoreVersionSHA256 := flag.String("restore-version-sha256", "", "download an immutable version by SHA-256")
|
||||
restoreOutput := flag.String("restore-output", "/work/output/restore.json.gz", "write the immutable version for validation")
|
||||
flag.Parse()
|
||||
*region = optionalCredential(*credentialsDirectory, "region", *region)
|
||||
*namespace = optionalCredential(*credentialsDirectory, "object_storage_namespace", *namespace)
|
||||
*bucket = optionalCredential(*credentialsDirectory, "bucket_name", *bucket)
|
||||
if *region == "" || *namespace == "" || *bucket == "" {
|
||||
fmt.Fprintln(os.Stderr, "region, namespace, and bucket are required")
|
||||
os.Exit(2)
|
||||
}
|
||||
if *endpoint == "" {
|
||||
*endpoint = "https://objectstorage." + *region + ".oraclecloud.com"
|
||||
}
|
||||
prefixes, err := publicationPrefixes(*objectPrefix, additionalObjectPrefixes)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
smtpSummary, err := readSMTPDenySummary(*smtpDenySummaryPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
identity, err := readCredentials(*credentialsDirectory)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
client := &http.Client{Timeout: 2 * time.Minute}
|
||||
if *restoreVersionSHA256 != "" {
|
||||
if !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(*restoreVersionSHA256) {
|
||||
fmt.Fprintln(os.Stderr, "restore version SHA-256 must be 64 lowercase hexadecimal characters")
|
||||
os.Exit(2)
|
||||
}
|
||||
versionObject, _ := publicationObjects(prefixes[0], *restoreVersionSHA256)
|
||||
bundle, err := download(ctx, client, *endpoint, *namespace, *bucket, versionObject, identity)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
digest := sha256.Sum256(bundle)
|
||||
if hex.EncodeToString(digest[:]) != *restoreVersionSHA256 {
|
||||
fmt.Fprintln(os.Stderr, "downloaded immutable version does not match requested SHA-256")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(*restoreOutput), 0o700); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "create restore output directory")
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(*restoreOutput, bundle, 0o600); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write restored immutable version")
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("restored_bundle_sha256=%s\n", *restoreVersionSHA256)
|
||||
return
|
||||
}
|
||||
bundle, err := os.ReadFile(*bundlePath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "read validated admission bundle")
|
||||
os.Exit(1)
|
||||
}
|
||||
digestText, publications, err := publishBundle(ctx, client, *endpoint, *namespace, *bucket, prefixes, bundle, identity, *promoteCurrent)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *resultFile != "" {
|
||||
result := struct {
|
||||
BundleSHA256 string `json:"bundle_sha256"`
|
||||
Publications []publication `json:"publications"`
|
||||
SMTPDenySummary *smtpDenySummary `json:"smtp_deny_summary,omitempty"`
|
||||
}{BundleSHA256: digestText, Publications: publications, SMTPDenySummary: smtpSummary}
|
||||
if err := writeJSONFile(*resultFile, result); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
fmt.Printf("bundle_sha256=%s publication_count=%d current_promoted=%t\n", digestText, len(publications), *promoteCurrent)
|
||||
}
|
||||
358
packages/oci-l4-relay-publisher/cmd/publish/main_test.go
Normal file
358
packages/oci-l4-relay-publisher/cmd/publish/main_test.go
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWriteJSONFileCreatesParentDirectory(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "workflow", "publication-result.json")
|
||||
if err := writeJSONFile(path, map[string]bool{"published": true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(contents) != "{\"published\":true}\n" {
|
||||
t.Fatalf("unexpected contents %q", contents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectURL(t *testing.T) {
|
||||
result, err := objectURL("https://objectstorage.example.invalid", "namespace", "bucket", "admission/v1/current.json.gz")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result != "https://objectstorage.example.invalid/n/namespace/b/bucket/o/admission%2Fv1%2Fcurrent.json.gz" {
|
||||
t.Fatalf("unexpected object URL %q", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicationPrefixes(t *testing.T) {
|
||||
prefixes, err := publicationPrefixes("admission/v1", []string{"web-allow/v1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(prefixes, ",") != "admission/v1,web-allow/v1" {
|
||||
t.Fatalf("unexpected prefixes %q", prefixes)
|
||||
}
|
||||
version, current := publicationObjects(prefixes[1], strings.Repeat("a", 64))
|
||||
if version != "web-allow/v1/versions/"+strings.Repeat("a", 64)+".json.gz" {
|
||||
t.Fatalf("unexpected version object %q", version)
|
||||
}
|
||||
if current != "web-allow/v1/current.json.gz" {
|
||||
t.Fatalf("unexpected current object %q", current)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicationPrefixesRejectInvalidOrDuplicateValues(t *testing.T) {
|
||||
for name, values := range map[string][]string{
|
||||
"empty": {""},
|
||||
"traversal": {"../web-allow/v1"},
|
||||
"nested": {"mail/smtp-deny/v1"},
|
||||
"duplicate": {"web-allow/v1", "web-allow/v1"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, err := publicationPrefixes(values[0], values[1:]); err == nil {
|
||||
t.Fatal("expected prefix validation failure")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishBundleUsesIdenticalContentForBothPrefixes(t *testing.T) {
|
||||
type uploadRequest struct {
|
||||
body string
|
||||
immutable bool
|
||||
escapedPath string
|
||||
}
|
||||
requests := make([]uploadRequest, 0, 4)
|
||||
objects := map[string][]byte{}
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method == http.MethodGet {
|
||||
body, exists := objects[request.URL.EscapedPath()]
|
||||
if !exists {
|
||||
response.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_, _ = response.Write(body)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
objects[request.URL.EscapedPath()] = body
|
||||
requests = append(requests, uploadRequest{
|
||||
body: string(body),
|
||||
immutable: request.Header.Get("If-None-Match") == "*",
|
||||
escapedPath: request.URL.EscapedPath(),
|
||||
})
|
||||
response.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("validated bundle")
|
||||
digest, publications, err := publishBundle(
|
||||
context.Background(),
|
||||
server.Client(),
|
||||
server.URL,
|
||||
"namespace",
|
||||
"bucket",
|
||||
[]string{"admission/v1", "web-allow/v1"},
|
||||
body,
|
||||
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
|
||||
true,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(publications) != 2 || len(requests) != 4 {
|
||||
t.Fatalf("publications=%d requests=%d", len(publications), len(requests))
|
||||
}
|
||||
expectedPaths := []string{
|
||||
"/n/namespace/b/bucket/o/admission%2Fv1%2Fversions%2F" + digest + ".json.gz",
|
||||
"/n/namespace/b/bucket/o/admission%2Fv1%2Fcurrent.json.gz",
|
||||
"/n/namespace/b/bucket/o/web-allow%2Fv1%2Fversions%2F" + digest + ".json.gz",
|
||||
"/n/namespace/b/bucket/o/web-allow%2Fv1%2Fcurrent.json.gz",
|
||||
}
|
||||
for index, request := range requests {
|
||||
if request.body != string(body) {
|
||||
t.Errorf("request %d changed bundle content", index)
|
||||
}
|
||||
if request.escapedPath != expectedPaths[index] {
|
||||
t.Errorf("request %d path %q", index, request.escapedPath)
|
||||
}
|
||||
if request.immutable != (index%2 == 0) {
|
||||
t.Errorf("request %d immutable=%t", index, request.immutable)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishBundleVerifiesExistingImmutableObject(t *testing.T) {
|
||||
body := []byte("validated bundle")
|
||||
digest := sha256.Sum256(body)
|
||||
versionPath := "/n/namespace/b/bucket/o/smtp-deny%2Fv1%2Fversions%2F" + fmt.Sprintf("%x", digest) + ".json.gz"
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method == http.MethodPut {
|
||||
response.WriteHeader(http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
if request.URL.EscapedPath() != versionPath {
|
||||
t.Errorf("unexpected path %q", request.URL.EscapedPath())
|
||||
}
|
||||
_, _ = response.Write(body)
|
||||
}))
|
||||
defer server.Close()
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, publications, err := publishBundle(
|
||||
context.Background(),
|
||||
server.Client(),
|
||||
server.URL,
|
||||
"namespace",
|
||||
"bucket",
|
||||
[]string{"smtp-deny/v1"},
|
||||
body,
|
||||
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
|
||||
false,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(publications) != 1 || publications[0].VersionCreated {
|
||||
t.Fatalf("unexpected publications %#v", publications)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishBundleRejectsMismatchedExistingImmutableObject(t *testing.T) {
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
if request.Method == http.MethodPut {
|
||||
response.WriteHeader(http.StatusPreconditionFailed)
|
||||
return
|
||||
}
|
||||
_, _ = response.Write([]byte("different bundle"))
|
||||
}))
|
||||
defer server.Close()
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = publishBundle(
|
||||
context.Background(),
|
||||
server.Client(),
|
||||
server.URL,
|
||||
"namespace",
|
||||
"bucket",
|
||||
[]string{"smtp-deny/v1"},
|
||||
[]byte("validated bundle"),
|
||||
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
|
||||
false,
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "SHA-256 mismatch") {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishBundleRejectsMismatchedCurrentObject(t *testing.T) {
|
||||
objects := map[string][]byte{}
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
path := request.URL.EscapedPath()
|
||||
if request.Method == http.MethodPut {
|
||||
body, err := io.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
objects[path] = body
|
||||
response.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if strings.HasSuffix(path, "current.json.gz") {
|
||||
_, _ = response.Write([]byte("different bundle"))
|
||||
return
|
||||
}
|
||||
_, _ = response.Write(objects[path])
|
||||
}))
|
||||
defer server.Close()
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err = publishBundle(
|
||||
context.Background(),
|
||||
server.Client(),
|
||||
server.URL,
|
||||
"namespace",
|
||||
"bucket",
|
||||
[]string{"smtp-deny/v1"},
|
||||
[]byte("validated bundle"),
|
||||
credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key},
|
||||
true,
|
||||
)
|
||||
if err == nil || !strings.Contains(err.Error(), "current.json.gz") || !strings.Contains(err.Error(), "SHA-256 mismatch") {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSMTPDenySummary(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "summary.json")
|
||||
contents := `{"generated_at":"2026-07-26T00:00:00Z","expires_at":"2026-07-27T00:00:00Z","denied_interval_count":2,"denied_address_count":17,"removed_interval_count":1}`
|
||||
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
summary, err := readSMTPDenySummary(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if summary.DeniedIntervalCount != 2 || summary.DeniedAddressCount != 17 || summary.RemovedIntervalCount != 1 {
|
||||
t.Fatalf("unexpected summary %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSMTPDenySummaryRejectsContentEvidence(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "summary.json")
|
||||
contents := `{"generated_at":"2026-07-26T00:00:00Z","expires_at":"2026-07-27T00:00:00Z","denied_interval_count":1,"denied_address_count":1,"removed_interval_count":0,"recipient":"[email protected]"}`
|
||||
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := readSMTPDenySummary(path); err == nil {
|
||||
t.Fatal("expected content-bearing summary rejection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignRequest(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := []byte("bundle")
|
||||
request, err := http.NewRequest(http.MethodPut, "https://objectstorage.example.invalid/n/ns/b/b/o/current.json.gz", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
identity := credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key}
|
||||
if err := signRequest(request, body, identity, time.Unix(0, 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
authorization := request.Header.Get("Authorization")
|
||||
marker := `signature="`
|
||||
start := strings.Index(authorization, marker)
|
||||
if start < 0 {
|
||||
t.Fatal("missing signature")
|
||||
}
|
||||
encoded := strings.TrimSuffix(authorization[start+len(marker):], `"`)
|
||||
signature, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signingString := strings.Join([]string{
|
||||
"(request-target): put /n/ns/b/b/o/current.json.gz",
|
||||
"host: objectstorage.example.invalid",
|
||||
"date: " + request.Header.Get("Date"),
|
||||
"x-content-sha256: " + request.Header.Get("X-Content-SHA256"),
|
||||
"content-type: application/gzip",
|
||||
"content-length: 6",
|
||||
}, "\n")
|
||||
digest := sha256.Sum256([]byte(signingString))
|
||||
if err := rsa.VerifyPKCS1v15(&key.PublicKey, 0, digest[:], signature); err == nil {
|
||||
t.Fatal("signature unexpectedly verified without SHA-256")
|
||||
}
|
||||
if err := rsa.VerifyPKCS1v15(&key.PublicKey, crypto.SHA256, digest[:], signature); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignDownloadRequest(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request, err := http.NewRequest(http.MethodGet, "https://objectstorage.example.invalid/n/ns/b/b/o/admission%2Fv1%2Fversions%2Fbundle.json.gz", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
identity := credentials{tenancy: "tenancy", user: "user", fingerprint: "fingerprint", privateKey: key}
|
||||
if err := signDownloadRequest(request, identity, time.Unix(0, 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
authorization := request.Header.Get("Authorization")
|
||||
marker := `signature="`
|
||||
start := strings.Index(authorization, marker)
|
||||
if start < 0 {
|
||||
t.Fatal("missing signature")
|
||||
}
|
||||
encoded := strings.TrimSuffix(authorization[start+len(marker):], `"`)
|
||||
signature, err := base64.StdEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signingString := strings.Join([]string{
|
||||
"(request-target): get /n/ns/b/b/o/admission%2Fv1%2Fversions%2Fbundle.json.gz",
|
||||
"host: objectstorage.example.invalid",
|
||||
"date: " + request.Header.Get("Date"),
|
||||
}, "\n")
|
||||
digest := sha256.Sum256([]byte(signingString))
|
||||
if err := rsa.VerifyPKCS1v15(&key.PublicKey, crypto.SHA256, digest[:], signature); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue