ce7cb702218693d08ea6d83464d2457c9250027c
This commit is contained in:
commit
34c5f8963c
1 changed files with 145336 additions and 0 deletions
401
packages/oci-l4-relay-publisher/cmd/fetch/main.go
Normal file
401
packages/oci-l4-relay-publisher/cmd/fetch/main.go
Normal file
|
|
@ -0,0 +1,401 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maximumSourceBytes = 512 << 20
|
||||
|
||||
type source struct {
|
||||
name string
|
||||
url string
|
||||
minimumInterval time.Duration
|
||||
headBeforeGet bool
|
||||
redactURL bool
|
||||
}
|
||||
|
||||
type sourceState struct {
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
NotBefore time.Time `json:"not_before"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
LastModified string `json:"last_modified,omitempty"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
}
|
||||
|
||||
type sourceResult struct {
|
||||
Name string `json:"name"`
|
||||
Changed bool `json:"changed"`
|
||||
State sourceState `json:"state"`
|
||||
}
|
||||
|
||||
func credential(path, name string) (string, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s credential", name)
|
||||
}
|
||||
value := strings.TrimSpace(string(contents))
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("%s credential is empty", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func azureURL(ctx context.Context, client *http.Client) (string, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetch Azure release page")
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("fetch Azure release page: HTTP %d", response.StatusCode)
|
||||
}
|
||||
contents, err := io.ReadAll(io.LimitReader(response.Body, 8<<20))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read Azure release page")
|
||||
}
|
||||
pattern := regexp.MustCompile(`https://download\.microsoft\.com/download/[^"']+ServiceTags_Public_[0-9]+\.json`)
|
||||
match := pattern.FindString(string(contents))
|
||||
if match == "" {
|
||||
return "", errors.New("discover Azure service tag release")
|
||||
}
|
||||
return strings.ReplaceAll(match, "&", "&"), nil
|
||||
}
|
||||
|
||||
func sourceCatalog(ctx context.Context, client *http.Client, selected map[string]bool, maxMindKeyPath string) (map[string]source, error) {
|
||||
catalog := map[string]source{
|
||||
"iplocate-country": {name: "iplocate-country.zip", url: "https://github.com/iplocate/ip-address-databases/raw/refs/heads/main/ip-to-country/ip-to-country.csv.zip", minimumInterval: 20 * time.Hour},
|
||||
"iplocate-asn": {name: "iplocate-asn.zip", url: "https://github.com/iplocate/ip-address-databases/raw/refs/heads/main/ip-to-asn/ip-to-asn.csv.zip", minimumInterval: 20 * time.Hour},
|
||||
"spamhaus-drop": {name: "spamhaus-drop.json", url: "https://www.spamhaus.org/drop/drop_v4.json", minimumInterval: 24 * time.Hour},
|
||||
"tor-exits": {name: "tor-exits.txt", url: "https://check.torproject.org/torbulkexitlist", minimumInterval: 6 * 24 * time.Hour},
|
||||
"aws": {name: "aws.json", url: "https://ip-ranges.amazonaws.com/ip-ranges.json", minimumInterval: 6 * 24 * time.Hour},
|
||||
"gcp": {name: "gcp.json", url: "https://www.gstatic.com/ipranges/cloud.json", minimumInterval: 6 * 24 * time.Hour},
|
||||
"oci": {name: "oci.json", url: "https://docs.oracle.com/iaas/tools/public_ip_ranges.json", minimumInterval: 24 * time.Hour},
|
||||
"cloudflare": {name: "cloudflare.txt", url: "https://www.cloudflare.com/ips-v4", minimumInterval: 6 * 24 * time.Hour},
|
||||
"fastly": {name: "fastly.json", url: "https://api.fastly.com/public-ip-list", minimumInterval: 6 * 24 * time.Hour},
|
||||
"github": {name: "github.json", url: "https://api.github.com/meta", minimumInterval: 6 * 24 * time.Hour},
|
||||
"atlassian": {name: "atlassian.json", url: "https://ip-ranges.atlassian.com/", minimumInterval: 6 * 24 * time.Hour},
|
||||
"digitalocean": {name: "digitalocean.csv", url: "https://digitalocean.com/geo/google.csv", minimumInterval: 6 * 24 * time.Hour},
|
||||
"linode": {name: "linode.csv", url: "https://geoip.linode.com/", minimumInterval: 6 * 24 * time.Hour},
|
||||
"vultr": {name: "vultr.csv", url: "https://geofeed.constant.com/?text", minimumInterval: 6 * 24 * time.Hour},
|
||||
}
|
||||
if selected["maxmind-country"] {
|
||||
key, err := credential(maxMindKeyPath, "MaxMind")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
query := url.Values{"edition_id": {"GeoLite2-Country-CSV"}, "license_key": {key}, "suffix": {"zip"}}
|
||||
catalog["maxmind-country"] = source{name: "maxmind-country.zip", url: "https://download.maxmind.com/app/geoip_download?" + query.Encode(), minimumInterval: 12 * time.Hour, headBeforeGet: true, redactURL: true}
|
||||
}
|
||||
if selected["azure"] {
|
||||
downloadURL, err := azureURL(ctx, client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
catalog["azure"] = source{name: "azure.json", url: downloadURL, minimumInterval: 6 * 24 * time.Hour}
|
||||
}
|
||||
return catalog, nil
|
||||
}
|
||||
|
||||
func readState(path string) (sourceState, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return sourceState{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return sourceState{}, err
|
||||
}
|
||||
var state sourceState
|
||||
if err := json.Unmarshal(contents, &state); err != nil {
|
||||
return sourceState{}, err
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func writeState(path string, state sourceState) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
contents, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporary, err := os.CreateTemp(filepath.Dir(path), ".state-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryPath) }()
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temporary.Write(append(contents, '\n')); err != nil {
|
||||
_ = temporary.Close()
|
||||
return err
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(temporaryPath, path)
|
||||
}
|
||||
|
||||
func writeJSON(path string, value any) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
contents, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, append(contents, '\n'), 0o644)
|
||||
}
|
||||
|
||||
func retryAfter(response *http.Response, now time.Time) time.Time {
|
||||
value := response.Header.Get("Retry-After")
|
||||
if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 {
|
||||
return now.Add(time.Duration(seconds) * time.Second)
|
||||
}
|
||||
if parsed, err := http.ParseTime(value); err == nil && parsed.After(now) {
|
||||
return parsed
|
||||
}
|
||||
return now.Add(time.Hour)
|
||||
}
|
||||
|
||||
func request(ctx context.Context, client *http.Client, method string, item source, state sourceState) (*http.Response, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, method, item.url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create %s request", item.name)
|
||||
}
|
||||
request.Header.Set("User-Agent", "oci-l4-relay-admission-publisher/1")
|
||||
if state.ETag != "" {
|
||||
request.Header.Set("If-None-Match", state.ETag)
|
||||
}
|
||||
if state.LastModified != "" {
|
||||
request.Header.Set("If-Modified-Since", state.LastModified)
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
if !item.redactURL {
|
||||
return nil, fmt.Errorf("fetch %s: %w", item.name, err)
|
||||
}
|
||||
return nil, fmt.Errorf("fetch %s", item.name)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func fetchSource(ctx context.Context, client *http.Client, key string, item source, dataDirectory, stateDirectory string, now time.Time) (bool, error) {
|
||||
statePath := filepath.Join(stateDirectory, key+".json")
|
||||
state, err := readState(statePath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read %s state", key)
|
||||
}
|
||||
dataPath := filepath.Join(dataDirectory, item.name)
|
||||
if now.Before(state.NotBefore) {
|
||||
if _, err := os.Stat(dataPath); err != nil || state.SHA256 == "" {
|
||||
return false, fmt.Errorf("%s is not due and has no cached generation", key)
|
||||
}
|
||||
fmt.Printf("source=%s due=false\n", key)
|
||||
return false, nil
|
||||
}
|
||||
method := http.MethodGet
|
||||
if item.headBeforeGet && state.SHA256 != "" {
|
||||
method = http.MethodHead
|
||||
}
|
||||
response, err := request(ctx, client, method, item, state)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode == http.StatusTooManyRequests || response.StatusCode >= 500 {
|
||||
state.NotBefore = retryAfter(response, now)
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, fmt.Errorf("fetch %s: HTTP %d", key, response.StatusCode)
|
||||
}
|
||||
if response.StatusCode == http.StatusNotModified {
|
||||
state.CheckedAt = now
|
||||
state.NotBefore = now.Add(item.minimumInterval)
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
fmt.Printf("source=%s changed=false\n", key)
|
||||
return false, nil
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return false, fmt.Errorf("fetch %s: HTTP %d", key, response.StatusCode)
|
||||
}
|
||||
if method == http.MethodHead {
|
||||
unchanged := response.Header.Get("ETag") == state.ETag && response.Header.Get("Last-Modified") == state.LastModified
|
||||
if unchanged {
|
||||
state.CheckedAt = now
|
||||
state.NotBefore = now.Add(item.minimumInterval)
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
fmt.Printf("source=%s changed=false\n", key)
|
||||
return false, nil
|
||||
}
|
||||
_ = response.Body.Close()
|
||||
response, err = request(ctx, client, http.MethodGet, item, sourceState{})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return false, fmt.Errorf("download %s: HTTP %d", key, response.StatusCode)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(dataDirectory, 0o700); err != nil {
|
||||
return false, err
|
||||
}
|
||||
temporary, err := os.CreateTemp(dataDirectory, ".download-*")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer func() { _ = os.Remove(temporaryPath) }()
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
_ = temporary.Close()
|
||||
return false, err
|
||||
}
|
||||
hash := sha256.New()
|
||||
written, copyErr := io.Copy(io.MultiWriter(temporary, hash), io.LimitReader(response.Body, maximumSourceBytes+1))
|
||||
closeErr := temporary.Close()
|
||||
if copyErr != nil {
|
||||
return false, copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return false, closeErr
|
||||
}
|
||||
if written > maximumSourceBytes {
|
||||
return false, fmt.Errorf("%s exceeds size limit", key)
|
||||
}
|
||||
digest := hex.EncodeToString(hash.Sum(nil))
|
||||
changed := digest != state.SHA256
|
||||
if changed {
|
||||
if err := os.Rename(temporaryPath, dataPath); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
state = sourceState{
|
||||
CheckedAt: now,
|
||||
UpdatedAt: state.UpdatedAt,
|
||||
NotBefore: now.Add(item.minimumInterval),
|
||||
ETag: response.Header.Get("ETag"),
|
||||
LastModified: response.Header.Get("Last-Modified"),
|
||||
SHA256: digest,
|
||||
Bytes: written,
|
||||
}
|
||||
if changed {
|
||||
state.UpdatedAt = now
|
||||
}
|
||||
if err := writeState(statePath, state); err != nil {
|
||||
return false, err
|
||||
}
|
||||
fmt.Printf("source=%s changed=%t bytes=%d sha256=%s\n", key, changed, written, digest)
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func selectedSources(value string) (map[string]bool, []string, error) {
|
||||
selected := make(map[string]bool)
|
||||
var order []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item == "" || selected[item] {
|
||||
continue
|
||||
}
|
||||
selected[item] = true
|
||||
order = append(order, item)
|
||||
}
|
||||
if len(order) == 0 {
|
||||
return nil, nil, errors.New("at least one source is required")
|
||||
}
|
||||
return selected, order, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
sourceNames := flag.String("sources", "", "comma-separated source names")
|
||||
dataDirectory := flag.String("data-directory", "/work/data", "validated source cache")
|
||||
stateDirectory := flag.String("state-directory", "/work/state", "source fetch state")
|
||||
changedFile := flag.String("changed-file", "/work/fetch-changed", "aggregate changed result")
|
||||
summaryFile := flag.String("summary-file", "/work/fetch-summary.json", "sanitized fetch summary")
|
||||
maxMindKeyPath := flag.String("maxmind-license-key-file", "/credentials/maxmind/license-key", "MaxMind license-key file")
|
||||
flag.Parse()
|
||||
selected, order, err := selectedSources(*sourceNames)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||||
defer cancel()
|
||||
client := &http.Client{Timeout: 15 * time.Minute}
|
||||
catalog, err := sourceCatalog(ctx, client, selected, *maxMindKeyPath)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
changed := false
|
||||
results := make([]sourceResult, 0, len(order))
|
||||
if contents, err := os.ReadFile(*changedFile); err == nil {
|
||||
changed, _ = strconv.ParseBool(strings.TrimSpace(string(contents)))
|
||||
}
|
||||
for _, name := range order {
|
||||
item, ok := catalog[name]
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "unknown source %q\n", name)
|
||||
os.Exit(2)
|
||||
}
|
||||
currentChanged, err := fetchSource(ctx, client, name, item, *dataDirectory, *stateDirectory, now)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
changed = changed || currentChanged
|
||||
state, err := readState(filepath.Join(*stateDirectory, name+".json"))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
results = append(results, sourceResult{Name: name, Changed: currentChanged, State: state})
|
||||
if err := os.MkdirAll(filepath.Dir(*changedFile), 0o755); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(*changedFile, []byte(strconv.FormatBool(changed)+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(*changedFile, []byte(strconv.FormatBool(changed)+"\n"), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := writeJSON(*summaryFile, results); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue