900 lines
25 KiB
Go
900 lines
25 KiB
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"archive/zip"
|
||
|
|
"bufio"
|
||
|
|
"compress/gzip"
|
||
|
|
"encoding/csv"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"flag"
|
||
|
|
"fmt"
|
||
|
|
"io"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"sort"
|
||
|
|
"strconv"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
func headerIndexes(header []string) map[string]int {
|
||
|
|
result := make(map[string]int, len(header))
|
||
|
|
for index, name := range header {
|
||
|
|
result[name] = index
|
||
|
|
}
|
||
|
|
return result
|
||
|
|
}
|
||
|
|
|
||
|
|
func fileWithSuffix(archive *zip.ReadCloser, suffix string) (*zip.File, error) {
|
||
|
|
for _, file := range archive.File {
|
||
|
|
if strings.HasSuffix(file.Name, suffix) {
|
||
|
|
return file, nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil, fmt.Errorf("missing %s", suffix)
|
||
|
|
}
|
||
|
|
|
||
|
|
func openCSV(file *zip.File) (*csv.Reader, io.Closer, error) {
|
||
|
|
contents, err := file.Open()
|
||
|
|
if err != nil {
|
||
|
|
return nil, nil, err
|
||
|
|
}
|
||
|
|
return csv.NewReader(contents), contents, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func selectedCountry(country, continent string) bool {
|
||
|
|
return country == "JP" || country == "KR" || country == "SG" || country == "US" || continent == "EU"
|
||
|
|
}
|
||
|
|
|
||
|
|
func vpnExceptionCountry(country string) bool {
|
||
|
|
return country == "JP" || country == "KR"
|
||
|
|
}
|
||
|
|
|
||
|
|
type geographySets struct {
|
||
|
|
admission []addressRange
|
||
|
|
vpnException []addressRange
|
||
|
|
}
|
||
|
|
|
||
|
|
type countrySelection struct {
|
||
|
|
admission bool
|
||
|
|
vpnException bool
|
||
|
|
}
|
||
|
|
|
||
|
|
func maxMindGeography(path string) (geographySets, error) {
|
||
|
|
archive, err := zip.OpenReader(path)
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
defer func() { _ = archive.Close() }()
|
||
|
|
locationFile, err := fileWithSuffix(archive, "Country-Locations-en.csv")
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
locationReader, locationContents, err := openCSV(locationFile)
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
locationHeader, err := locationReader.Read()
|
||
|
|
if err != nil {
|
||
|
|
_ = locationContents.Close()
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
locationFields := headerIndexes(locationHeader)
|
||
|
|
locations := make(map[int]countrySelection)
|
||
|
|
for {
|
||
|
|
row, err := locationReader.Read()
|
||
|
|
if err == io.EOF {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
_ = locationContents.Close()
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
id, err := strconv.Atoi(row[locationFields["geoname_id"]])
|
||
|
|
if err == nil {
|
||
|
|
country := row[locationFields["country_iso_code"]]
|
||
|
|
locations[id] = countrySelection{
|
||
|
|
admission: selectedCountry(country, row[locationFields["continent_code"]]),
|
||
|
|
vpnException: vpnExceptionCountry(country),
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if err := locationContents.Close(); err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
|
||
|
|
blocksFile, err := fileWithSuffix(archive, "Country-Blocks-IPv4.csv")
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
blocksReader, blocksContents, err := openCSV(blocksFile)
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
defer func() { _ = blocksContents.Close() }()
|
||
|
|
blocksHeader, err := blocksReader.Read()
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
blockFields := headerIndexes(blocksHeader)
|
||
|
|
result := geographySets{}
|
||
|
|
for {
|
||
|
|
row, err := blocksReader.Read()
|
||
|
|
if err == io.EOF {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
idText := row[blockFields["geoname_id"]]
|
||
|
|
if idText == "" {
|
||
|
|
idText = row[blockFields["registered_country_geoname_id"]]
|
||
|
|
}
|
||
|
|
id, err := strconv.Atoi(idText)
|
||
|
|
selection, found := locations[id]
|
||
|
|
if err != nil || !found {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if network, ok := rangeFromPrefix(row[blockFields["network"]]); ok {
|
||
|
|
if selection.admission {
|
||
|
|
result.admission = append(result.admission, network)
|
||
|
|
}
|
||
|
|
if selection.vpnException {
|
||
|
|
result.vpnException = append(result.vpnException, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
result.admission = mergeRanges(result.admission)
|
||
|
|
result.vpnException = mergeRanges(result.vpnException)
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func ipLocateGeography(path string) (geographySets, error) {
|
||
|
|
archive, err := zip.OpenReader(path)
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
defer func() { _ = archive.Close() }()
|
||
|
|
file, err := fileWithSuffix(archive, ".csv")
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
reader, contents, err := openCSV(file)
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
defer func() { _ = contents.Close() }()
|
||
|
|
header, err := reader.Read()
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
fields := headerIndexes(header)
|
||
|
|
result := geographySets{}
|
||
|
|
for {
|
||
|
|
row, err := reader.Read()
|
||
|
|
if err == io.EOF {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
return geographySets{}, err
|
||
|
|
}
|
||
|
|
country := row[fields["country_code"]]
|
||
|
|
if network, ok := rangeFromPrefix(row[fields["network"]]); ok {
|
||
|
|
if selectedCountry(country, row[fields["continent_code"]]) {
|
||
|
|
result.admission = append(result.admission, network)
|
||
|
|
}
|
||
|
|
if vpnExceptionCountry(country) {
|
||
|
|
result.vpnException = append(result.vpnException, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
result.admission = mergeRanges(result.admission)
|
||
|
|
result.vpnException = mergeRanges(result.vpnException)
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func parseASN(value string) (uint32, error) {
|
||
|
|
value = strings.TrimSpace(value)
|
||
|
|
if len(value) > 2 && strings.EqualFold(value[:2], "AS") {
|
||
|
|
value = value[2:]
|
||
|
|
}
|
||
|
|
parsed, err := strconv.ParseUint(value, 10, 32)
|
||
|
|
if err != nil || parsed == 0 {
|
||
|
|
return 0, fmt.Errorf("invalid ASN %q", value)
|
||
|
|
}
|
||
|
|
return uint32(parsed), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func ipLocateASNRanges(path string, selected map[uint32]struct{}) ([]addressRange, error) {
|
||
|
|
if len(selected) == 0 {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
archive, err := zip.OpenReader(path)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
defer func() { _ = archive.Close() }()
|
||
|
|
file, err := fileWithSuffix(archive, ".csv")
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
reader, contents, err := openCSV(file)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
defer func() { _ = contents.Close() }()
|
||
|
|
header, err := reader.Read()
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
fields := headerIndexes(header)
|
||
|
|
var result []addressRange
|
||
|
|
for {
|
||
|
|
row, err := reader.Read()
|
||
|
|
if err == io.EOF {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
asn, err := parseASN(row[fields["asn"]])
|
||
|
|
if err != nil {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if _, found := selected[asn]; !found {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if network, ok := rangeFromPrefix(row[fields["network"]]); ok {
|
||
|
|
result = append(result, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return mergeRanges(result), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func jsonContents(path string, target any) error {
|
||
|
|
contents, err := os.ReadFile(path)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return json.Unmarshal(contents, target)
|
||
|
|
}
|
||
|
|
|
||
|
|
func prefixRanges(prefixes []string) []addressRange {
|
||
|
|
result := make([]addressRange, 0, len(prefixes))
|
||
|
|
for _, prefix := range prefixes {
|
||
|
|
if network, ok := rangeFromPrefix(prefix); ok {
|
||
|
|
result = append(result, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return mergeRanges(result)
|
||
|
|
}
|
||
|
|
|
||
|
|
func awsRanges(path string) ([]addressRange, error) {
|
||
|
|
var document struct {
|
||
|
|
Prefixes []struct {
|
||
|
|
Prefix string `json:"ip_prefix"`
|
||
|
|
} `json:"prefixes"`
|
||
|
|
}
|
||
|
|
if err := jsonContents(path, &document); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
prefixes := make([]string, 0, len(document.Prefixes))
|
||
|
|
for _, item := range document.Prefixes {
|
||
|
|
prefixes = append(prefixes, item.Prefix)
|
||
|
|
}
|
||
|
|
return prefixRanges(prefixes), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func gcpRanges(path string) ([]addressRange, error) {
|
||
|
|
var document struct {
|
||
|
|
Prefixes []struct {
|
||
|
|
Prefix string `json:"ipv4Prefix"`
|
||
|
|
} `json:"prefixes"`
|
||
|
|
}
|
||
|
|
if err := jsonContents(path, &document); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
prefixes := make([]string, 0, len(document.Prefixes))
|
||
|
|
for _, item := range document.Prefixes {
|
||
|
|
prefixes = append(prefixes, item.Prefix)
|
||
|
|
}
|
||
|
|
return prefixRanges(prefixes), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func azureRanges(path string) ([]addressRange, error) {
|
||
|
|
var document struct {
|
||
|
|
Values []struct {
|
||
|
|
Properties struct {
|
||
|
|
Prefixes []string `json:"addressPrefixes"`
|
||
|
|
} `json:"properties"`
|
||
|
|
} `json:"values"`
|
||
|
|
}
|
||
|
|
if err := jsonContents(path, &document); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
var prefixes []string
|
||
|
|
for _, value := range document.Values {
|
||
|
|
prefixes = append(prefixes, value.Properties.Prefixes...)
|
||
|
|
}
|
||
|
|
return prefixRanges(prefixes), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func ociRanges(path string) ([]addressRange, error) {
|
||
|
|
var document struct {
|
||
|
|
Regions []struct {
|
||
|
|
CIDRs []struct {
|
||
|
|
CIDR string `json:"cidr"`
|
||
|
|
} `json:"cidrs"`
|
||
|
|
} `json:"regions"`
|
||
|
|
}
|
||
|
|
if err := jsonContents(path, &document); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
var prefixes []string
|
||
|
|
for _, region := range document.Regions {
|
||
|
|
for _, cidr := range region.CIDRs {
|
||
|
|
prefixes = append(prefixes, cidr.CIDR)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return prefixRanges(prefixes), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func fastlyRanges(path string) ([]addressRange, error) {
|
||
|
|
var document struct {
|
||
|
|
Addresses []string `json:"addresses"`
|
||
|
|
}
|
||
|
|
if err := jsonContents(path, &document); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
return prefixRanges(document.Addresses), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func githubRanges(path string) ([]addressRange, error) {
|
||
|
|
var document map[string]json.RawMessage
|
||
|
|
if err := jsonContents(path, &document); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
var prefixes []string
|
||
|
|
for _, contents := range document {
|
||
|
|
var values []string
|
||
|
|
if err := json.Unmarshal(contents, &values); err == nil {
|
||
|
|
prefixes = append(prefixes, values...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return prefixRanges(prefixes), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func atlassianRanges(path string) ([]addressRange, error) {
|
||
|
|
var document struct {
|
||
|
|
Items []struct {
|
||
|
|
CIDR string `json:"cidr"`
|
||
|
|
} `json:"items"`
|
||
|
|
}
|
||
|
|
if err := jsonContents(path, &document); err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
prefixes := make([]string, 0, len(document.Items))
|
||
|
|
for _, item := range document.Items {
|
||
|
|
prefixes = append(prefixes, item.CIDR)
|
||
|
|
}
|
||
|
|
return prefixRanges(prefixes), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func geofeedRanges(path string) ([]addressRange, error) {
|
||
|
|
file, err := os.Open(path)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
defer func() { _ = file.Close() }()
|
||
|
|
reader := csv.NewReader(file)
|
||
|
|
reader.Comment = '#'
|
||
|
|
reader.FieldsPerRecord = -1
|
||
|
|
var result []addressRange
|
||
|
|
for {
|
||
|
|
row, err := reader.Read()
|
||
|
|
if err == io.EOF {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
if len(row) == 0 {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if network, ok := rangeFromPrefix(strings.TrimSpace(row[0])); ok {
|
||
|
|
result = append(result, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return mergeRanges(result), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func lineRanges(path string) ([]addressRange, error) {
|
||
|
|
file, err := os.Open(path)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
defer func() { _ = file.Close() }()
|
||
|
|
var result []addressRange
|
||
|
|
scanner := bufio.NewScanner(file)
|
||
|
|
for scanner.Scan() {
|
||
|
|
value := strings.TrimSpace(scanner.Text())
|
||
|
|
if value == "" {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if !strings.Contains(value, "/") {
|
||
|
|
value += "/32"
|
||
|
|
}
|
||
|
|
if network, ok := rangeFromPrefix(value); ok {
|
||
|
|
result = append(result, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return mergeRanges(result), scanner.Err()
|
||
|
|
}
|
||
|
|
|
||
|
|
func spamhausRanges(path string) ([]addressRange, error) {
|
||
|
|
file, err := os.Open(path)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
defer func() { _ = file.Close() }()
|
||
|
|
decoder := json.NewDecoder(file)
|
||
|
|
var result []addressRange
|
||
|
|
for {
|
||
|
|
var record struct {
|
||
|
|
CIDR string `json:"cidr"`
|
||
|
|
}
|
||
|
|
if err := decoder.Decode(&record); err == io.EOF {
|
||
|
|
break
|
||
|
|
} else if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
if network, ok := rangeFromPrefix(record.CIDR); ok {
|
||
|
|
result = append(result, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return mergeRanges(result), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func unionRanges(sets ...[]addressRange) []addressRange {
|
||
|
|
var result []addressRange
|
||
|
|
for _, set := range sets {
|
||
|
|
result = append(result, set...)
|
||
|
|
}
|
||
|
|
return mergeRanges(result)
|
||
|
|
}
|
||
|
|
|
||
|
|
func percent(part, whole uint64) float64 {
|
||
|
|
if whole == 0 {
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
return float64(part) * 100 / float64(whole)
|
||
|
|
}
|
||
|
|
|
||
|
|
type categoryStat struct {
|
||
|
|
records int
|
||
|
|
addresses uint64
|
||
|
|
}
|
||
|
|
|
||
|
|
type ip2ProxyStats struct {
|
||
|
|
all []addressRange
|
||
|
|
recentSevenDays []addressRange
|
||
|
|
threat []addressRange
|
||
|
|
highFraudScore []addressRange
|
||
|
|
usageTypes map[string]categoryStat
|
||
|
|
proxyTypes map[string]categoryStat
|
||
|
|
threatRecords int
|
||
|
|
positiveFraudScores int
|
||
|
|
minimumLastSeen int
|
||
|
|
maximumLastSeen int
|
||
|
|
}
|
||
|
|
|
||
|
|
func incrementCategory(categories map[string]categoryStat, name string, size uint64) {
|
||
|
|
current := categories[name]
|
||
|
|
current.records++
|
||
|
|
current.addresses += size
|
||
|
|
categories[name] = current
|
||
|
|
}
|
||
|
|
|
||
|
|
func ip2ProxyClassifications(path string) (ip2ProxyStats, error) {
|
||
|
|
archive, err := zip.OpenReader(path)
|
||
|
|
if err != nil {
|
||
|
|
return ip2ProxyStats{}, err
|
||
|
|
}
|
||
|
|
defer func() { _ = archive.Close() }()
|
||
|
|
file, err := fileWithSuffix(archive, "IP2PROXY-LITE-PX12.CSV")
|
||
|
|
if err != nil {
|
||
|
|
return ip2ProxyStats{}, err
|
||
|
|
}
|
||
|
|
reader, contents, err := openCSV(file)
|
||
|
|
if err != nil {
|
||
|
|
return ip2ProxyStats{}, err
|
||
|
|
}
|
||
|
|
defer func() { _ = contents.Close() }()
|
||
|
|
result := ip2ProxyStats{usageTypes: make(map[string]categoryStat), proxyTypes: make(map[string]categoryStat)}
|
||
|
|
for {
|
||
|
|
row, err := reader.Read()
|
||
|
|
if err == io.EOF {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
return ip2ProxyStats{}, err
|
||
|
|
}
|
||
|
|
if len(row) != 16 {
|
||
|
|
return ip2ProxyStats{}, fmt.Errorf("unexpected IP2Proxy field count %d", len(row))
|
||
|
|
}
|
||
|
|
start, err := strconv.ParseUint(row[0], 10, 32)
|
||
|
|
if err != nil {
|
||
|
|
return ip2ProxyStats{}, err
|
||
|
|
}
|
||
|
|
end, err := strconv.ParseUint(row[1], 10, 32)
|
||
|
|
if err != nil {
|
||
|
|
return ip2ProxyStats{}, err
|
||
|
|
}
|
||
|
|
network := addressRange{start: uint32(start), end: uint32(end)}
|
||
|
|
size := end - start + 1
|
||
|
|
proxyType := row[2]
|
||
|
|
usageType := row[9]
|
||
|
|
incrementCategory(result.proxyTypes, proxyType, size)
|
||
|
|
incrementCategory(result.usageTypes, usageType, size)
|
||
|
|
result.all = append(result.all, network)
|
||
|
|
if row[13] != "-" {
|
||
|
|
result.threatRecords++
|
||
|
|
result.threat = append(result.threat, network)
|
||
|
|
}
|
||
|
|
lastSeen, err := strconv.Atoi(row[12])
|
||
|
|
if err == nil && lastSeen > 0 {
|
||
|
|
if result.minimumLastSeen == 0 || lastSeen < result.minimumLastSeen {
|
||
|
|
result.minimumLastSeen = lastSeen
|
||
|
|
}
|
||
|
|
if lastSeen > result.maximumLastSeen {
|
||
|
|
result.maximumLastSeen = lastSeen
|
||
|
|
}
|
||
|
|
if lastSeen <= 7 {
|
||
|
|
result.recentSevenDays = append(result.recentSevenDays, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
fraudScore, err := strconv.Atoi(row[15])
|
||
|
|
if err == nil && fraudScore > 0 {
|
||
|
|
result.positiveFraudScores++
|
||
|
|
}
|
||
|
|
if err == nil && fraudScore >= 80 {
|
||
|
|
result.highFraudScore = append(result.highFraudScore, network)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
result.all = mergeRanges(result.all)
|
||
|
|
result.recentSevenDays = mergeRanges(result.recentSevenDays)
|
||
|
|
result.threat = mergeRanges(result.threat)
|
||
|
|
result.highFraudScore = mergeRanges(result.highFraudScore)
|
||
|
|
return result, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func loadIP2ProxyClassifications(path string, now time.Time, maximumAge time.Duration) (ip2ProxyStats, string, time.Duration, error) {
|
||
|
|
empty := ip2ProxyStats{usageTypes: make(map[string]categoryStat), proxyTypes: make(map[string]categoryStat)}
|
||
|
|
if maximumAge <= 0 {
|
||
|
|
return empty, "", 0, errors.New("maximum IP2Proxy age must be positive")
|
||
|
|
}
|
||
|
|
info, err := os.Stat(path)
|
||
|
|
if errors.Is(err, os.ErrNotExist) {
|
||
|
|
return empty, "absent", 0, nil
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
return empty, "", 0, err
|
||
|
|
}
|
||
|
|
if info.ModTime().After(now.Add(5 * time.Minute)) {
|
||
|
|
return empty, "", 0, errors.New("IP2Proxy cache modification time is in the future")
|
||
|
|
}
|
||
|
|
age := now.Sub(info.ModTime())
|
||
|
|
if age > maximumAge {
|
||
|
|
return empty, "stale", age, nil
|
||
|
|
}
|
||
|
|
classifications, err := ip2ProxyClassifications(path)
|
||
|
|
if err != nil {
|
||
|
|
return empty, "", age, err
|
||
|
|
}
|
||
|
|
return classifications, "fresh", age, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func printCategories(name string, categories map[string]categoryStat) {
|
||
|
|
keys := make([]string, 0, len(categories))
|
||
|
|
for key := range categories {
|
||
|
|
keys = append(keys, key)
|
||
|
|
}
|
||
|
|
sort.Strings(keys)
|
||
|
|
for _, key := range keys {
|
||
|
|
label := key
|
||
|
|
if label == "-" {
|
||
|
|
label = "unclassified"
|
||
|
|
}
|
||
|
|
fmt.Printf("%s_%s_records=%d\n", name, strings.ToLower(label), categories[key].records)
|
||
|
|
fmt.Printf("%s_%s_addresses=%d\n", name, strings.ToLower(label), categories[key].addresses)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func printSet(name string, ranges []addressRange) {
|
||
|
|
fmt.Printf("%s_intervals=%d\n", name, len(ranges))
|
||
|
|
fmt.Printf("%s_addresses=%d\n", name, addressCount(ranges))
|
||
|
|
}
|
||
|
|
|
||
|
|
func ipv4String(value uint32) string {
|
||
|
|
return fmt.Sprintf("%d.%d.%d.%d", byte(value>>24), byte(value>>16), byte(value>>8), byte(value))
|
||
|
|
}
|
||
|
|
|
||
|
|
func writeNftSet(writer *bufio.Writer, name string, ranges []addressRange) error {
|
||
|
|
if _, err := fmt.Fprintf(writer, " set %s {\n type ipv4_addr\n flags interval\n auto-merge\n elements = {\n", name); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
for index, current := range ranges {
|
||
|
|
separator := ","
|
||
|
|
if index == len(ranges)-1 {
|
||
|
|
separator = ""
|
||
|
|
}
|
||
|
|
value := ipv4String(current.start)
|
||
|
|
if current.start != current.end {
|
||
|
|
value += "-" + ipv4String(current.end)
|
||
|
|
}
|
||
|
|
if _, err := fmt.Fprintf(writer, " %s%s\n", value, separator); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
_, err := fmt.Fprint(writer, " }\n }\n")
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
func writeNft(path string, geography, denied []addressRange) error {
|
||
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
writer := bufio.NewWriter(file)
|
||
|
|
if _, err := fmt.Fprintln(writer, "table inet relay_spike {"); err != nil {
|
||
|
|
_ = file.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if err := writeNftSet(writer, "geography_v4", geography); err != nil {
|
||
|
|
_ = file.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if err := writeNftSet(writer, "deny_v4", denied); err != nil {
|
||
|
|
_ = file.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if _, err := fmt.Fprintln(writer, "}"); err != nil {
|
||
|
|
_ = file.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if err := writer.Flush(); err != nil {
|
||
|
|
_ = file.Close()
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
return file.Close()
|
||
|
|
}
|
||
|
|
|
||
|
|
type bundleRange struct {
|
||
|
|
Start string `json:"start"`
|
||
|
|
End string `json:"end"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type admissionBundle struct {
|
||
|
|
SchemaVersion int `json:"schema_version"`
|
||
|
|
GeneratedAt time.Time `json:"generated_at"`
|
||
|
|
ExpiresAt time.Time `json:"expires_at"`
|
||
|
|
AdmittedIPv4 []bundleRange `json:"admitted_ipv4"`
|
||
|
|
ExceptionIPv4 []bundleRange `json:"exception_ipv4"`
|
||
|
|
AdmittedIntervalCount int `json:"admitted_interval_count"`
|
||
|
|
AdmittedAddressCount uint64 `json:"admitted_address_count"`
|
||
|
|
ExceptionIntervalCount int `json:"exception_interval_count"`
|
||
|
|
ExceptionAddressCount uint64 `json:"exception_address_count"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func writeBundle(path string, admitted, exceptions []addressRange, now time.Time) error {
|
||
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
compressed := gzip.NewWriter(file)
|
||
|
|
document := admissionBundle{
|
||
|
|
SchemaVersion: 1,
|
||
|
|
GeneratedAt: now.UTC(),
|
||
|
|
ExpiresAt: now.UTC().Add(72 * time.Hour),
|
||
|
|
AdmittedIPv4: make([]bundleRange, 0, len(admitted)),
|
||
|
|
ExceptionIPv4: make([]bundleRange, 0, len(exceptions)),
|
||
|
|
AdmittedIntervalCount: len(admitted),
|
||
|
|
AdmittedAddressCount: addressCount(admitted),
|
||
|
|
ExceptionIntervalCount: len(exceptions),
|
||
|
|
ExceptionAddressCount: addressCount(exceptions),
|
||
|
|
}
|
||
|
|
for _, item := range admitted {
|
||
|
|
document.AdmittedIPv4 = append(document.AdmittedIPv4, bundleRange{Start: ipv4String(item.start), End: ipv4String(item.end)})
|
||
|
|
}
|
||
|
|
for _, item := range exceptions {
|
||
|
|
document.ExceptionIPv4 = append(document.ExceptionIPv4, bundleRange{Start: ipv4String(item.start), End: ipv4String(item.end)})
|
||
|
|
}
|
||
|
|
encodeErr := json.NewEncoder(compressed).Encode(document)
|
||
|
|
closeGzipErr := compressed.Close()
|
||
|
|
closeFileErr := file.Close()
|
||
|
|
if encodeErr != nil {
|
||
|
|
return encodeErr
|
||
|
|
}
|
||
|
|
if closeGzipErr != nil {
|
||
|
|
return closeGzipErr
|
||
|
|
}
|
||
|
|
return closeFileErr
|
||
|
|
}
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
nftOutput := flag.String("nft-output", "", "write selective nftables sets")
|
||
|
|
nftFullOutput := flag.String("nft-full-output", "", "write complete nftables sets")
|
||
|
|
bundleOutput := flag.String("bundle-output", "", "write normalized selective admission bundle")
|
||
|
|
maximumIP2ProxyAge := flag.Duration("maximum-ip2proxy-age", 35*24*time.Hour, "omit IP2Proxy classifications after this cache age")
|
||
|
|
vpnExceptionASNs := map[uint32]struct{}{}
|
||
|
|
flag.Func("vpn-exception-asn", "VPN operator ASN admitted only within MaxMind or IPLocate JP/KR ranges", func(value string) error {
|
||
|
|
asn, err := parseASN(value)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
vpnExceptionASNs[asn] = struct{}{}
|
||
|
|
return nil
|
||
|
|
})
|
||
|
|
flag.Parse()
|
||
|
|
now := time.Now().UTC()
|
||
|
|
maxMind, err := maxMindGeography("data/maxmind-country.zip")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
ipLocate, err := ipLocateGeography("data/iplocate-country.zip")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
vpnASN, err := ipLocateASNRanges("data/iplocate-asn.zip", vpnExceptionASNs)
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
aws, err := awsRanges("data/aws.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
gcp, err := gcpRanges("data/gcp.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
azure, err := azureRanges("data/azure.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
oci, err := ociRanges("data/oci.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
cloudflare, err := lineRanges("data/cloudflare.txt")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
tor, err := lineRanges("data/tor-exits.txt")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
spamhaus, err := spamhausRanges("data/spamhaus-drop.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
fastly, err := fastlyRanges("data/fastly.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
github, err := githubRanges("data/github.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
atlassian, err := atlassianRanges("data/atlassian.json")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
digitalOcean, err := geofeedRanges("data/digitalocean.csv")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
linode, err := geofeedRanges("data/linode.csv")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
vultr, err := geofeedRanges("data/vultr.csv")
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
ip2Proxy, ip2ProxyState, ip2ProxyAge, err := loadIP2ProxyClassifications("data/ip2proxy.zip", now, *maximumIP2ProxyAge)
|
||
|
|
if err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
fmt.Printf("ip2proxy_cache_state=%s\n", ip2ProxyState)
|
||
|
|
if ip2ProxyState != "absent" {
|
||
|
|
fmt.Printf("ip2proxy_cache_age_hours=%.2f\n", ip2ProxyAge.Hours())
|
||
|
|
}
|
||
|
|
fmt.Printf("ip2proxy_cache_omitted=%t\n", ip2ProxyState != "fresh")
|
||
|
|
|
||
|
|
consensus := intersectRanges(maxMind.admission, ipLocate.admission)
|
||
|
|
geoUnion := unionRanges(maxMind.admission, ipLocate.admission)
|
||
|
|
geoDisagreement := addressCount(geoUnion) - addressCount(consensus)
|
||
|
|
vpnExceptionGeography := unionRanges(maxMind.vpnException, ipLocate.vpnException)
|
||
|
|
vpnExceptionCandidate := subtractRanges(intersectRanges(vpnASN, vpnExceptionGeography), consensus)
|
||
|
|
cloud := unionRanges(aws, gcp, azure, oci, cloudflare, fastly, github, atlassian, digitalOcean, linode, vultr)
|
||
|
|
denied := unionRanges(cloud, tor, spamhaus, ip2Proxy.all)
|
||
|
|
selectiveProxy := unionRanges(ip2Proxy.recentSevenDays, ip2Proxy.threat)
|
||
|
|
selectiveDenied := unionRanges(cloud, tor, spamhaus, selectiveProxy)
|
||
|
|
deniedInConsensus := intersectRanges(consensus, denied)
|
||
|
|
selectiveDeniedInConsensus := intersectRanges(consensus, selectiveDenied)
|
||
|
|
candidate := subtractRanges(consensus, denied)
|
||
|
|
selectiveCandidate := subtractRanges(consensus, selectiveDenied)
|
||
|
|
vpnExceptions := subtractRanges(vpnExceptionCandidate, denied)
|
||
|
|
selectiveVPNExceptions := subtractRanges(vpnExceptionCandidate, selectiveDenied)
|
||
|
|
|
||
|
|
printSet("maxmind_geo", maxMind.admission)
|
||
|
|
printSet("iplocate_geo", ipLocate.admission)
|
||
|
|
printSet("geo_consensus", consensus)
|
||
|
|
fmt.Printf("geo_disagreement_addresses=%d\n", geoDisagreement)
|
||
|
|
fmt.Printf("geo_disagreement_percent_of_union=%.6f\n", percent(geoDisagreement, addressCount(geoUnion)))
|
||
|
|
printSet("vpn_exception_geo_union", vpnExceptionGeography)
|
||
|
|
printSet("vpn_exception_asn", vpnASN)
|
||
|
|
printSet("vpn_exception_candidate_before_denies", vpnExceptionCandidate)
|
||
|
|
printSet("vpn_exception_after_denies", vpnExceptions)
|
||
|
|
printSet("selective_vpn_exception_after_denies", selectiveVPNExceptions)
|
||
|
|
printSet("aws", aws)
|
||
|
|
printSet("gcp", gcp)
|
||
|
|
printSet("azure", azure)
|
||
|
|
printSet("oci", oci)
|
||
|
|
printSet("cloudflare", cloudflare)
|
||
|
|
printSet("fastly", fastly)
|
||
|
|
printSet("github", github)
|
||
|
|
printSet("atlassian", atlassian)
|
||
|
|
printSet("digitalocean", digitalOcean)
|
||
|
|
printSet("linode", linode)
|
||
|
|
printSet("vultr", vultr)
|
||
|
|
printSet("cloud_union", cloud)
|
||
|
|
printSet("tor", tor)
|
||
|
|
printSet("spamhaus", spamhaus)
|
||
|
|
printSet("deny_union", denied)
|
||
|
|
printSet("deny_in_geo_consensus", deniedInConsensus)
|
||
|
|
fmt.Printf("deny_percent_of_geo_consensus=%.6f\n", percent(addressCount(deniedInConsensus), addressCount(consensus)))
|
||
|
|
printSet("candidate_after_denies", candidate)
|
||
|
|
printSet("selective_proxy", selectiveProxy)
|
||
|
|
printSet("selective_deny_union", selectiveDenied)
|
||
|
|
printSet("selective_deny_in_geo_consensus", selectiveDeniedInConsensus)
|
||
|
|
fmt.Printf("selective_deny_percent_of_geo_consensus=%.6f\n", percent(addressCount(selectiveDeniedInConsensus), addressCount(consensus)))
|
||
|
|
printSet("selective_candidate_after_denies", selectiveCandidate)
|
||
|
|
printSet("ip2proxy_open_proxy", ip2Proxy.all)
|
||
|
|
printSet("ip2proxy_recent_seven_days", ip2Proxy.recentSevenDays)
|
||
|
|
printSet("ip2proxy_threat", ip2Proxy.threat)
|
||
|
|
printSet("ip2proxy_high_fraud_score", ip2Proxy.highFraudScore)
|
||
|
|
printCategories("ip2proxy_usage", ip2Proxy.usageTypes)
|
||
|
|
printCategories("ip2proxy_proxy", ip2Proxy.proxyTypes)
|
||
|
|
fmt.Printf("ip2proxy_threat_records=%d\n", ip2Proxy.threatRecords)
|
||
|
|
fmt.Printf("ip2proxy_positive_fraud_score_records=%d\n", ip2Proxy.positiveFraudScores)
|
||
|
|
fmt.Printf("ip2proxy_minimum_last_seen_days=%d\n", ip2Proxy.minimumLastSeen)
|
||
|
|
fmt.Printf("ip2proxy_maximum_last_seen_days=%d\n", ip2Proxy.maximumLastSeen)
|
||
|
|
fmt.Println("open_proxy_classification=true")
|
||
|
|
fmt.Println("consumer_classification=false")
|
||
|
|
fmt.Println("generic_vpn_classification=false")
|
||
|
|
fmt.Println("residential_proxy_classification=false")
|
||
|
|
fmt.Printf("explicit_vpn_exception_asns=%d\n", len(vpnExceptionASNs))
|
||
|
|
if *nftOutput != "" {
|
||
|
|
if err := writeNft(*nftOutput, unionRanges(consensus, vpnExceptionCandidate), selectiveDenied); err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
fmt.Println("nft_output_written=true")
|
||
|
|
}
|
||
|
|
if *nftFullOutput != "" {
|
||
|
|
if err := writeNft(*nftFullOutput, unionRanges(consensus, vpnExceptionCandidate), denied); err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
fmt.Println("nft_full_output_written=true")
|
||
|
|
}
|
||
|
|
if *bundleOutput != "" {
|
||
|
|
if err := writeBundle(*bundleOutput, selectiveCandidate, selectiveVPNExceptions, now); err != nil {
|
||
|
|
panic(err)
|
||
|
|
}
|
||
|
|
fmt.Println("bundle_output_written=true")
|
||
|
|
}
|
||
|
|
}
|