Added external IP and ASN data. ASN filter feature. ASN lookup tool.

Signed-off-by: RonniSkansing <rskansing@gmail.com>
This commit is contained in:
RonniSkansing committed 2026-09-30 19:14:00 +02:00
1 parent d30b788c2b
commit 4d3859bcf1
25 files changed
+2483 -253

No files matched your search

+14
View File
@@ -226,6 +226,13 @@ const (
// geoip
ROUTE_V1_GEOIP_METADATA = "/api/v1/geoip/metadata"
ROUTE_V1_GEOIP_LOOKUP = "/api/v1/geoip/lookup"
ROUTE_V1_IPDATA_STATUS = "/api/v1/ipdata/status"
ROUTE_V1_IPDATA_PACKAGE_KIND = "/api/v1/ipdata/package/:kind"
ROUTE_V1_IPDATA_PACKAGE_DOWNLOAD = "/api/v1/ipdata/package/:kind/download"
ROUTE_V1_IPDATA_ASN_SEARCH = "/api/v1/ipdata/asn/search"
ROUTE_V1_IPDATA_ASN_RESOLVE = "/api/v1/ipdata/asn/resolve"
ROUTE_V1_IPDATA_ASN_LOOKUP = "/api/v1/ipdata/asn/lookup"
// web hooks
ROUTE_V1_WEBHOOK = "/api/v1/webhook"
ROUTE_V1_WEBHOOK_ID = "/api/v1/webhook/:id"
@@ -590,6 +597,13 @@ func setupRoutes(
// geoip
GET(ROUTE_V1_GEOIP_METADATA, middleware.SessionHandler, controllers.GeoIP.GetMetadata).
GET(ROUTE_V1_GEOIP_LOOKUP, middleware.SessionHandler, controllers.GeoIP.Lookup).
// ip data packages (country + asn)
GET(ROUTE_V1_IPDATA_STATUS, middleware.SessionHandler, controllers.IPData.Status).
POST(ROUTE_V1_IPDATA_PACKAGE_DOWNLOAD, middleware.SessionHandler, controllers.IPData.Download).
DELETE(ROUTE_V1_IPDATA_PACKAGE_KIND, middleware.SessionHandler, controllers.IPData.Remove).
GET(ROUTE_V1_IPDATA_ASN_SEARCH, middleware.SessionHandler, controllers.IPData.SearchASN).
POST(ROUTE_V1_IPDATA_ASN_RESOLVE, middleware.SessionHandler, controllers.IPData.ResolveASNs).
GET(ROUTE_V1_IPDATA_ASN_LOOKUP, middleware.SessionHandler, controllers.IPData.LookupASN).
// web hooks
GET(ROUTE_V1_WEBHOOK, middleware.SessionHandler, controllers.Webhook.GetAll).
GET(ROUTE_V1_WEBHOOK_ID, middleware.SessionHandler, controllers.Webhook.GetByID).
+6
View File
@@ -31,6 +31,7 @@ type Controllers struct {
APISender *controller.APISender
AllowDeny *controller.AllowDeny
GeoIP *controller.GeoIP
IPData *controller.IPData
Webhook *controller.Webhook
Identifier *controller.Identifier
Version *controller.Version
@@ -205,6 +206,10 @@ func NewControllers(
geoIP := &controller.GeoIP{
Common: common,
}
ipData := &controller.IPData{
Common: common,
IPDataService: services.IPData,
}
oauthProvider := &controller.OAuthProvider{
Common: common,
OAuthProviderService: services.OAuthProvider,
@@ -270,6 +275,7 @@ func NewControllers(
APISender: apiSender,
AllowDeny: allowDeny,
GeoIP: geoIP,
IPData: ipData,
Webhook: webhook,
Identifier: identifier,
Version: version,
+11 -8
View File
@@ -33,7 +33,7 @@ import (
"github.com/phishingclub/phishingclub/data"
"github.com/phishingclub/phishingclub/database"
"github.com/phishingclub/phishingclub/errs"
"github.com/phishingclub/phishingclub/geoip"
"github.com/phishingclub/phishingclub/ipdata"
"github.com/phishingclub/phishingclub/middleware"
"github.com/phishingclub/phishingclub/model"
"github.com/phishingclub/phishingclub/proxy"
@@ -2461,14 +2461,14 @@ func (s *Server) checkIPFilter(
// get ja4 fingerprint from context
ja4 := middleware.GetJA4FromContext(ctx)
// get country code from GeoIP lookup
var countryCode string
if geo, err := geoip.Instance(); err == nil {
countryCode, _ = geo.Lookup(ip)
}
// get country code and ASNs from the IP data lookup
store := ipdata.Get()
countryCode, _ := store.LookupCountry(ip)
asns := store.LookupASNs(ip)
s.logger.Debugw("checking geo ip",
"ip", ip,
"country", countryCode,
"asns", asns,
)
allowDenyLEntries, err := s.repositories.Campaign.GetAllDenyByCampaignID(ctx, campaignID)
@@ -2508,6 +2508,9 @@ func (s *Server) checkIPFilter(
// check country code filter
countryOk := allowDeny.IsCountryAllowed(countryCode)
// check ASN filter
asnOk := allowDeny.IsASNAllowed(asns)
// check header filter
headers := ctx.Request.Header
headerOk, err := allowDeny.IsHeaderAllowed(headers)
@@ -2519,7 +2522,7 @@ func (s *Server) checkIPFilter(
// for deny lists: any filter failing blocks the request
if isAllowListing {
// allow list: all must be allowed
if ipOk && ja4Ok && countryOk && headerOk {
if ipOk && ja4Ok && countryOk && asnOk && headerOk {
s.logger.Debugw("IP, JA4, country, and headers are allow listed",
"ip", ip,
"ja4", ja4,
@@ -2532,7 +2535,7 @@ func (s *Server) checkIPFilter(
}
} else {
// deny list: if any filter denies, block the request
if !ipOk || !ja4Ok || !countryOk || !headerOk {
if !ipOk || !ja4Ok || !countryOk || !asnOk || !headerOk {
s.logger.Debugw("IP, JA4, country, or headers is deny listed",
"ip", ip,
"ja4", ja4,
+7
View File
@@ -5,6 +5,7 @@ import (
"time"
"github.com/caddyserver/certmagic"
"github.com/phishingclub/phishingclub/ipdata"
"github.com/phishingclub/phishingclub/script"
"github.com/phishingclub/phishingclub/service"
"go.uber.org/zap"
@@ -38,6 +39,7 @@ type Services struct {
Version *service.Version
SSO *service.SSO
Update *service.Update
IPData *service.IPData
Import *service.Import
Backup *service.Backup
IPAllowList *service.IPAllowListService
@@ -326,6 +328,10 @@ func NewServices(
Common: common,
OptionService: optionService,
}
ipDataService := &service.IPData{
Common: common,
Store: ipdata.Get(),
}
importService := &service.Import{
Common: common,
Asset: asset,
@@ -406,6 +412,7 @@ func NewServices(
Version: versionService,
SSO: ssoService,
Update: updateService,
IPData: ipDataService,
Import: importService,
Backup: backupService,
IPAllowList: ipAllowListService,
+11 -28
View File
@@ -2,66 +2,49 @@ package controller
import (
"github.com/gin-gonic/gin"
"github.com/phishingclub/phishingclub/geoip"
"github.com/phishingclub/phishingclub/ipdata"
)
// GeoIP is a controller for GeoIP-related endpoints
// GeoIP is a controller for GeoIP related endpoints
type GeoIP struct {
Common
}
// GetMetadata returns the GeoIP metadata including available country codes
// GetMetadata returns the available country codes for the filter UI
func (c *GeoIP) GetMetadata(g *gin.Context) {
_, _, ok := c.handleSession(g)
if !ok {
return
}
// get geoip instance
geo, err := geoip.Instance()
if ok := c.handleErrors(g, err); !ok {
return
}
codes := ipdata.Get().CountryCodes()
// get metadata
metadata := geo.GetMetadata()
if metadata == nil {
c.Response.BadRequest(g)
return
}
c.Response.OK(g, metadata)
c.Response.OK(g, gin.H{
"country_codes": codes,
"countries": ipdata.Get().Countries(),
"asn_available": ipdata.Get().ASNLoaded(),
})
}
// Lookup performs a GeoIP lookup for the provided IP address
// Lookup performs a country lookup for the provided IP address
func (c *GeoIP) Lookup(g *gin.Context) {
_, _, ok := c.handleSession(g)
if !ok {
return
}
// get ip from query parameter
ip := g.Query("ip")
if ip == "" {
c.Response.BadRequest(g)
return
}
// get geoip instance
geo, err := geoip.Instance()
if ok := c.handleErrors(g, err); !ok {
return
}
countryCode, found := ipdata.Get().LookupCountry(ip)
// perform lookup
countryCode, found := geo.Lookup(ip)
// return result
result := gin.H{
"ip": ip,
"found": found,
}
if found {
result["country_code"] = countryCode
}
+137
View File
@@ -0,0 +1,137 @@
package controller
import (
"strconv"
"strings"
"github.com/gin-gonic/gin"
"github.com/phishingclub/phishingclub/ipdata"
"github.com/phishingclub/phishingclub/service"
)
// IPData is a controller for the country and ASN data packages
type IPData struct {
Common
IPDataService *service.IPData
}
// Status returns the state of both data packages
func (c *IPData) Status(g *gin.Context) {
session, _, ok := c.handleSession(g)
if !ok {
return
}
status, err := c.IPDataService.Status(g, session)
if ok := c.handleErrors(g, err); !ok {
return
}
c.Response.OK(g, gin.H{"packages": status})
}
// Download fetches and installs the package named in the path
func (c *IPData) Download(g *gin.Context) {
session, _, ok := c.handleSession(g)
if !ok {
return
}
kind := g.Param("kind")
err := c.IPDataService.Download(g, session, kind)
if ok := c.handleErrors(g, err); !ok {
return
}
c.Response.OK(g, nil)
}
// Remove deletes the installed package named in the path
func (c *IPData) Remove(g *gin.Context) {
session, _, ok := c.handleSession(g)
if !ok {
return
}
kind := g.Param("kind")
err := c.IPDataService.Remove(g, session, kind)
if ok := c.handleErrors(g, err); !ok {
return
}
c.Response.OK(g, nil)
}
// SearchASN returns ASNs matching the query for the filter typeahead
func (c *IPData) SearchASN(g *gin.Context) {
_, _, ok := c.handleSession(g)
if !ok {
return
}
q := g.Query("q")
limit := 25
if v := g.Query("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil {
limit = n
}
}
results := ipdata.Get().SearchASN(q, limit)
c.Response.OK(g, gin.H{"results": results})
}
// LookupASN returns the autonomous systems that announce the given IP address
func (c *IPData) LookupASN(g *gin.Context) {
_, _, ok := c.handleSession(g)
if !ok {
return
}
ip := g.Query("ip")
if ip == "" {
c.Response.BadRequest(g)
return
}
store := ipdata.Get()
results := store.LookupASNDetails(ip)
c.Response.OK(g, gin.H{
"ip": ip,
"available": store.ASNLoaded(),
"found": len(results) > 0,
"results": results,
})
}
// ResolveASNRequest is the body for resolving configured ASNs to names.
type ResolveASNRequest struct {
Asns []string `json:"asns"`
}
// ResolveASNs returns the details of the given ASNs that exist in the dataset.
// ASNs that are absent are left out, which lets the UI flag orphaned entries.
func (c *IPData) ResolveASNs(g *gin.Context) {
_, _, ok := c.handleSession(g)
if !ok {
return
}
var req ResolveASNRequest
if ok := c.handleParseRequest(g, &req); !ok {
return
}
nums := make([]uint32, 0, len(req.Asns))
for _, s := range req.Asns {
if n, ok := parseASNInput(s); ok {
nums = append(nums, n)
}
}
results := ipdata.Get().ResolveASNs(nums)
c.Response.OK(g, gin.H{"results": results})
}
// parseASNInput parses one ASN string with an optional AS or ASN prefix.
func parseASNInput(s string) (uint32, bool) {
v := strings.ToLower(strings.TrimSpace(s))
v = strings.TrimPrefix(v, "asn")
v = strings.TrimPrefix(v, "as")
v = strings.TrimSpace(v)
if v == "" {
return 0, false
}
n, err := strconv.ParseUint(v, 10, 32)
if err != nil {
return 0, false
}
return uint32(n), true
}
+1
View File
@@ -21,6 +21,7 @@ type AllowDeny struct {
Cidrs string `gorm:"not null;default:''"`
JA4Fingerprints string `gorm:"not null;default:''"`
CountryCodes string `gorm:"not null;default:''"`
Asns string `gorm:"not null;default:''"`
Headers string `gorm:"not null;default:''"`
Allowed bool `gorm:"not null;"`
}
-185
View File
@@ -1,185 +0,0 @@
package geoip
import (
"encoding/json"
"fmt"
"net"
"strings"
"sync"
"github.com/phishingclub/phishingclub/embedded"
)
// countryData represents the structure from embedded geoip files
type countryData struct {
Code string `json:"code"`
Name string `json:"name"`
IPv4 []string `json:"ipv4"`
IPv6 []string `json:"ipv6"`
}
// metadata represents the geoip metadata file
type Metadata struct {
Generated string `json:"generated"`
Source string `json:"source"`
License string `json:"license"`
Countries int `json:"countries"`
CountryCodes []string `json:"country_codes"`
}
// ipRange represents a single IP range with its country code
type ipRange struct {
network *net.IPNet
countryCode string
}
// GeoIP provides IP to country lookup functionality
type GeoIP struct {
ranges []ipRange
metadata *Metadata
mu sync.RWMutex
}
var (
instance *GeoIP
once sync.Once
initErr error
)
// Instance returns the singleton GeoIP instance
func Instance() (*GeoIP, error) {
once.Do(func() {
instance, initErr = New()
})
return instance, initErr
}
// New creates a new GeoIP instance by loading embedded data
func New() (*GeoIP, error) {
geo := &GeoIP{
ranges: make([]ipRange, 0),
}
// load metadata
if err := geo.loadMetadata(); err != nil {
return nil, fmt.Errorf("failed to load metadata: %w", err)
}
// load all country data
if err := geo.loadAllCountries(); err != nil {
return nil, fmt.Errorf("failed to load countries: %w", err)
}
return geo, nil
}
// loadMetadata loads the metadata.json file
func (g *GeoIP) loadMetadata() error {
data, err := embedded.GeoIPData.ReadFile("geoip/metadata.json")
if err != nil {
return fmt.Errorf("failed to read metadata: %w", err)
}
var meta Metadata
if err := json.Unmarshal(data, &meta); err != nil {
return fmt.Errorf("failed to parse metadata: %w", err)
}
g.metadata = &meta
return nil
}
// loadAllCountries loads all country IP ranges
func (g *GeoIP) loadAllCountries() error {
if g.metadata == nil {
return fmt.Errorf("metadata not loaded")
}
for _, code := range g.metadata.CountryCodes {
if err := g.loadCountry(code); err != nil {
// log warning but continue - some countries may not have data
continue
}
}
return nil
}
// loadCountry loads IP ranges for a single country
func (g *GeoIP) loadCountry(countryCode string) error {
filename := fmt.Sprintf("geoip/%s.json", strings.ToLower(countryCode))
data, err := embedded.GeoIPData.ReadFile(filename)
if err != nil {
return fmt.Errorf("failed to read %s: %w", countryCode, err)
}
var country countryData
if err := json.Unmarshal(data, &country); err != nil {
return fmt.Errorf("failed to parse %s: %w", countryCode, err)
}
// add all IPv4 ranges
for _, cidr := range country.IPv4 {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
continue // skip invalid entries
}
g.ranges = append(g.ranges, ipRange{
network: network,
countryCode: country.Code,
})
}
// add all IPv6 ranges
for _, cidr := range country.IPv6 {
_, network, err := net.ParseCIDR(cidr)
if err != nil {
continue // skip invalid entries
}
g.ranges = append(g.ranges, ipRange{
network: network,
countryCode: country.Code,
})
}
return nil
}
// Lookup finds the country code for an IP address
// returns (countryCode, found)
func (g *GeoIP) Lookup(ipStr string) (string, bool) {
g.mu.RLock()
defer g.mu.RUnlock()
ip := net.ParseIP(ipStr)
if ip == nil {
return "", false
}
// linear search through ranges
// for better performance, consider using a trie or radix tree
for _, r := range g.ranges {
if r.network.Contains(ip) {
return r.countryCode, true
}
}
return "", false
}
// GetMetadata returns the GeoIP metadata
func (g *GeoIP) GetMetadata() *Metadata {
g.mu.RLock()
defer g.mu.RUnlock()
return g.metadata
}
// GetCountryCodes returns a list of all available country codes
func (g *GeoIP) GetCountryCodes() []string {
g.mu.RLock()
defer g.mu.RUnlock()
if g.metadata == nil {
return []string{}
}
return g.metadata.CountryCodes
}
+515
View File
@@ -0,0 +1,515 @@
// Package ipdata loads the IP to country and IP to ASN data used by the
// allow and deny filters. It reads two sources: the country data embedded in
// the binary, and packages the operator downloads into the data directory. A
// downloaded package wins over the embedded copy. ASN data only exists as a
// download.
//
// Prefixes are held in memory as fixed compact records and looked up by
// longest prefix. The store can be rebuilt at runtime after a download without
// a restart.
package ipdata
import (
"encoding/binary"
"fmt"
"net/netip"
"sort"
"strings"
"sync"
"go.uber.org/zap"
)
const (
// KindGeoIP is the country data package name.
KindGeoIP = "geoip"
// KindASN is the autonomous system data package name.
KindASN = "asn"
)
// rec4 is one IPv4 prefix. val indexes into a dataset value table.
type rec4 struct {
addr uint32
bits uint8
val uint32
}
// rec6 is one IPv6 prefix. val indexes into a dataset value table.
type rec6 struct {
addr [16]byte
bits uint8
val uint32
}
// index holds the sorted prefixes of one dataset for lookup.
type index struct {
v4 []rec4
v6 []rec6
}
// add parses a CIDR, masks off host bits and appends it under the value.
// Bad or reserved input is skipped. It returns whether the prefix was kept.
func (x *index) add(cidr string, val uint32) bool {
p, err := netip.ParsePrefix(strings.TrimSpace(cidr))
if err != nil {
return false
}
p = p.Masked()
a := p.Addr()
if a.Is4() {
b := a.As4()
x.v4 = append(x.v4, rec4{binary.BigEndian.Uint32(b[:]), uint8(p.Bits()), val})
return true
}
if a.Is6() && !a.Is4In6() {
x.v6 = append(x.v6, rec6{a.As16(), uint8(p.Bits()), val})
return true
}
return false
}
// sortRecords orders the prefixes by address then prefix length so lookup can
// binary search each length.
func (x *index) sortRecords() {
sort.Slice(x.v4, func(i, j int) bool {
if x.v4[i].addr != x.v4[j].addr {
return x.v4[i].addr < x.v4[j].addr
}
return x.v4[i].bits < x.v4[j].bits
})
sort.Slice(x.v6, func(i, j int) bool {
if c := byteCmp(x.v6[i].addr, x.v6[j].addr); c != 0 {
return c < 0
}
return x.v6[i].bits < x.v6[j].bits
})
}
// lookup returns the value indexes of every prefix that matches the address at
// the longest matching prefix length. More than one is returned when several
// entries announce the same prefix, which happens with ASN data.
func (x *index) lookup(ip netip.Addr) []uint32 {
if ip.Is4() {
b := ip.As4()
return x.lookup4(binary.BigEndian.Uint32(b[:]))
}
if ip.Is4In6() {
b := ip.Unmap().As4()
return x.lookup4(binary.BigEndian.Uint32(b[:]))
}
return x.lookup6(ip.As16())
}
func (x *index) lookup4(ip uint32) []uint32 {
for bits := 32; bits >= 0; bits-- {
var mask uint32 = 0xffffffff
if bits < 32 {
mask <<= uint(32 - bits)
}
if bits == 0 {
mask = 0
}
masked := ip & mask
lo := sort.Search(len(x.v4), func(i int) bool { return x.v4[i].addr >= masked })
var vals []uint32
for i := lo; i < len(x.v4) && x.v4[i].addr == masked; i++ {
if x.v4[i].bits == uint8(bits) {
vals = append(vals, x.v4[i].val)
}
}
if len(vals) > 0 {
return vals
}
}
return nil
}
func (x *index) lookup6(ip [16]byte) []uint32 {
for bits := 128; bits >= 0; bits-- {
masked := maskV6(ip, bits)
lo := sort.Search(len(x.v6), func(i int) bool { return byteCmp(x.v6[i].addr, masked) >= 0 })
var vals []uint32
for i := lo; i < len(x.v6) && x.v6[i].addr == masked; i++ {
if x.v6[i].bits == uint8(bits) {
vals = append(vals, x.v6[i].val)
}
}
if len(vals) > 0 {
return vals
}
}
return nil
}
func maskV6(a [16]byte, bits int) [16]byte {
var out [16]byte
full := bits / 8
rem := bits % 8
copy(out[:full], a[:full])
if rem > 0 && full < 16 {
out[full] = a[full] & (byte(0xff) << uint(8-rem))
}
return out
}
func byteCmp(a, b [16]byte) int {
for i := 0; i < 16; i++ {
if a[i] != b[i] {
if a[i] < b[i] {
return -1
}
return 1
}
}
return 0
}
// Info describes a loaded dataset for the status endpoint.
type Info struct {
Name string `json:"name"`
Source string `json:"source"`
Version string `json:"version"`
Created string `json:"created"`
License string `json:"license"`
Entries int `json:"entries"`
IPv4Prefixes int `json:"ipv4Prefixes"`
IPv6Prefixes int `json:"ipv6Prefixes"`
ContentHash string `json:"contentHash"`
Downloaded bool `json:"downloaded"`
}
// geoDataset maps prefixes to country codes.
type geoDataset struct {
idx index
codes []string
names []string
info Info
}
// asnDataset maps prefixes to autonomous systems.
type asnDataset struct {
idx index
nums []uint32
handles []string
names []string
countries []string
byNum map[uint32]int
info Info
}
// Store holds the loaded datasets and swaps them safely on reload.
type Store struct {
mu sync.RWMutex
dataDir string
logger *zap.SugaredLogger
geo *geoDataset
asn *asnDataset
}
var std *Store
// Init loads the datasets from the embedded data and the data directory and
// installs the package level store. It is called once at startup. A failure to
// load a package is logged and leaves that dataset empty rather than stopping
// the server.
func Init(dataDir string, logger *zap.SugaredLogger) *Store {
s := &Store{dataDir: dataDir, logger: logger}
s.reloadGeo()
s.reloadASN()
std = s
return s
}
// Get returns the package level store. Its methods are safe to call on a nil
// store and on empty datasets.
func Get() *Store { return std }
// reloadGeo builds the country dataset from a downloaded package when present,
// otherwise from the embedded data.
func (s *Store) reloadGeo() {
if ds, err := s.loadGeoPackage(); err == nil {
s.setGeo(ds)
return
} else if s.logger != nil {
s.logger.Debugw("no downloaded geoip package, using embedded", "error", err)
}
ds, err := s.loadEmbeddedGeo()
if err != nil {
if s.logger != nil {
s.logger.Errorw("failed to load embedded geoip data", "error", err)
}
return
}
s.setGeo(ds)
}
// reloadASN builds the ASN dataset from a downloaded package. There is no
// embedded fallback, so a missing package leaves ASN lookups empty.
func (s *Store) reloadASN() {
ds, err := s.loadASNPackage()
if err != nil {
s.setASN(nil)
if s.logger != nil {
s.logger.Debugw("no downloaded asn package", "error", err)
}
return
}
s.setASN(ds)
}
func (s *Store) setGeo(ds *geoDataset) {
s.mu.Lock()
s.geo = ds
s.mu.Unlock()
}
func (s *Store) setASN(ds *asnDataset) {
s.mu.Lock()
s.asn = ds
s.mu.Unlock()
}
// LookupCountry returns the country code for an IP and whether one was found.
func (s *Store) LookupCountry(ipStr string) (string, bool) {
if s == nil {
return "", false
}
ip, err := netip.ParseAddr(ipStr)
if err != nil {
return "", false
}
s.mu.RLock()
geo := s.geo
s.mu.RUnlock()
if geo == nil {
return "", false
}
vals := geo.idx.lookup(ip)
if len(vals) == 0 {
return "", false
}
return geo.codes[vals[0]], true
}
// LookupASNs returns every autonomous system number that announces the longest
// prefix containing the IP. Usually one, sometimes several.
func (s *Store) LookupASNs(ipStr string) []uint32 {
if s == nil {
return nil
}
ip, err := netip.ParseAddr(ipStr)
if err != nil {
return nil
}
s.mu.RLock()
asn := s.asn
s.mu.RUnlock()
if asn == nil {
return nil
}
vals := asn.idx.lookup(ip)
if len(vals) == 0 {
return nil
}
out := make([]uint32, 0, len(vals))
for _, v := range vals {
out = append(out, asn.nums[v])
}
return out
}
// LookupASNDetails returns the details of every ASN that announces the longest
// prefix containing the IP. Used by the ASN lookup tool.
func (s *Store) LookupASNDetails(ipStr string) []ASN {
nums := s.LookupASNs(ipStr)
if len(nums) == 0 {
return nil
}
return s.ResolveASNs(nums)
}
// Country is one entry of the country list.
type Country struct {
Code string `json:"code"`
Name string `json:"name"`
}
// Countries returns the available country codes and names, sorted by code.
func (s *Store) Countries() []Country {
if s == nil {
return nil
}
s.mu.RLock()
geo := s.geo
s.mu.RUnlock()
if geo == nil {
return nil
}
out := make([]Country, len(geo.codes))
for i := range geo.codes {
out[i] = Country{Code: geo.codes[i], Name: geo.names[i]}
}
return out
}
// CountryCodes returns just the available country codes, sorted.
func (s *Store) CountryCodes() []string {
if s == nil {
return nil
}
s.mu.RLock()
geo := s.geo
s.mu.RUnlock()
if geo == nil {
return nil
}
out := make([]string, len(geo.codes))
copy(out, geo.codes)
return out
}
// ASN is one entry of an ASN search or resolve result.
type ASN struct {
ASN uint32 `json:"asn"`
Handle string `json:"handle"`
Name string `json:"name"`
Country string `json:"country"`
}
// ASNLoaded reports whether ASN data is currently loaded and usable.
func (s *Store) ASNLoaded() bool {
if s == nil {
return false
}
s.mu.RLock()
defer s.mu.RUnlock()
return s.asn != nil
}
// HasASN reports whether the ASN exists in the loaded dataset.
func (s *Store) HasASN(num uint32) bool {
if s == nil {
return false
}
s.mu.RLock()
asn := s.asn
s.mu.RUnlock()
if asn == nil {
return false
}
_, ok := asn.byNum[num]
return ok
}
// ResolveASNs returns the details of the given ASN numbers that exist. Numbers
// that are absent from the dataset are left out, which lets the caller show a
// warning for orphaned filter entries.
func (s *Store) ResolveASNs(nums []uint32) []ASN {
if s == nil {
return nil
}
s.mu.RLock()
asn := s.asn
s.mu.RUnlock()
if asn == nil {
return nil
}
out := []ASN{}
for _, n := range nums {
if i, ok := asn.byNum[n]; ok {
out = append(out, ASN{ASN: asn.nums[i], Handle: asn.handles[i], Name: asn.names[i], Country: asn.countries[i]})
}
}
return out
}
// SearchASN returns ASNs matching the query by number, name or handle, up to
// limit results. Matches are ranked so exact and prefix matches come before a
// match in the middle of a name, so a query like "M247" lists the M247 systems
// first.
func (s *Store) SearchASN(query string, limit int) []ASN {
if s == nil {
return nil
}
s.mu.RLock()
asn := s.asn
s.mu.RUnlock()
if asn == nil || query == "" {
return nil
}
if limit <= 0 || limit > 100 {
limit = 25
}
q := strings.ToLower(strings.TrimSpace(query))
type scored struct {
a ASN
score int
}
var matches []scored
for i := range asn.nums {
numStr := fmt.Sprintf("%d", asn.nums[i])
name := strings.ToLower(asn.names[i])
handle := strings.ToLower(asn.handles[i])
score := -1
switch {
case numStr == q || handle == q || name == q:
score = 0
case strings.HasPrefix(numStr, q) || strings.HasPrefix(handle, q) || strings.HasPrefix(name, q):
score = 1
case strings.Contains(name, q) || strings.Contains(handle, q):
score = 2
}
if score >= 0 {
matches = append(matches, scored{
a: ASN{ASN: asn.nums[i], Handle: asn.handles[i], Name: asn.names[i], Country: asn.countries[i]},
score: score,
})
}
}
// best score first, then by ascending ASN number for a stable order
sort.SliceStable(matches, func(i, j int) bool {
if matches[i].score != matches[j].score {
return matches[i].score < matches[j].score
}
return matches[i].a.ASN < matches[j].a.ASN
})
out := []ASN{}
for i := 0; i < len(matches) && i < limit; i++ {
out = append(out, matches[i].a)
}
return out
}
// Status returns the state of both packages for the settings screen.
func (s *Store) Status() map[string]Info {
out := map[string]Info{}
if s == nil {
return out
}
s.mu.RLock()
defer s.mu.RUnlock()
if s.geo != nil {
out[KindGeoIP] = s.geo.info
}
if s.asn != nil {
out[KindASN] = s.asn.info
}
return out
}
// Reload rebuilds one dataset from disk after a download or a removal.
func (s *Store) Reload(kind string) {
if s == nil {
return
}
switch kind {
case KindGeoIP:
s.reloadGeo()
case KindASN:
s.reloadASN()
}
}
+124
View File
@@ -0,0 +1,124 @@
package ipdata
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"go.uber.org/zap"
)
func TestLongestPrefixAndMultiOrigin(t *testing.T) {
// two countries, one nested prefix to prove longest match wins
geo := buildGeo([]geoEntry{
{Code: "US", Name: "United States", IPv4: []string{"8.0.0.0/8"}},
{Code: "DE", Name: "Germany", IPv4: []string{"8.8.0.0/16"}, IPv6: []string{"2a01:4f8::/32"}},
})
s := &Store{geo: geo}
if code, ok := s.LookupCountry("8.8.8.8"); !ok || code != "DE" {
t.Fatalf("8.8.8.8 got %q %v, want DE", code, ok)
}
if code, ok := s.LookupCountry("8.9.0.1"); !ok || code != "US" {
t.Fatalf("8.9.0.1 got %q %v, want US", code, ok)
}
if code, ok := s.LookupCountry("2a01:4f8::1"); !ok || code != "DE" {
t.Fatalf("ipv6 got %q %v, want DE", code, ok)
}
if _, ok := s.LookupCountry("1.2.3.4"); ok {
t.Fatal("1.2.3.4 should not match")
}
// same prefix announced by two ASNs must return both
asn := buildASN([]asnEntry{
{ASN: 64500, Handle: "A", Name: "Alpha", IPv4: []string{"203.0.113.0/24"}},
{ASN: 64501, Handle: "B", Name: "Beta", IPv4: []string{"203.0.113.0/24"}},
{ASN: 15169, Handle: "GOOGLE", Name: "Google LLC", IPv4: []string{"8.8.8.0/24"}},
})
s.asn = asn
got := s.LookupASNs("203.0.113.9")
sort.Slice(got, func(i, j int) bool { return got[i] < got[j] })
if !reflect.DeepEqual(got, []uint32{64500, 64501}) {
t.Fatalf("multi origin got %v, want [64500 64501]", got)
}
if got := s.LookupASNs("8.8.8.8"); len(got) != 1 || got[0] != 15169 {
t.Fatalf("8.8.8.8 asn got %v, want [15169]", got)
}
if !s.HasASN(15169) || s.HasASN(1) {
t.Fatal("HasASN wrong")
}
// search and resolve
if res := s.SearchASN("goog", 10); len(res) != 1 || res[0].ASN != 15169 {
t.Fatalf("search got %v", res)
}
if res := s.ResolveASNs([]uint32{15169, 99999}); len(res) != 1 || res[0].ASN != 15169 {
t.Fatalf("resolve got %v, want only 15169", res)
}
}
func TestNilStoreSafe(t *testing.T) {
var s *Store
if _, ok := s.LookupCountry("8.8.8.8"); ok {
t.Fatal("nil store should not match")
}
if s.LookupASNs("8.8.8.8") != nil {
t.Fatal("nil store asn should be nil")
}
if s.HasASN(1) {
t.Fatal("nil store HasASN should be false")
}
}
func TestLoadPackageAndSearchRanking(t *testing.T) {
root := t.TempDir()
dir := filepath.Join(root, "ipdata", "asn")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
entries := []byte(`[
{"asn":9009,"handle":"M247","name":"M247 Europe SRL","country":"RO","ipv4":["5.62.0.0/16"],"ipv6":[]},
{"asn":329035,"handle":"M247AI-AS-US","name":"M247 LLC","country":"US","ipv4":["23.19.0.0/16"],"ipv6":[]},
{"asn":15169,"handle":"GOOGLE","name":"Google LLC","country":"US","ipv4":["8.8.8.0/24"],"ipv6":[]}
]`)
sum := sha256.Sum256(entries)
info := map[string]any{
"format": 1, "name": "asn", "version": "test", "created": "2026-01-01T00:00:00Z",
"source": "test", "license": "CC0-1.0", "entries": 3,
"ipv4_prefixes": 3, "ipv6_prefixes": 0, "content_hash": hex.EncodeToString(sum[:]),
}
infoBytes, _ := json.Marshal(info)
if err := os.WriteFile(filepath.Join(dir, "package.json"), infoBytes, 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "entries.json"), entries, 0o644); err != nil {
t.Fatal(err)
}
s := Init(root+"/", zap.NewNop().Sugar())
// a name search returns the matching systems, exact handle match first
res := s.SearchASN("M247", 10)
if len(res) != 2 {
t.Fatalf("search M247 got %d results, want 2: %+v", len(res), res)
}
if res[0].ASN != 9009 {
t.Fatalf("expected handle-exact AS9009 first, got %d", res[0].ASN)
}
// number search
if r := s.SearchASN("15169", 10); len(r) != 1 || r[0].Handle != "GOOGLE" {
t.Fatalf("number search got %+v", r)
}
// ip to asn still works from the loaded package
if got := s.LookupASNDetails("5.62.0.1"); len(got) != 1 || got[0].ASN != 9009 {
t.Fatalf("ip lookup got %+v", got)
}
}
+263
View File
@@ -0,0 +1,263 @@
package ipdata
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"github.com/phishingclub/phishingclub/embedded"
)
// packageFormat is the format version this reader understands.
const packageFormat = 1
// FilePackage and FileEntries are the two files inside a downloaded package.
const (
filePackage = "package.json"
fileEntries = "entries.json"
)
// packageInfo mirrors package.json in a downloaded package.
type packageInfo struct {
Format int `json:"format"`
Name string `json:"name"`
Version string `json:"version"`
Created string `json:"created"`
Source string `json:"source"`
License string `json:"license"`
Entries int `json:"entries"`
IPv4Prefixes int `json:"ipv4_prefixes"`
IPv6Prefixes int `json:"ipv6_prefixes"`
ContentHash string `json:"content_hash"`
}
type geoEntry struct {
Code string `json:"code"`
Name string `json:"name"`
IPv4 []string `json:"ipv4"`
IPv6 []string `json:"ipv6"`
}
type asnEntry struct {
ASN uint32 `json:"asn"`
Handle string `json:"handle"`
Name string `json:"name"`
Country string `json:"country"`
IPv4 []string `json:"ipv4"`
IPv6 []string `json:"ipv6"`
}
// packageDir is where a downloaded package of the given kind is extracted.
func (s *Store) packageDir(kind string) string {
return filepath.Join(s.dataDir, "ipdata", kind)
}
// PackageDir returns the directory a package of the given kind is installed to.
func (s *Store) PackageDir(kind string) string {
return s.packageDir(kind)
}
// DataRoot returns the directory that holds all installed packages.
func (s *Store) DataRoot() string {
return filepath.Join(s.dataDir, "ipdata")
}
// Validate checks that a directory holds a readable package of the kind, so a
// download can be rejected before it is moved into place.
func Validate(dir, kind string) error {
_, entries, err := readPackage(dir, kind)
if err != nil {
return err
}
switch kind {
case KindGeoIP:
var l []geoEntry
if err := json.Unmarshal(entries, &l); err != nil {
return fmt.Errorf("parse %s: %w", fileEntries, err)
}
if len(l) == 0 {
return fmt.Errorf("geoip package has no entries")
}
case KindASN:
var l []asnEntry
if err := json.Unmarshal(entries, &l); err != nil {
return fmt.Errorf("parse %s: %w", fileEntries, err)
}
if len(l) == 0 {
return fmt.Errorf("asn package has no entries")
}
default:
return fmt.Errorf("unknown package kind %q", kind)
}
return nil
}
// readPackage reads and verifies a downloaded package directory.
func readPackage(dir, wantName string) (*packageInfo, []byte, error) {
infoBytes, err := os.ReadFile(filepath.Join(dir, filePackage))
if err != nil {
return nil, nil, err
}
var info packageInfo
if err := json.Unmarshal(infoBytes, &info); err != nil {
return nil, nil, fmt.Errorf("parse %s: %w", filePackage, err)
}
if info.Format != packageFormat {
return nil, nil, fmt.Errorf("package format %d, expected %d", info.Format, packageFormat)
}
if info.Name != wantName {
return nil, nil, fmt.Errorf("package is %q, expected %q", info.Name, wantName)
}
entries, err := os.ReadFile(filepath.Join(dir, fileEntries))
if err != nil {
return nil, nil, err
}
sum := sha256.Sum256(entries)
if hex.EncodeToString(sum[:]) != info.ContentHash {
return nil, nil, fmt.Errorf("content hash does not match %s", filePackage)
}
return &info, entries, nil
}
// loadGeoPackage builds the country dataset from a downloaded package.
func (s *Store) loadGeoPackage() (*geoDataset, error) {
info, entries, err := readPackage(s.packageDir(KindGeoIP), KindGeoIP)
if err != nil {
return nil, err
}
var list []geoEntry
if err := json.Unmarshal(entries, &list); err != nil {
return nil, fmt.Errorf("parse %s: %w", fileEntries, err)
}
ds := buildGeo(list)
ds.info = Info{
Name: KindGeoIP, Source: info.Source, Version: info.Version, Created: info.Created,
License: info.License, Entries: len(list),
IPv4Prefixes: len(ds.idx.v4), IPv6Prefixes: len(ds.idx.v6),
ContentHash: info.ContentHash, Downloaded: true,
}
return ds, nil
}
// loadASNPackage builds the ASN dataset from a downloaded package.
func (s *Store) loadASNPackage() (*asnDataset, error) {
info, entries, err := readPackage(s.packageDir(KindASN), KindASN)
if err != nil {
return nil, err
}
var list []asnEntry
if err := json.Unmarshal(entries, &list); err != nil {
return nil, fmt.Errorf("parse %s: %w", fileEntries, err)
}
ds := buildASN(list)
ds.info = Info{
Name: KindASN, Source: info.Source, Version: info.Version, Created: info.Created,
License: info.License, Entries: len(list),
IPv4Prefixes: len(ds.idx.v4), IPv6Prefixes: len(ds.idx.v6),
ContentHash: info.ContentHash, Downloaded: true,
}
return ds, nil
}
// embeddedMetadata mirrors the embedded geoip metadata.json.
type embeddedMetadata struct {
Generated string `json:"generated"`
Source string `json:"source"`
License string `json:"license"`
CountryCodes []string `json:"country_codes"`
}
// embeddedCountry mirrors an embedded per country file.
type embeddedCountry struct {
Code string `json:"code"`
Name string `json:"name"`
IPv4 []string `json:"ipv4"`
IPv6 []string `json:"ipv6"`
}
// loadEmbeddedGeo builds the country dataset from the data embedded in the
// binary. This keeps the current embedded format untouched.
func (s *Store) loadEmbeddedGeo() (*geoDataset, error) {
metaBytes, err := embedded.GeoIPData.ReadFile("geoip/metadata.json")
if err != nil {
return nil, err
}
var meta embeddedMetadata
if err := json.Unmarshal(metaBytes, &meta); err != nil {
return nil, fmt.Errorf("parse embedded metadata: %w", err)
}
list := make([]geoEntry, 0, len(meta.CountryCodes))
for _, code := range meta.CountryCodes {
name := fmt.Sprintf("geoip/%s.json", strings.ToLower(code))
data, err := embedded.GeoIPData.ReadFile(name)
if err != nil {
continue
}
var c embeddedCountry
if err := json.Unmarshal(data, &c); err != nil {
continue
}
list = append(list, geoEntry{Code: c.Code, Name: c.Name, IPv4: c.IPv4, IPv6: c.IPv6})
}
ds := buildGeo(list)
ds.info = Info{
Name: KindGeoIP, Source: meta.Source, Version: "embedded", Created: meta.Generated,
License: meta.License, Entries: len(list),
IPv4Prefixes: len(ds.idx.v4), IPv6Prefixes: len(ds.idx.v6),
Downloaded: false,
}
return ds, nil
}
// buildGeo turns country entries into a sorted dataset.
func buildGeo(list []geoEntry) *geoDataset {
sort.Slice(list, func(i, j int) bool { return list[i].Code < list[j].Code })
ds := &geoDataset{
codes: make([]string, len(list)),
names: make([]string, len(list)),
}
for i, e := range list {
ds.codes[i] = e.Code
ds.names[i] = e.Name
for _, c := range e.IPv4 {
ds.idx.add(c, uint32(i))
}
for _, c := range e.IPv6 {
ds.idx.add(c, uint32(i))
}
}
ds.idx.sortRecords()
return ds
}
// buildASN turns ASN entries into a sorted dataset.
func buildASN(list []asnEntry) *asnDataset {
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
ds := &asnDataset{
nums: make([]uint32, len(list)),
handles: make([]string, len(list)),
names: make([]string, len(list)),
countries: make([]string, len(list)),
byNum: make(map[uint32]int, len(list)),
}
for i, e := range list {
ds.nums[i] = e.ASN
ds.handles[i] = e.Handle
ds.names[i] = e.Name
ds.countries[i] = e.Country
ds.byNum[e.ASN] = i
for _, c := range e.IPv4 {
ds.idx.add(c, uint32(i))
}
for _, c := range e.IPv6 {
ds.idx.add(c, uint32(i))
}
}
ds.idx.sortRecords()
return ds
}
+4
View File
@@ -27,6 +27,7 @@ import (
"github.com/phishingclub/phishingclub/database"
"github.com/phishingclub/phishingclub/errs"
"github.com/phishingclub/phishingclub/install"
"github.com/phishingclub/phishingclub/ipdata"
"github.com/phishingclub/phishingclub/middleware"
"github.com/phishingclub/phishingclub/model"
"github.com/phishingclub/phishingclub/repository"
@@ -235,6 +236,9 @@ func main() {
logger.Errorw("failed to setup certmagic", "error", err)
return
}
// load the country and ASN data into memory. a downloaded package in the
// data directory wins over the embedded country data.
ipdata.Init(*flagFilePath, logger)
// setup services, middleware and controllers
services := app.NewServices(
db,
+108 -15
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"net"
"regexp"
"strconv"
"strings"
"time"
@@ -26,6 +27,7 @@ type AllowDeny struct {
Cidrs nullable.Nullable[vo.IPNetSlice] `json:"cidrs"`
JA4Fingerprints nullable.Nullable[string] `json:"ja4Fingerprints"`
CountryCodes nullable.Nullable[string] `json:"countryCodes"`
Asns nullable.Nullable[string] `json:"asns"`
Headers nullable.Nullable[string] `json:"headers"`
Allowed nullable.Nullable[bool] `json:"allowed"`
CompanyID nullable.Nullable[uuid.UUID] `json:"companyID"`
@@ -62,6 +64,13 @@ func (r *AllowDeny) Validate() error {
}
}
hasASNs := false
if r.Asns.IsSpecified() {
if asns, err := r.Asns.Get(); err == nil && asns != "" {
hasASNs = true
}
}
hasHeaders := false
if r.Headers.IsSpecified() {
if headers, err := r.Headers.Get(); err == nil && headers != "" {
@@ -69,12 +78,24 @@ func (r *AllowDeny) Validate() error {
}
}
if !hasCidrs && !hasJA4 && !hasCountryCodes && !hasHeaders {
if !hasCidrs && !hasJA4 && !hasCountryCodes && !hasASNs && !hasHeaders {
return errs.NewValidationError(
errors.New("at least one of CIDRs, JA4 fingerprints, country codes, or headers must be provided"),
errors.New("at least one of CIDRs, JA4 fingerprints, country codes, ASNs, or headers must be provided"),
)
}
// each ASN line must be a number, optionally prefixed with AS
if hasASNs {
asns, _ := r.Asns.Get()
for _, line := range parseCountryCodes(asns) {
if _, ok := parseASN(line); !ok {
return errs.NewValidationError(
fmt.Errorf("invalid ASN: %s", line),
)
}
}
}
return nil
}
@@ -117,6 +138,12 @@ func (r *AllowDeny) ToDBMap() map[string]any {
m["country_codes"] = codes
}
}
if r.Asns.IsSpecified() {
m["asns"] = ""
if asns, err := r.Asns.Get(); err == nil {
m["asns"] = asns
}
}
if r.Headers.IsSpecified() {
m["headers"] = ""
if headers, err := r.Headers.Get(); err == nil {
@@ -327,28 +354,19 @@ func isSpace(b byte) bool {
// IsCountryAllowed checks if a country code is allowed based on the filter rules
func (r *AllowDeny) IsCountryAllowed(countryCode string) bool {
if countryCode == "" {
// if no country code available, skip country check
return true
}
isTypeAllowList := r.Allowed.MustGet()
// get country codes list
countryCodesStr, err := r.CountryCodes.Get()
if err != nil || countryCodesStr == "" {
// if no country codes configured, skip country check
// no country filter configured, this dimension does not restrict
return true
}
// if country code is empty but we have country filters configured
if countryCode == "" {
// in allow list mode: unknown country should be denied
// in deny list mode: unknown country should be allowed
if isTypeAllowList {
return false
}
return true
// a country filter is configured but the visitor country is unknown.
// allow list denies the unknown visitor, deny list allows it.
return !isTypeAllowList
}
// parse country codes (newline separated)
@@ -382,6 +400,81 @@ func (r *AllowDeny) IsCountryAllowed(countryCode string) bool {
return true
}
// IsASNAllowed checks if the autonomous systems that announce the visitor IP
// are allowed based on the filter rules. asns is every ASN that announces the
// longest prefix containing the IP, usually one, sometimes several.
func (r *AllowDeny) IsASNAllowed(asns []uint32) bool {
isTypeAllowList := r.Allowed.MustGet()
// get asn list
asnStr, err := r.Asns.Get()
if err != nil || asnStr == "" {
// no asn filter configured, this dimension does not restrict
return true
}
if len(asns) == 0 {
// an asn filter is configured but the visitor ASN is unknown.
// allow list denies the unknown visitor, deny list allows it.
return !isTypeAllowList
}
// parse configured asns into a set
configured := map[uint32]struct{}{}
for _, line := range parseCountryCodes(asnStr) {
if n, ok := parseASN(line); ok {
configured[n] = struct{}{}
}
}
// a visitor matches if any of its announcing ASNs is configured
isMatch := false
for _, n := range asns {
if _, ok := configured[n]; ok {
isMatch = true
break
}
}
// if allow list and asn matches
if isTypeAllowList && isMatch {
return true
}
// if deny list and asn matches
if !isTypeAllowList && isMatch {
return false
}
// If this is an allow list and asn didn't match, not allowed
if isTypeAllowList {
return false
}
// If this is a deny list and asn didn't match, it is allowed
return true
}
// parseASN parses a single ASN line into a number. It accepts an optional AS
// or ASN prefix, so "AS15169", "asn15169" and "15169" are all valid.
func parseASN(line string) (uint32, bool) {
s := trimSpace(line)
if s == "" {
return 0, false
}
lower := strings.ToLower(s)
lower = strings.TrimPrefix(lower, "asn")
lower = strings.TrimPrefix(lower, "as")
lower = trimSpace(lower)
if lower == "" {
return 0, false
}
n, err := strconv.ParseUint(lower, 10, 32)
if err != nil {
return 0, false
}
return uint32(n), true
}
// parseCountryCodes splits newline-separated country codes and trims whitespace
func parseCountryCodes(input string) []string {
var result []string
+92
View File
@@ -0,0 +1,92 @@
package model
import (
"testing"
"github.com/oapi-codegen/nullable"
"github.com/phishingclub/phishingclub/vo"
)
func filter(allowed bool, country, asns string) *AllowDeny {
r := &AllowDeny{Allowed: nullable.NewNullableWithValue(allowed)}
if country != "" {
r.CountryCodes = nullable.NewNullableWithValue(country)
}
if asns != "" {
r.Asns = nullable.NewNullableWithValue(asns)
}
return r
}
func TestIsCountryAllowed(t *testing.T) {
cases := []struct {
name string
allowed bool
country string
visitor string
want bool
}{
{"no filter configured passes", true, "", "DK", true},
{"allow list match", true, "DK\nUS", "DK", true},
{"allow list no match denies", true, "DK\nUS", "FR", false},
{"deny list match denies", false, "RU", "RU", false},
{"deny list no match allows", false, "RU", "DK", true},
// the fail open fix: an unknown visitor country must be denied by an
// allow list and allowed by a deny list
{"allow list unknown visitor denied", true, "DK", "", false},
{"deny list unknown visitor allowed", false, "RU", "", true},
{"case insensitive", true, "dk", "DK", true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := filter(c.allowed, c.country, "").IsCountryAllowed(c.visitor)
if got != c.want {
t.Fatalf("got %v want %v", got, c.want)
}
})
}
}
func TestIsASNAllowed(t *testing.T) {
cases := []struct {
name string
allowed bool
asns string
visitor []uint32
want bool
}{
{"no filter configured passes", true, "", []uint32{15169}, true},
{"allow list match", true, "15169\n13335", []uint32{15169}, true},
{"allow list no match denies", true, "15169", []uint32{13335}, false},
{"allow list matches any announcing asn", true, "13335", []uint32{15169, 13335}, true},
{"deny list match denies", false, "13335", []uint32{13335}, false},
{"deny list no match allows", false, "13335", []uint32{15169}, true},
{"AS prefix accepted", true, "AS15169", []uint32{15169}, true},
// unknown visitor asn: allow list denies, deny list allows
{"allow list unknown visitor denied", true, "15169", nil, false},
{"deny list unknown visitor allowed", false, "15169", nil, true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := filter(c.allowed, "", c.asns).IsASNAllowed(c.visitor)
if got != c.want {
t.Fatalf("got %v want %v", got, c.want)
}
})
}
}
func TestValidateRejectsBadASN(t *testing.T) {
r := &AllowDeny{
Name: nullable.NewNullableWithValue(*vo.NewString127Must("test")),
Allowed: nullable.NewNullableWithValue(true),
Asns: nullable.NewNullableWithValue("15169\nnotanumber"),
}
if err := r.Validate(); err == nil {
t.Fatal("expected validation error for bad ASN")
}
r.Asns = nullable.NewNullableWithValue("15169\nAS13335")
if err := r.Validate(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
+11 -8
View File
@@ -29,7 +29,7 @@ import (
"github.com/phishingclub/phishingclub/cache"
"github.com/phishingclub/phishingclub/data"
"github.com/phishingclub/phishingclub/database"
"github.com/phishingclub/phishingclub/geoip"
"github.com/phishingclub/phishingclub/ipdata"
"github.com/phishingclub/phishingclub/model"
"github.com/phishingclub/phishingclub/repository"
"github.com/phishingclub/phishingclub/server"
@@ -5333,14 +5333,14 @@ func (m *ProxyHandler) checkFilter(req *http.Request, reqCtx *RequestContext) (b
// get ja4 fingerprint from request header (set by middleware)
ja4 := req.Header.Get(HEADER_JA4)
// get country code from GeoIP lookup
var countryCode string
if geo, err := geoip.Instance(); err == nil {
countryCode, _ = geo.Lookup(ip)
}
// get country code and ASNs from the IP data lookup
store := ipdata.Get()
countryCode, _ := store.LookupCountry(ip)
asns := store.LookupASNs(ip)
m.logger.Debugw("checking geo ip",
"ip", ip,
"country", countryCode,
"asns", asns,
)
// check IP, JA4, and country code against allow/deny lists
@@ -5371,6 +5371,9 @@ func (m *ProxyHandler) checkFilter(req *http.Request, reqCtx *RequestContext) (b
// check country code filter
countryOk := allowDeny.IsCountryAllowed(countryCode)
// check ASN filter
asnOk := allowDeny.IsASNAllowed(asns)
// check header filter
headers := req.Header
headerOk, err := allowDeny.IsHeaderAllowed(headers)
@@ -5382,13 +5385,13 @@ func (m *ProxyHandler) checkFilter(req *http.Request, reqCtx *RequestContext) (b
// for deny lists: any filter failing blocks the request
if isAllowListing {
// allow list: all must be allowed
if ipOk && ja4Ok && countryOk && headerOk {
if ipOk && ja4Ok && countryOk && asnOk && headerOk {
allowed = true
break
}
} else {
// deny list: if any filter denies, block the request
if !ipOk || !ja4Ok || !countryOk || !headerOk {
if !ipOk || !ja4Ok || !countryOk || !asnOk || !headerOk {
allowed = false
break
}
+2
View File
@@ -188,6 +188,7 @@ func ToAllowDeny(row *database.AllowDeny) *model.AllowDeny {
ja4Fingerprints := nullable.NewNullableWithValue(row.JA4Fingerprints)
countryCodes := nullable.NewNullableWithValue(row.CountryCodes)
asns := nullable.NewNullableWithValue(row.Asns)
headers := nullable.NewNullableWithValue(row.Headers)
return &model.AllowDeny{
@@ -198,6 +199,7 @@ func ToAllowDeny(row *database.AllowDeny) *model.AllowDeny {
Cidrs: cidrsNullable,
JA4Fingerprints: ja4Fingerprints,
CountryCodes: countryCodes,
Asns: asns,
Headers: headers,
Allowed: nullable.NewNullableWithValue(row.Allowed),
CompanyID: companyID,
+3
View File
@@ -133,6 +133,9 @@ func (s *AllowDeny) Update(
if v, err := incoming.CountryCodes.Get(); err == nil {
current.CountryCodes.Set(v)
}
if v, err := incoming.Asns.Get(); err == nil {
current.Asns.Set(v)
}
if v, err := incoming.Headers.Get(); err == nil {
current.Headers.Set(v)
}
+373
View File
@@ -0,0 +1,373 @@
package service
import (
"archive/tar"
"compress/gzip"
"context"
"crypto/sha256"
"crypto/tls"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sync"
"time"
"github.com/go-errors/errors"
"github.com/phishingclub/phishingclub/build"
"github.com/phishingclub/phishingclub/data"
"github.com/phishingclub/phishingclub/errs"
"github.com/phishingclub/phishingclub/ipdata"
"github.com/phishingclub/phishingclub/model"
)
const (
// ipdataBaseURL is the fixed location of the latest data packages. It takes
// no user input, so it adds no request forgery surface.
ipdataBaseURL = "https://github.com/phishingclub/ipdata/releases/latest/download"
// ipdataMaxDownload caps a package download. The packages are a few MB, so
// this only stops a runaway response.
ipdataMaxDownload = 64 << 20
// ipdataMaxFile caps a single extracted file.
ipdataMaxFile = 128 << 20
)
// IPData manages the downloadable country and ASN data packages.
type IPData struct {
Common
Store *ipdata.Store
// mu serializes install and remove so two operators cannot race on the
// package directory
mu sync.Mutex
}
// manifestPackage mirrors one package entry in the ipdata manifest.
type manifestPackage struct {
File string `json:"file"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
ContentHash string `json:"content_hash"`
Entries int `json:"entries"`
IPv4Prefixes int `json:"ipv4_prefixes"`
IPv6Prefixes int `json:"ipv6_prefixes"`
}
// manifest mirrors the ipdata manifest.json.
type manifest struct {
Format int `json:"format"`
Version string `json:"version"`
Created string `json:"created"`
Packages map[string]manifestPackage `json:"packages"`
}
// PackageStatus is the state of one package for the settings screen.
type PackageStatus struct {
Kind string `json:"kind"`
Installed bool `json:"installed"`
Info *ipdata.Info `json:"info,omitempty"`
UpdateAvailable bool `json:"updateAvailable"`
LatestVersion string `json:"latestVersion"`
LatestCreated string `json:"latestCreated"`
}
func validKind(kind string) bool {
return kind == ipdata.KindGeoIP || kind == ipdata.KindASN
}
func (s *IPData) httpClient(timeout time.Duration) *http.Client {
client := &http.Client{Timeout: timeout}
if !build.Flags.Production {
client.Transport = &http.Transport{
// #nosec
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
return client
}
// fetchManifest downloads and parses the current manifest.
func (s *IPData) fetchManifest() (*manifest, error) {
req, err := http.NewRequest(http.MethodGet, ipdataBaseURL+"/manifest.json", nil)
if err != nil {
return nil, errs.Wrap(err)
}
req.Header.Set("User-Agent", "PhishingClub-Client")
resp, err := s.httpClient(20 * time.Second).Do(req)
if err != nil {
return nil, errs.Wrap(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, errors.New("unexpected response fetching ipdata manifest")
}
var m manifest
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&m); err != nil {
return nil, errs.Wrap(err)
}
return &m, nil
}
// Status returns the state of both packages, including whether a newer version
// is available. The remote check is best effort and never fails the call.
func (s *IPData) Status(
ctx context.Context,
session *model.Session,
) ([]PackageStatus, error) {
isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL)
if err != nil {
s.LogAuthError(err)
return nil, errs.Wrap(err)
}
if !isAuthorized {
return nil, errors.New("unauthorized")
}
installed := s.Store.Status()
var remote *manifest
if m, err := s.fetchManifest(); err == nil {
remote = m
} else {
s.Logger.Debugw("could not fetch ipdata manifest", "error", err)
}
out := make([]PackageStatus, 0, 2)
for _, kind := range []string{ipdata.KindGeoIP, ipdata.KindASN} {
ps := PackageStatus{Kind: kind}
if info, ok := installed[kind]; ok {
infoCopy := info
ps.Installed = info.Downloaded
ps.Info = &infoCopy
}
if remote != nil {
if rp, ok := remote.Packages[kind]; ok {
ps.LatestVersion = remote.Version
ps.LatestCreated = remote.Created
current := ""
if ps.Info != nil {
current = ps.Info.ContentHash
}
ps.UpdateAvailable = current != rp.ContentHash
}
}
out = append(out, ps)
}
return out, nil
}
// Download fetches, verifies and installs the package of the given kind, then
// reloads it into the running store without a restart.
func (s *IPData) Download(
ctx context.Context,
session *model.Session,
kind string,
) error {
s.mu.Lock()
defer s.mu.Unlock()
ae := NewAuditEvent("IPData.Download", session)
ae.Details["kind"] = kind
isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL)
if err != nil {
s.LogAuthError(err)
return errs.Wrap(err)
}
if !isAuthorized {
s.AuditLogNotAuthorized(ae)
return errors.New("unauthorized")
}
if !validKind(kind) {
return errs.NewValidationError(fmt.Errorf("unknown package kind: %s", kind))
}
m, err := s.fetchManifest()
if err != nil {
return errs.NewOperationalError(
"Could not reach the data server. The data packages may not be published yet, or this server has no outbound internet access.",
err,
)
}
pkg, ok := m.Packages[kind]
if !ok {
return errs.NewOperationalError(
fmt.Sprintf("The %s package is not available for download yet.", kind),
fmt.Errorf("manifest has no package %q", kind),
)
}
// download into the ipdata directory so the final rename stays on one
// filesystem
root := s.Store.DataRoot()
if err := os.MkdirAll(root, 0o750); err != nil {
return errs.Wrap(err)
}
tmpArchive, err := os.CreateTemp(root, ".dl-*.tar.gz")
if err != nil {
return errs.Wrap(err)
}
tmpArchivePath := tmpArchive.Name()
defer os.Remove(tmpArchivePath)
// build the URL from the fixed kind, not from a manifest field, so no part
// of the path is influenced by the fetched manifest
url := ipdataBaseURL + "/" + kind + ".tar.gz"
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
tmpArchive.Close()
return errs.Wrap(err)
}
req.Header.Set("User-Agent", "PhishingClub-Client")
resp, err := s.httpClient(3 * time.Minute).Do(req)
if err != nil {
tmpArchive.Close()
return errs.NewOperationalError("Could not reach the data server to download the package.", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
tmpArchive.Close()
return errs.NewOperationalError(
"The data server did not return the package.",
fmt.Errorf("http %d downloading %s", resp.StatusCode, pkg.File),
)
}
// hash while copying, cap the size
hasher := sha256.New()
n, err := io.Copy(io.MultiWriter(tmpArchive, hasher), io.LimitReader(resp.Body, ipdataMaxDownload+1))
tmpArchive.Close()
if err != nil {
return errs.NewOperationalError("The package download was interrupted.", err)
}
if n > ipdataMaxDownload {
return errs.NewOperationalError("The package is larger than the allowed size.", nil)
}
if got := hex.EncodeToString(hasher.Sum(nil)); got != pkg.SHA256 {
return errs.NewOperationalError(
"The downloaded package failed its integrity check.",
fmt.Errorf("sha256 mismatch for %s: want %s got %s", kind, pkg.SHA256, got),
)
}
// extract into a temp directory, then validate before moving into place
tmpDir, err := os.MkdirTemp(root, ".extract-*")
if err != nil {
return errs.Wrap(err)
}
defer os.RemoveAll(tmpDir)
if err := extractPackage(tmpArchivePath, tmpDir); err != nil {
return errs.NewOperationalError("The downloaded package could not be read.", err)
}
if err := ipdata.Validate(tmpDir, kind); err != nil {
return errs.NewOperationalError("The downloaded package is not valid.", err)
}
// swap into the final location
finalDir := s.Store.PackageDir(kind)
oldDir := finalDir + ".old"
_ = os.RemoveAll(oldDir)
if _, err := os.Stat(finalDir); err == nil {
if err := os.Rename(finalDir, oldDir); err != nil {
return errs.Wrap(err)
}
}
if err := os.Rename(tmpDir, finalDir); err != nil {
// try to restore the previous package
_ = os.Rename(oldDir, finalDir)
return errs.Wrap(err)
}
_ = os.RemoveAll(oldDir)
s.Store.Reload(kind)
ae.Details["version"] = pkg.ContentHash
s.AuditLogAuthorized(ae)
return nil
}
// Remove deletes the installed package of the given kind and reloads the
// store. Removing geoip falls back to the embedded data, removing asn turns
// ASN lookups off.
func (s *IPData) Remove(
ctx context.Context,
session *model.Session,
kind string,
) error {
s.mu.Lock()
defer s.mu.Unlock()
ae := NewAuditEvent("IPData.Remove", session)
ae.Details["kind"] = kind
isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL)
if err != nil {
s.LogAuthError(err)
return errs.Wrap(err)
}
if !isAuthorized {
s.AuditLogNotAuthorized(ae)
return errors.New("unauthorized")
}
if !validKind(kind) {
return errs.NewValidationError(fmt.Errorf("unknown package kind: %s", kind))
}
if err := os.RemoveAll(s.Store.PackageDir(kind)); err != nil {
return errs.Wrap(err)
}
s.Store.Reload(kind)
s.AuditLogAuthorized(ae)
return nil
}
// extractPackage unpacks package.json and entries.json from a gzip tar into
// dir. It reads only those two files by base name and rejects anything else.
func extractPackage(archivePath, dir string) error {
f, err := os.Open(archivePath)
if err != nil {
return err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return err
}
defer gz.Close()
tr := tar.NewReader(gz)
allowed := map[string]bool{"package.json": true, "entries.json": true}
seen := map[string]bool{}
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if hdr.Typeflag != tar.TypeReg {
continue
}
name := filepath.Base(hdr.Name)
if !allowed[name] {
continue
}
out, err := os.Create(filepath.Join(dir, name))
if err != nil {
return err
}
if _, err := io.Copy(out, io.LimitReader(tr, ipdataMaxFile)); err != nil {
out.Close()
return err
}
out.Close()
seen[name] = true
}
if !seen["package.json"] || !seen["entries.json"] {
return errors.New("package archive is missing package.json or entries.json")
}
return nil
}
+83 -5
View File
@@ -900,9 +900,7 @@ export class API {
* @returns {Promise<ApiResponse>}
*/
getGroupedResultStats: async (campaignID, by = 'position') => {
return await getJSON(
this.getPath(`/campaign/${campaignID}/grouped-statistics?by=${by}`)
);
return await getJSON(this.getPath(`/campaign/${campaignID}/grouped-statistics?by=${by}`));
},
/**
@@ -3170,17 +3168,28 @@ export class API {
* @param {string} allowdeny.cidrs
* @param {string} allowdeny.ja4Fingerprints
* @param {string} allowdeny.countryCodes
* @param {string} allowdeny.asns
* @param {string} allowdeny.headers
* @param {boolean} allowdeny.allowed
* @param {string} allowdeny.companyID
* @returns {Promise<ApiResponse>}
*/
create: async ({ name, cidrs, ja4Fingerprints, countryCodes, headers, allowed, companyID }) => {
create: async ({
name,
cidrs,
ja4Fingerprints,
countryCodes,
asns,
headers,
allowed,
companyID
}) => {
return await postJSON(this.getPath('/allow-deny'), {
name: name,
cidrs: cidrs,
ja4Fingerprints: ja4Fingerprints,
countryCodes: countryCodes,
asns: asns,
headers: headers,
allowed: allowed,
companyID: companyID
@@ -3238,12 +3247,22 @@ export class API {
* @param {string} allowdeny.companyID
* @returns {Promise<ApiResponse>}
*/
update: async ({ id, name, cidrs, ja4Fingerprints, countryCodes, headers, companyID }) => {
update: async ({
id,
name,
cidrs,
ja4Fingerprints,
countryCodes,
asns,
headers,
companyID
}) => {
return await patchJSON(this.getPath(`/allow-deny/${id}`), {
name: name,
cidrs: cidrs,
ja4Fingerprints: ja4Fingerprints,
countryCodes: countryCodes,
asns: asns,
headers: headers,
companyID: companyID
});
@@ -3274,6 +3293,65 @@ export class API {
}
};
/**
* ipdata is the API for the downloadable country and ASN data packages.
*/
ipdata = {
/**
* Get the state of both data packages, including whether an update is
* available.
*
* @returns {Promise<ApiResponse>}
*/
status: async () => {
return await getJSON(this.getPath('/ipdata/status'));
},
/**
* Download and install a package.
*
* @param {string} kind - "geoip" or "asn"
* @returns {Promise<ApiResponse>}
*/
download: async (kind) => {
return await postJSON(this.getPath(`/ipdata/package/${kind}/download`), {});
},
/**
* Remove an installed package.
*
* @param {string} kind - "geoip" or "asn"
* @returns {Promise<ApiResponse>}
*/
remove: async (kind) => {
return await deleteJSON(this.getPath(`/ipdata/package/${kind}`));
},
/**
* Search ASNs by number, name or handle for the filter typeahead.
*
* @param {string} query
* @param {number} [limit]
* @returns {Promise<ApiResponse>}
*/
searchASN: async (query, limit = 25) => {
return await getJSON(
this.getPath(`/ipdata/asn/search?q=${encodeURIComponent(query)}&limit=${limit}`)
);
},
/**
* Resolve configured ASN numbers to their details. ASNs absent from the
* dataset are left out of the result so the UI can flag them.
*
* @param {string[]} asns
* @returns {Promise<ApiResponse>}
*/
resolveASN: async (asns) => {
return await postJSON(this.getPath('/ipdata/asn/resolve'), { asns });
}
};
/**
* webhook is the API for web hook related operations.
*/
@@ -0,0 +1,362 @@
<script>
import { onMount, onDestroy } from 'svelte';
import { api } from '$lib/api/apiProxy.js';
import ToolTip from './ToolTip.svelte';
// value is an array of ASN number strings, for example ["15169", "13335"]
export let value = [];
export let toolTipText = '';
export let installed = true;
export let disabled = false;
export let placeholder = 'Search by number or name...';
let inputValue = '';
let inputElement;
let dropdownElement;
let showDropdown = false;
let results = [];
let searchTimer = null;
let dropdownPosition = { top: 0, left: 0, width: 0 };
// details keyed by asn number string, filled from search and resolve
let details = {};
// asn numbers that are configured but absent from the installed data
let orphans = new Set();
const parseNum = (s) => {
let v = String(s).toLowerCase().trim();
v = v.replace(/^asn/, '').replace(/^as/, '').trim();
if (v === '' || !/^\d+$/.test(v)) {
return null;
}
return v;
};
const label = (raw) => {
const n = parseNum(raw);
if (n && details[n] && (details[n].name || details[n].handle)) {
return `AS${n} ${details[n].name || details[n].handle}`;
}
return n ? `AS${n}` : String(raw);
};
const resolveSelected = async () => {
if (!installed || !value || value.length === 0) {
orphans = new Set();
return;
}
try {
const res = await api.ipdata.resolveASN(value);
if (res.success && res.data) {
const found = new Set();
for (const r of res.data.results || []) {
details[String(r.asn)] = r;
found.add(String(r.asn));
}
details = details;
const next = new Set();
for (const raw of value) {
const n = parseNum(raw);
if (n && !found.has(n)) next.add(n);
}
orphans = next;
}
} catch (e) {
console.error('failed to resolve ASNs', e);
}
};
const updateDropdownPosition = () => {
if (inputElement) {
const rect = inputElement.getBoundingClientRect();
const spaceBelow = window.innerHeight - rect.bottom;
const spaceAbove = rect.top;
// open upward when there is little room below and more room above, so
// the list stays on screen when the field is near the bottom
const openUp = spaceBelow < 260 && spaceAbove > spaceBelow;
dropdownPosition = {
left: rect.left,
width: rect.width,
openUp,
top: rect.bottom + 4,
bottom: window.innerHeight - rect.top + 4
};
}
};
const runSearch = () => {
clearTimeout(searchTimer);
const q = inputValue.trim();
if (q === '') {
results = [];
showDropdown = false;
return;
}
searchTimer = setTimeout(async () => {
try {
const res = await api.ipdata.searchASN(q, 50);
if (res.success && res.data) {
results = res.data.results || [];
} else {
results = [];
}
updateDropdownPosition();
showDropdown = results.length > 0;
} catch (e) {
console.error('failed to search ASNs', e);
}
}, 200);
};
const selectResult = (r) => {
const n = String(r.asn);
details[n] = r;
details = details;
const current = (value || []).map(parseNum);
if (!current.includes(n)) {
value = [...(value || []), n];
}
inputValue = '';
results = [];
showDropdown = false;
setTimeout(() => inputElement?.focus(), 0);
};
const addTyped = () => {
const n = parseNum(inputValue);
if (!n) return;
const current = (value || []).map(parseNum);
if (!current.includes(n)) {
value = [...(value || []), n];
resolveSelected();
}
inputValue = '';
results = [];
showDropdown = false;
};
// add every ASN currently shown in the results, for example all M247 systems
const addAll = () => {
const current = new Set((value || []).map(parseNum));
const next = [...(value || [])];
for (const r of results) {
const n = String(r.asn);
details[n] = r;
if (!current.has(n)) {
next.push(n);
current.add(n);
}
}
details = details;
value = next;
inputValue = '';
results = [];
showDropdown = false;
setTimeout(() => inputElement?.focus(), 0);
};
const remove = (raw) => {
const n = parseNum(raw);
value = (value || []).filter((v) => parseNum(v) !== n);
orphans.delete(n);
orphans = orphans;
};
const clearAll = () => {
value = [];
orphans = new Set();
inputValue = '';
inputElement?.focus();
};
const closeDropdown = () => {
showDropdown = false;
};
const handleBlur = () => {
setTimeout(() => {
const focused = document.activeElement;
const container = inputElement?.closest('.asn-select-container');
if (!container?.contains(focused)) {
closeDropdown();
}
}, 100);
};
const handleKeyDown = (e) => {
if (e.key === 'Enter') {
e.preventDefault();
if (parseNum(inputValue)) {
addTyped();
}
} else if (e.key === 'Escape') {
closeDropdown();
}
};
const handleOutsideClick = (e) => {
if (!showDropdown) return;
const container = inputElement?.closest('.asn-select-container');
if (container && !container.contains(e.target)) {
closeDropdown();
}
};
onMount(() => {
resolveSelected();
document.addEventListener('click', handleOutsideClick);
window.addEventListener('scroll', updateDropdownPosition, true);
window.addEventListener('resize', updateDropdownPosition);
return () => {
document.removeEventListener('click', handleOutsideClick);
window.removeEventListener('scroll', updateDropdownPosition, true);
window.removeEventListener('resize', updateDropdownPosition);
};
});
onDestroy(() => {
document.removeEventListener('click', handleOutsideClick);
window.removeEventListener('scroll', updateDropdownPosition, true);
window.removeEventListener('resize', updateDropdownPosition);
});
$: displayValue = value.length > 0 ? `${value.length} selected` : '';
</script>
<div class="flex justify-start flex-col asn-select-container">
<div class="flex flex-col py-2 relative">
<div class="flex items-center">
<p
class="font-semibold text-slate-600 dark:text-gray-400 py-2 transition-colors duration-200"
>
ASNs
</p>
{#if toolTipText.length > 0}
<ToolTip>
{toolTipText}
</ToolTip>
{/if}
<div
class="bg-gray-100 dark:bg-gray-800/60 ml-2 px-2 rounded-md transition-colors duration-200 h-6 flex items-center"
>
<p class="text-slate-600 dark:text-gray-400 text-xs">optional</p>
</div>
</div>
</div>
{#if !installed}
<p class="text-sm text-slate-500 dark:text-gray-400">
No ASN data downloaded. Enable it in
<a class="text-cta-blue dark:text-highlight-blue hover:opacity-80" href="/settings#ipdata"
>Settings, IP Data</a
>.
</p>
{:else}
<div class="relative">
<div class="flex items-center relative w-60">
<input
bind:this={inputElement}
type="text"
autocomplete="off"
bind:value={inputValue}
on:input={runSearch}
on:keydown={handleKeyDown}
on:blur={handleBlur}
{disabled}
class="w-full relative rounded-md py-2 pr-10 text-gray-600 dark:text-gray-300 border border-transparent focus:outline-none focus:border-solid focus:border focus:border-slate-400 dark:focus:border-highlight-blue/80 focus:bg-gray-100 dark:focus:bg-gray-700/60 bg-grayblue-light dark:bg-gray-900/60 font-normal transition-colors duration-200"
class:pl-10={showDropdown}
class:pl-4={!showDropdown}
placeholder={showDropdown ? '' : value.length > 0 ? displayValue : placeholder}
/>
{#if showDropdown}
<img
class="absolute w-4 left-3 select-none pointer-events-none z-10"
src="/search-icon.svg"
alt=""
aria-hidden="true"
/>
{/if}
{#if value.length > 0}
<button
class="absolute right-3 z-10"
type="button"
aria-label="Clear selection"
on:click={(e) => {
e.stopPropagation();
clearAll();
}}
>
<img class="w-4" src="/remove-value.svg" alt="" />
</button>
{/if}
</div>
{#if showDropdown}
<div
bind:this={dropdownElement}
class="fixed z-[9999]"
style="{dropdownPosition.openUp
? `bottom: ${dropdownPosition.bottom}px`
: `top: ${dropdownPosition.top}px`}; left: {dropdownPosition.left}px; width: {dropdownPosition.width}px;"
>
<ul
class="bg-gray-100 dark:bg-gray-900 list-none z-[9999] rounded-md min-w-fit shadow-lg border border-gray-200 dark:border-gray-700/60 max-h-56 overflow-y-auto transition-colors duration-200"
>
{#if results.length > 1}
<li role="none">
<button
type="button"
class="w-full text-left rounded-md text-cta-blue dark:text-highlight-blue hover:bg-grayblue-dark dark:hover:bg-highlight-blue/40 hover:text-white py-2 px-2 cursor-pointer focus:bg-grayblue-dark dark:focus:bg-highlight-blue/40 focus:text-white focus:outline-none font-medium transition-colors duration-200"
on:click|preventDefault|stopPropagation={addAll}
on:blur={handleBlur}
>
Add all {results.length} results
</button>
</li>
{/if}
{#each results as r (r.asn)}
<li role="none">
<button
type="button"
class="w-full text-left bg-slate-100 dark:bg-gray-900 rounded-md text-gray-600 dark:text-gray-300 hover:bg-grayblue-dark dark:hover:bg-highlight-blue/40 hover:text-white py-2 px-2 cursor-pointer focus:bg-grayblue-dark dark:focus:bg-highlight-blue/40 focus:text-white focus:outline-none transition-colors duration-200"
on:click|preventDefault|stopPropagation={() => selectResult(r)}
on:blur={handleBlur}
>
AS{r.asn}
{r.name || r.handle}{#if r.country}
<span class="opacity-70"> · {r.country}</span>{/if}
</button>
</li>
{/each}
</ul>
</div>
{/if}
{#if value.length > 0}
<div class="flex flex-row flex-wrap mt-4 gap-2 max-h-48 overflow-y-auto">
{#each value as raw (raw)}
<button
type="button"
on:click|preventDefault={() => remove(raw)}
class="flex flex-row items-center px-2 py-1 rounded-md text-xs transition-colors duration-200 flex-shrink-0 {orphans.has(
parseNum(raw)
)
? 'bg-amber-100 dark:bg-amber-900/40 text-amber-800 dark:text-amber-200 hover:bg-amber-200 dark:hover:bg-amber-900/60'
: 'bg-gray-100 dark:bg-gray-800/60 hover:bg-gray-200 dark:hover:bg-gray-700/80 text-gray-900 dark:text-gray-300'}"
title={orphans.has(parseNum(raw))
? 'This ASN is not in the installed data. The filter keeps it, but it will not match until the data is updated.'
: `Remove ${label(raw)}`}
style="max-width: 280px;"
>
<span class="truncate block" style="max-width: 250px;">{label(raw)}</span>
{#if orphans.has(parseNum(raw))}
<span class="ml-1" aria-hidden="true">⚠</span>
{/if}
<img class="w-3 ml-1 pointer-events-none flex-shrink-0" src="/delete2.svg" alt="" />
</button>
{/each}
</div>
{/if}
</div>
{/if}
</div>
+35 -3
View File
@@ -42,6 +42,7 @@
runBulkDelete
} from '$lib/service/tableSelection.js';
import TextFieldMultiSelect from '$lib/components/TextFieldMultiSelect.svelte';
import AsnSelect from '$lib/components/AsnSelect.svelte';
// services
const appStateService = AppStateService.instance;
@@ -54,6 +55,7 @@
cidrs: null,
ja4Fingerprints: null,
countryCodes: [],
asns: [],
headers: [],
allowed: null
};
@@ -83,6 +85,7 @@
let modalMode = null;
let modalText = '';
let availableCountryCodes = [];
let asnAvailable = false;
let isDeleteAlertVisible = false;
let deleteValues = {
@@ -121,6 +124,7 @@
const res = await api.geoip.getMetadata();
if (res.success && res.data) {
availableCountryCodes = res.data.country_codes || [];
asnAvailable = res.data.asn_available || false;
}
} catch (e) {
console.error('failed to load geoip metadata', e);
@@ -175,15 +179,16 @@
};
const onClickSubmit = async () => {
// validate that at least one of cidrs, ja4Fingerprints, countryCodes, or headers is provided
// validate that at least one of cidrs, ja4Fingerprints, countryCodes, asns, or headers is provided
const hasCidrs = formValues.cidrs && formValues.cidrs.trim().length > 0;
const hasJA4 = formValues.ja4Fingerprints && formValues.ja4Fingerprints.trim().length > 0;
const hasCountryCodes = formValues.countryCodes && formValues.countryCodes.length > 0;
const hasAsns = formValues.asns && formValues.asns.length > 0;
const hasHeaders = formValues.headers && formValues.headers.length > 0;
if (!hasCidrs && !hasJA4 && !hasCountryCodes && !hasHeaders) {
if (!hasCidrs && !hasJA4 && !hasCountryCodes && !hasAsns && !hasHeaders) {
formError =
'At least one of CIDRs, JA4 fingerprints, Country Codes, or Headers must be provided';
'At least one of CIDRs, JA4 fingerprints, Country Codes, ASNs, or Headers must be provided';
return;
}
@@ -226,6 +231,7 @@
cidrs: formValues.cidrs,
ja4Fingerprints: formValues.ja4Fingerprints || '',
countryCodes: formValues.countryCodes.join('\n'),
asns: formValues.asns.join('\n'),
headers: headersStr,
allowed: formValues.allowed,
companyID: contextCompanyID
@@ -269,6 +275,7 @@
cidrs: formValues.cidrs,
ja4Fingerprints: formValues.ja4Fingerprints || '',
countryCodes: formValues.countryCodes.join('\n'),
asns: formValues.asns.join('\n'),
headers: headersStr,
companyID: formValues.companyID
});
@@ -312,6 +319,16 @@
const openCreateModal = () => {
modalMode = 'create';
formValues = {
id: null,
name: null,
cidrs: null,
ja4Fingerprints: null,
countryCodes: [],
asns: [],
headers: [],
allowed: null
};
isModalVisible = true;
};
@@ -373,6 +390,15 @@
.filter((code) => code.length > 0);
}
// parse asns from newline-separated string to array
let asnsArray = [];
if (allowDeny.asns) {
asnsArray = allowDeny.asns
.split('\n')
.map((a) => a.trim())
.filter((a) => a.length > 0);
}
// parse headers from json string to array
let headersArray = [];
if (allowDeny.headers) {
@@ -390,6 +416,7 @@
cidrs: allowDeny.cidrs,
ja4Fingerprints: allowDeny.ja4Fingerprints || '',
countryCodes: countryCodesArray,
asns: asnsArray,
headers: headersArray,
allowed: allowDeny.allowed,
companyID: allowDeny.companyID
@@ -597,6 +624,11 @@
>
GeoIP Country Codes
</TextFieldMultiSelect>
<AsnSelect
bind:value={formValues.asns}
installed={asnAvailable}
toolTipText="Filter based on the autonomous system that announces the visitor IP. Requires the ASN data package under Settings."
/>
</FormColumn>
</FormColumns>
<FormError message={formError} />
@@ -12,6 +12,7 @@
import RedTeam from './panels/RedTeam.svelte';
import System from './panels/System.svelte';
import Branding from './panels/Branding.svelte';
import IPData from './panels/IPData.svelte';
// Red Team panel is only relevant in red team phishing (blackbox) mode
$: tabs = [
@@ -19,6 +20,7 @@
{ id: 'access', label: 'Access', component: Access },
{ id: 'scim', label: 'SCIM', component: Scim },
{ id: 'data', label: 'Data', component: Data },
{ id: 'ipdata', label: 'IP Data', component: IPData },
{ id: 'reports', label: 'Reports', component: Reports },
...($displayMode === DISPLAY_MODE.BLACKBOX
? [{ id: 'redteam', label: 'Red Team', component: RedTeam }]
@@ -0,0 +1,154 @@
<script>
import { onMount } from 'svelte';
import { api } from '$lib/api/apiProxy.js';
import { addToast } from '$lib/store/toast';
import SettingsCard from '$lib/components/SettingsCard.svelte';
import Button from '$lib/components/Button.svelte';
import { showIsLoading, hideIsLoading } from '$lib/store/loading';
let loaded = false;
let packages = [];
let busy = {};
const meta = {
geoip: {
title: 'Geo IP Data',
blurb: 'Maps visitor IPs to a country for country filters. Built in, download to refresh.'
},
asn: {
title: 'ASN Data',
blurb: 'Maps visitor IPs to their network operator for ASN filters. Download to enable.'
}
};
const refresh = async () => {
try {
const res = await api.ipdata.status();
if (res.success && res.data) {
packages = res.data.packages || [];
}
} catch (e) {
console.error('failed to load ip data status', e);
}
};
onMount(async () => {
showIsLoading();
try {
await refresh();
} finally {
loaded = true;
hideIsLoading();
}
});
const download = async (kind) => {
busy = { ...busy, [kind]: true };
showIsLoading();
try {
const res = await api.ipdata.download(kind);
if (res.success) {
addToast('Downloaded latest data', 'Success');
await refresh();
} else {
addToast(res.error || 'Failed to download data', 'Error');
}
} catch (e) {
addToast('Failed to download data', 'Error');
console.error('failed to download ip data', e);
} finally {
busy = { ...busy, [kind]: false };
hideIsLoading();
}
};
const fmtDate = (s) => {
if (!s) return '';
try {
return new Date(s).toLocaleDateString();
} catch (_) {
return s;
}
};
const remove = async (kind) => {
busy = { ...busy, [kind]: true };
showIsLoading();
try {
const res = await api.ipdata.remove(kind);
if (res.success) {
addToast('Removed downloaded data', 'Success');
await refresh();
} else {
addToast(res.error || 'Failed to remove data', 'Error');
}
} catch (e) {
addToast('Failed to remove data', 'Error');
console.error('failed to remove ip data', e);
} finally {
busy = { ...busy, [kind]: false };
hideIsLoading();
}
};
</script>
{#if loaded}
<div class="flex flex-wrap gap-6">
{#each packages as p (p.kind)}
<SettingsCard title={meta[p.kind]?.title || p.kind}>
<div class="space-y-4">
<p class="text-gray-600 dark:text-gray-300 text-sm transition-colors duration-200">
{meta[p.kind]?.blurb || ''}
</p>
<div class="space-y-1">
<p
class="text-sm font-medium transition-colors duration-200"
class:text-green-600={p.info && p.info.downloaded}
class:dark:text-green-400={p.info && p.info.downloaded}
class:text-gray-500={!(p.info && p.info.downloaded)}
class:dark:text-gray-400={!(p.info && p.info.downloaded)}
>
{#if p.info && p.info.downloaded}
Downloaded
{:else if p.info}
Built in
{:else}
Not downloaded
{/if}
</p>
{#if p.info}
<p class="text-sm text-gray-500 dark:text-gray-400 transition-colors duration-200">
{#if p.info.created}{fmtDate(p.info.created)} ·
{/if}{(p.info.ipv4Prefixes + p.info.ipv6Prefixes).toLocaleString()} prefixes
</p>
{/if}
{#if p.updateAvailable}
<p
class="text-sm text-cta-blue dark:text-highlight-blue transition-colors duration-200"
>
Update available
</p>
{/if}
</div>
</div>
<svelte:fragment slot="footer">
<div class="flex gap-3">
{#if p.info && p.info.downloaded}
<Button
size="medium"
backgroundColor="bg-red-600"
disabled={busy[p.kind]}
on:click={() => remove(p.kind)}
>
Remove
</Button>
{/if}
<Button size="medium" disabled={busy[p.kind]} on:click={() => download(p.kind)}>
{p.info && p.info.downloaded ? 'Update' : 'Download'}
</Button>
</div>
</svelte:fragment>
</SettingsCard>
{/each}
</div>
{/if}
+3 -1
View File
@@ -5,11 +5,13 @@
import JA4Builder from './panels/JA4Builder.svelte';
import CalendarBuilder from './panels/CalendarBuilder.svelte';
import GeoIP from './panels/GeoIP.svelte';
import ASN from './panels/ASN.svelte';
const tabs = [
{ id: 'calendar', label: 'Calendar Invitation Builder', component: CalendarBuilder },
{ id: 'ja4', label: 'JA4 Fingerprint Builder', component: JA4Builder },
{ id: 'geoip', label: 'GeoIP Lookup', component: GeoIP }
{ id: 'geoip', label: 'GeoIP Lookup', component: GeoIP },
{ id: 'asn', label: 'ASN Lookup', component: ASN }
];
let active = 'calendar';
+162
View File
@@ -0,0 +1,162 @@
<script>
import { onMount } from 'svelte';
import TextField from '$lib/components/TextField.svelte';
import FormError from '$lib/components/FormError.svelte';
import Button from '$lib/components/Button.svelte';
import SettingsCard from '$lib/components/SettingsCard.svelte';
let query = '';
let isSubmitting = false;
let lookupError = '';
let results = null;
let mode = 'search'; // 'ip' when the query was an IP address
let asnAvailable = true;
let searchTimer = null;
onMount(async () => {
try {
const res = await fetch('/api/v1/geoip/metadata', { credentials: 'include' });
if (res.ok) {
const data = await res.json();
asnAvailable = !!data.data?.asn_available;
}
} catch (_) {
// leave asnAvailable true, a failed lookup will surface the reason
}
});
const isIP = (s) => {
const v = s.trim();
return /^(\d{1,3}\.){3}\d{1,3}$/.test(v) || v.includes(':');
};
async function run(q) {
isSubmitting = true;
lookupError = '';
try {
let url;
if (isIP(q)) {
mode = 'ip';
url = `/api/v1/ipdata/asn/lookup?ip=${encodeURIComponent(q)}`;
} else {
mode = 'search';
url = `/api/v1/ipdata/asn/search?q=${encodeURIComponent(q)}&limit=50`;
}
const response = await fetch(url, { method: 'GET', credentials: 'include' });
if (!response.ok) {
const errorData = await response.json();
lookupError = errorData.message || 'failed to look up ASN';
return;
}
const data = await response.json();
if (mode === 'ip' && data.data && data.data.available === false) {
asnAvailable = false;
}
results = data.data?.results || [];
} catch (error) {
lookupError = 'an error occurred while looking up the ASN';
console.error('asn lookup error:', error);
} finally {
isSubmitting = false;
}
}
// search live as the user types a name or number, but leave IPs for Enter or
// the button since a partial IP is not meaningful
function handleInput() {
clearTimeout(searchTimer);
const q = query.trim();
lookupError = '';
if (q === '' || isIP(q)) {
results = null;
return;
}
searchTimer = setTimeout(() => run(q), 200);
}
function handleLookup() {
clearTimeout(searchTimer);
const q = query.trim();
if (!q) {
lookupError = 'please enter an IP, ASN number, or name';
return;
}
run(q);
}
function handleKey(event) {
if (event.key === 'Enter') {
handleLookup();
return;
}
handleInput();
}
</script>
<div class="flex flex-wrap gap-6">
<SettingsCard title="ASN Lookup">
<div class="space-y-4">
<TextField
type="text"
bind:value={query}
placeholder="IP, ASN number, or name (e.g. 8.8.8.8, 3292, M247)"
on:keyup={handleKey}
>
Search
</TextField>
<FormError message={lookupError} />
<div class="text-xs text-gray-500 dark:text-gray-400 transition-colors duration-200">
Data from
<a
href="https://github.com/ipverse/asn-ip"
target="_blank"
rel="noopener noreferrer"
class="text-blue-600 dark:text-blue-400 hover:underline"
>
ipverse/asn-ip
</a>
</div>
{#if !asnAvailable}
<div
class="p-3 rounded-md bg-yellow-50 dark:bg-yellow-900/20 transition-colors duration-200"
>
<p class="text-sm font-medium text-yellow-700 dark:text-yellow-300">
No ASN data downloaded. Enable it in
<a class="underline" href="/settings#ipdata">Settings, IP Data</a>.
</p>
</div>
{:else if results !== null}
{#if results.length > 0}
<div
class="p-3 rounded-md bg-green-50 dark:bg-green-900/20 transition-colors duration-200 space-y-1"
>
{#each results as r (r.asn)}
<p class="text-sm font-medium text-green-700 dark:text-green-300">
<strong>AS{r.asn}</strong>
{r.name || r.handle}{#if r.country}
· {r.country}{/if}{#if r.handle}
<span class="opacity-70"> · {r.handle}</span>{/if}
</p>
{/each}
</div>
{:else}
<div
class="p-3 rounded-md bg-yellow-50 dark:bg-yellow-900/20 transition-colors duration-200"
>
<p class="text-sm font-medium text-yellow-700 dark:text-yellow-300">
{mode === 'ip' ? 'No match' : 'No results'}
</p>
</div>
{/if}
{/if}
</div>
<svelte:fragment slot="footer">
<Button size={'large'} on:click={handleLookup} disabled={isSubmitting}>
{#if isSubmitting}Looking up...{:else}Lookup{/if}
</Button>
</svelte:fragment>
</SettingsCard>
</div>