diff --git a/backend/app/administration.go b/backend/app/administration.go index 103e0dc2..2911856a 100644 --- a/backend/app/administration.go +++ b/backend/app/administration.go @@ -226,6 +226,13 @@ const ( // geoip ROUTE_V1_GEOIP_METADATA = "/api/v1/geoip/metadata" ROUTE_V1_GEOIP_LOOKUP = "/api/v1/geoip/lookup" + + ROUTE_V1_IPDATA_STATUS = "/api/v1/ipdata/status" + ROUTE_V1_IPDATA_PACKAGE_KIND = "/api/v1/ipdata/package/:kind" + ROUTE_V1_IPDATA_PACKAGE_DOWNLOAD = "/api/v1/ipdata/package/:kind/download" + ROUTE_V1_IPDATA_ASN_SEARCH = "/api/v1/ipdata/asn/search" + ROUTE_V1_IPDATA_ASN_RESOLVE = "/api/v1/ipdata/asn/resolve" + ROUTE_V1_IPDATA_ASN_LOOKUP = "/api/v1/ipdata/asn/lookup" // web hooks ROUTE_V1_WEBHOOK = "/api/v1/webhook" ROUTE_V1_WEBHOOK_ID = "/api/v1/webhook/:id" @@ -590,6 +597,13 @@ func setupRoutes( // geoip GET(ROUTE_V1_GEOIP_METADATA, middleware.SessionHandler, controllers.GeoIP.GetMetadata). GET(ROUTE_V1_GEOIP_LOOKUP, middleware.SessionHandler, controllers.GeoIP.Lookup). + // ip data packages (country + asn) + GET(ROUTE_V1_IPDATA_STATUS, middleware.SessionHandler, controllers.IPData.Status). + POST(ROUTE_V1_IPDATA_PACKAGE_DOWNLOAD, middleware.SessionHandler, controllers.IPData.Download). + DELETE(ROUTE_V1_IPDATA_PACKAGE_KIND, middleware.SessionHandler, controllers.IPData.Remove). + GET(ROUTE_V1_IPDATA_ASN_SEARCH, middleware.SessionHandler, controllers.IPData.SearchASN). + POST(ROUTE_V1_IPDATA_ASN_RESOLVE, middleware.SessionHandler, controllers.IPData.ResolveASNs). + GET(ROUTE_V1_IPDATA_ASN_LOOKUP, middleware.SessionHandler, controllers.IPData.LookupASN). // web hooks GET(ROUTE_V1_WEBHOOK, middleware.SessionHandler, controllers.Webhook.GetAll). GET(ROUTE_V1_WEBHOOK_ID, middleware.SessionHandler, controllers.Webhook.GetByID). diff --git a/backend/app/controllers.go b/backend/app/controllers.go index f7352610..1e33e0bd 100644 --- a/backend/app/controllers.go +++ b/backend/app/controllers.go @@ -31,6 +31,7 @@ type Controllers struct { APISender *controller.APISender AllowDeny *controller.AllowDeny GeoIP *controller.GeoIP + IPData *controller.IPData Webhook *controller.Webhook Identifier *controller.Identifier Version *controller.Version @@ -205,6 +206,10 @@ func NewControllers( geoIP := &controller.GeoIP{ Common: common, } + ipData := &controller.IPData{ + Common: common, + IPDataService: services.IPData, + } oauthProvider := &controller.OAuthProvider{ Common: common, OAuthProviderService: services.OAuthProvider, @@ -270,6 +275,7 @@ func NewControllers( APISender: apiSender, AllowDeny: allowDeny, GeoIP: geoIP, + IPData: ipData, Webhook: webhook, Identifier: identifier, Version: version, diff --git a/backend/app/server.go b/backend/app/server.go index 7cd81ed3..cbff73a9 100644 --- a/backend/app/server.go +++ b/backend/app/server.go @@ -33,7 +33,7 @@ import ( "github.com/phishingclub/phishingclub/data" "github.com/phishingclub/phishingclub/database" "github.com/phishingclub/phishingclub/errs" - "github.com/phishingclub/phishingclub/geoip" + "github.com/phishingclub/phishingclub/ipdata" "github.com/phishingclub/phishingclub/middleware" "github.com/phishingclub/phishingclub/model" "github.com/phishingclub/phishingclub/proxy" @@ -2461,14 +2461,14 @@ func (s *Server) checkIPFilter( // get ja4 fingerprint from context ja4 := middleware.GetJA4FromContext(ctx) - // get country code from GeoIP lookup - var countryCode string - if geo, err := geoip.Instance(); err == nil { - countryCode, _ = geo.Lookup(ip) - } + // get country code and ASNs from the IP data lookup + store := ipdata.Get() + countryCode, _ := store.LookupCountry(ip) + asns := store.LookupASNs(ip) s.logger.Debugw("checking geo ip", "ip", ip, "country", countryCode, + "asns", asns, ) allowDenyLEntries, err := s.repositories.Campaign.GetAllDenyByCampaignID(ctx, campaignID) @@ -2508,6 +2508,9 @@ func (s *Server) checkIPFilter( // check country code filter countryOk := allowDeny.IsCountryAllowed(countryCode) + // check ASN filter + asnOk := allowDeny.IsASNAllowed(asns) + // check header filter headers := ctx.Request.Header headerOk, err := allowDeny.IsHeaderAllowed(headers) @@ -2519,7 +2522,7 @@ func (s *Server) checkIPFilter( // for deny lists: any filter failing blocks the request if isAllowListing { // allow list: all must be allowed - if ipOk && ja4Ok && countryOk && headerOk { + if ipOk && ja4Ok && countryOk && asnOk && headerOk { s.logger.Debugw("IP, JA4, country, and headers are allow listed", "ip", ip, "ja4", ja4, @@ -2532,7 +2535,7 @@ func (s *Server) checkIPFilter( } } else { // deny list: if any filter denies, block the request - if !ipOk || !ja4Ok || !countryOk || !headerOk { + if !ipOk || !ja4Ok || !countryOk || !asnOk || !headerOk { s.logger.Debugw("IP, JA4, country, or headers is deny listed", "ip", ip, "ja4", ja4, diff --git a/backend/app/services.go b/backend/app/services.go index 78da8d1c..b52ed732 100644 --- a/backend/app/services.go +++ b/backend/app/services.go @@ -5,6 +5,7 @@ import ( "time" "github.com/caddyserver/certmagic" + "github.com/phishingclub/phishingclub/ipdata" "github.com/phishingclub/phishingclub/script" "github.com/phishingclub/phishingclub/service" "go.uber.org/zap" @@ -38,6 +39,7 @@ type Services struct { Version *service.Version SSO *service.SSO Update *service.Update + IPData *service.IPData Import *service.Import Backup *service.Backup IPAllowList *service.IPAllowListService @@ -326,6 +328,10 @@ func NewServices( Common: common, OptionService: optionService, } + ipDataService := &service.IPData{ + Common: common, + Store: ipdata.Get(), + } importService := &service.Import{ Common: common, Asset: asset, @@ -406,6 +412,7 @@ func NewServices( Version: versionService, SSO: ssoService, Update: updateService, + IPData: ipDataService, Import: importService, Backup: backupService, IPAllowList: ipAllowListService, diff --git a/backend/controller/geoip.go b/backend/controller/geoip.go index d12ac4aa..6f24f630 100644 --- a/backend/controller/geoip.go +++ b/backend/controller/geoip.go @@ -2,66 +2,49 @@ package controller import ( "github.com/gin-gonic/gin" - "github.com/phishingclub/phishingclub/geoip" + "github.com/phishingclub/phishingclub/ipdata" ) -// GeoIP is a controller for GeoIP-related endpoints +// GeoIP is a controller for GeoIP related endpoints type GeoIP struct { Common } -// GetMetadata returns the GeoIP metadata including available country codes +// GetMetadata returns the available country codes for the filter UI func (c *GeoIP) GetMetadata(g *gin.Context) { _, _, ok := c.handleSession(g) if !ok { return } - // get geoip instance - geo, err := geoip.Instance() - if ok := c.handleErrors(g, err); !ok { - return - } + codes := ipdata.Get().CountryCodes() - // get metadata - metadata := geo.GetMetadata() - if metadata == nil { - c.Response.BadRequest(g) - return - } - - c.Response.OK(g, metadata) + c.Response.OK(g, gin.H{ + "country_codes": codes, + "countries": ipdata.Get().Countries(), + "asn_available": ipdata.Get().ASNLoaded(), + }) } -// Lookup performs a GeoIP lookup for the provided IP address +// Lookup performs a country lookup for the provided IP address func (c *GeoIP) Lookup(g *gin.Context) { _, _, ok := c.handleSession(g) if !ok { return } - // get ip from query parameter ip := g.Query("ip") if ip == "" { c.Response.BadRequest(g) return } - // get geoip instance - geo, err := geoip.Instance() - if ok := c.handleErrors(g, err); !ok { - return - } + countryCode, found := ipdata.Get().LookupCountry(ip) - // perform lookup - countryCode, found := geo.Lookup(ip) - - // return result result := gin.H{ "ip": ip, "found": found, } - if found { result["country_code"] = countryCode } diff --git a/backend/controller/ipdata.go b/backend/controller/ipdata.go new file mode 100644 index 00000000..f9cfe1a3 --- /dev/null +++ b/backend/controller/ipdata.go @@ -0,0 +1,137 @@ +package controller + +import ( + "strconv" + "strings" + + "github.com/gin-gonic/gin" + "github.com/phishingclub/phishingclub/ipdata" + "github.com/phishingclub/phishingclub/service" +) + +// IPData is a controller for the country and ASN data packages +type IPData struct { + Common + IPDataService *service.IPData +} + +// Status returns the state of both data packages +func (c *IPData) Status(g *gin.Context) { + session, _, ok := c.handleSession(g) + if !ok { + return + } + status, err := c.IPDataService.Status(g, session) + if ok := c.handleErrors(g, err); !ok { + return + } + c.Response.OK(g, gin.H{"packages": status}) +} + +// Download fetches and installs the package named in the path +func (c *IPData) Download(g *gin.Context) { + session, _, ok := c.handleSession(g) + if !ok { + return + } + kind := g.Param("kind") + err := c.IPDataService.Download(g, session, kind) + if ok := c.handleErrors(g, err); !ok { + return + } + c.Response.OK(g, nil) +} + +// Remove deletes the installed package named in the path +func (c *IPData) Remove(g *gin.Context) { + session, _, ok := c.handleSession(g) + if !ok { + return + } + kind := g.Param("kind") + err := c.IPDataService.Remove(g, session, kind) + if ok := c.handleErrors(g, err); !ok { + return + } + c.Response.OK(g, nil) +} + +// SearchASN returns ASNs matching the query for the filter typeahead +func (c *IPData) SearchASN(g *gin.Context) { + _, _, ok := c.handleSession(g) + if !ok { + return + } + q := g.Query("q") + limit := 25 + if v := g.Query("limit"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + limit = n + } + } + results := ipdata.Get().SearchASN(q, limit) + c.Response.OK(g, gin.H{"results": results}) +} + +// LookupASN returns the autonomous systems that announce the given IP address +func (c *IPData) LookupASN(g *gin.Context) { + _, _, ok := c.handleSession(g) + if !ok { + return + } + ip := g.Query("ip") + if ip == "" { + c.Response.BadRequest(g) + return + } + store := ipdata.Get() + results := store.LookupASNDetails(ip) + c.Response.OK(g, gin.H{ + "ip": ip, + "available": store.ASNLoaded(), + "found": len(results) > 0, + "results": results, + }) +} + +// ResolveASNRequest is the body for resolving configured ASNs to names. +type ResolveASNRequest struct { + Asns []string `json:"asns"` +} + +// ResolveASNs returns the details of the given ASNs that exist in the dataset. +// ASNs that are absent are left out, which lets the UI flag orphaned entries. +func (c *IPData) ResolveASNs(g *gin.Context) { + _, _, ok := c.handleSession(g) + if !ok { + return + } + var req ResolveASNRequest + if ok := c.handleParseRequest(g, &req); !ok { + return + } + nums := make([]uint32, 0, len(req.Asns)) + for _, s := range req.Asns { + if n, ok := parseASNInput(s); ok { + nums = append(nums, n) + } + } + results := ipdata.Get().ResolveASNs(nums) + c.Response.OK(g, gin.H{"results": results}) +} + +// parseASNInput parses one ASN string with an optional AS or ASN prefix. +func parseASNInput(s string) (uint32, bool) { + v := strings.ToLower(strings.TrimSpace(s)) + v = strings.TrimPrefix(v, "asn") + v = strings.TrimPrefix(v, "as") + v = strings.TrimSpace(v) + if v == "" { + return 0, false + } + n, err := strconv.ParseUint(v, 10, 32) + if err != nil { + return 0, false + } + return uint32(n), true +} diff --git a/backend/database/allowDeny.go b/backend/database/allowDeny.go index e845abba..5aee2e2f 100644 --- a/backend/database/allowDeny.go +++ b/backend/database/allowDeny.go @@ -21,6 +21,7 @@ type AllowDeny struct { Cidrs string `gorm:"not null;default:''"` JA4Fingerprints string `gorm:"not null;default:''"` CountryCodes string `gorm:"not null;default:''"` + Asns string `gorm:"not null;default:''"` Headers string `gorm:"not null;default:''"` Allowed bool `gorm:"not null;"` } diff --git a/backend/geoip/geoip.go b/backend/geoip/geoip.go deleted file mode 100644 index 35afbb12..00000000 --- a/backend/geoip/geoip.go +++ /dev/null @@ -1,185 +0,0 @@ -package geoip - -import ( - "encoding/json" - "fmt" - "net" - "strings" - "sync" - - "github.com/phishingclub/phishingclub/embedded" -) - -// countryData represents the structure from embedded geoip files -type countryData struct { - Code string `json:"code"` - Name string `json:"name"` - IPv4 []string `json:"ipv4"` - IPv6 []string `json:"ipv6"` -} - -// metadata represents the geoip metadata file -type Metadata struct { - Generated string `json:"generated"` - Source string `json:"source"` - License string `json:"license"` - Countries int `json:"countries"` - CountryCodes []string `json:"country_codes"` -} - -// ipRange represents a single IP range with its country code -type ipRange struct { - network *net.IPNet - countryCode string -} - -// GeoIP provides IP to country lookup functionality -type GeoIP struct { - ranges []ipRange - metadata *Metadata - mu sync.RWMutex -} - -var ( - instance *GeoIP - once sync.Once - initErr error -) - -// Instance returns the singleton GeoIP instance -func Instance() (*GeoIP, error) { - once.Do(func() { - instance, initErr = New() - }) - return instance, initErr -} - -// New creates a new GeoIP instance by loading embedded data -func New() (*GeoIP, error) { - geo := &GeoIP{ - ranges: make([]ipRange, 0), - } - - // load metadata - if err := geo.loadMetadata(); err != nil { - return nil, fmt.Errorf("failed to load metadata: %w", err) - } - - // load all country data - if err := geo.loadAllCountries(); err != nil { - return nil, fmt.Errorf("failed to load countries: %w", err) - } - - return geo, nil -} - -// loadMetadata loads the metadata.json file -func (g *GeoIP) loadMetadata() error { - data, err := embedded.GeoIPData.ReadFile("geoip/metadata.json") - if err != nil { - return fmt.Errorf("failed to read metadata: %w", err) - } - - var meta Metadata - if err := json.Unmarshal(data, &meta); err != nil { - return fmt.Errorf("failed to parse metadata: %w", err) - } - - g.metadata = &meta - return nil -} - -// loadAllCountries loads all country IP ranges -func (g *GeoIP) loadAllCountries() error { - if g.metadata == nil { - return fmt.Errorf("metadata not loaded") - } - - for _, code := range g.metadata.CountryCodes { - if err := g.loadCountry(code); err != nil { - // log warning but continue - some countries may not have data - continue - } - } - - return nil -} - -// loadCountry loads IP ranges for a single country -func (g *GeoIP) loadCountry(countryCode string) error { - filename := fmt.Sprintf("geoip/%s.json", strings.ToLower(countryCode)) - data, err := embedded.GeoIPData.ReadFile(filename) - if err != nil { - return fmt.Errorf("failed to read %s: %w", countryCode, err) - } - - var country countryData - if err := json.Unmarshal(data, &country); err != nil { - return fmt.Errorf("failed to parse %s: %w", countryCode, err) - } - - // add all IPv4 ranges - for _, cidr := range country.IPv4 { - _, network, err := net.ParseCIDR(cidr) - if err != nil { - continue // skip invalid entries - } - g.ranges = append(g.ranges, ipRange{ - network: network, - countryCode: country.Code, - }) - } - - // add all IPv6 ranges - for _, cidr := range country.IPv6 { - _, network, err := net.ParseCIDR(cidr) - if err != nil { - continue // skip invalid entries - } - g.ranges = append(g.ranges, ipRange{ - network: network, - countryCode: country.Code, - }) - } - - return nil -} - -// Lookup finds the country code for an IP address -// returns (countryCode, found) -func (g *GeoIP) Lookup(ipStr string) (string, bool) { - g.mu.RLock() - defer g.mu.RUnlock() - - ip := net.ParseIP(ipStr) - if ip == nil { - return "", false - } - - // linear search through ranges - // for better performance, consider using a trie or radix tree - for _, r := range g.ranges { - if r.network.Contains(ip) { - return r.countryCode, true - } - } - - return "", false -} - -// GetMetadata returns the GeoIP metadata -func (g *GeoIP) GetMetadata() *Metadata { - g.mu.RLock() - defer g.mu.RUnlock() - return g.metadata -} - -// GetCountryCodes returns a list of all available country codes -func (g *GeoIP) GetCountryCodes() []string { - g.mu.RLock() - defer g.mu.RUnlock() - if g.metadata == nil { - return []string{} - } - return g.metadata.CountryCodes -} diff --git a/backend/ipdata/ipdata.go b/backend/ipdata/ipdata.go new file mode 100644 index 00000000..2db30750 --- /dev/null +++ b/backend/ipdata/ipdata.go @@ -0,0 +1,515 @@ +// Package ipdata loads the IP to country and IP to ASN data used by the +// allow and deny filters. It reads two sources: the country data embedded in +// the binary, and packages the operator downloads into the data directory. A +// downloaded package wins over the embedded copy. ASN data only exists as a +// download. +// +// Prefixes are held in memory as fixed compact records and looked up by +// longest prefix. The store can be rebuilt at runtime after a download without +// a restart. +package ipdata + +import ( + "encoding/binary" + "fmt" + "net/netip" + "sort" + "strings" + "sync" + + "go.uber.org/zap" +) + +const ( + // KindGeoIP is the country data package name. + KindGeoIP = "geoip" + // KindASN is the autonomous system data package name. + KindASN = "asn" +) + +// rec4 is one IPv4 prefix. val indexes into a dataset value table. +type rec4 struct { + addr uint32 + bits uint8 + val uint32 +} + +// rec6 is one IPv6 prefix. val indexes into a dataset value table. +type rec6 struct { + addr [16]byte + bits uint8 + val uint32 +} + +// index holds the sorted prefixes of one dataset for lookup. +type index struct { + v4 []rec4 + v6 []rec6 +} + +// add parses a CIDR, masks off host bits and appends it under the value. +// Bad or reserved input is skipped. It returns whether the prefix was kept. +func (x *index) add(cidr string, val uint32) bool { + p, err := netip.ParsePrefix(strings.TrimSpace(cidr)) + if err != nil { + return false + } + p = p.Masked() + a := p.Addr() + if a.Is4() { + b := a.As4() + x.v4 = append(x.v4, rec4{binary.BigEndian.Uint32(b[:]), uint8(p.Bits()), val}) + return true + } + if a.Is6() && !a.Is4In6() { + x.v6 = append(x.v6, rec6{a.As16(), uint8(p.Bits()), val}) + return true + } + return false +} + +// sortRecords orders the prefixes by address then prefix length so lookup can +// binary search each length. +func (x *index) sortRecords() { + sort.Slice(x.v4, func(i, j int) bool { + if x.v4[i].addr != x.v4[j].addr { + return x.v4[i].addr < x.v4[j].addr + } + return x.v4[i].bits < x.v4[j].bits + }) + sort.Slice(x.v6, func(i, j int) bool { + if c := byteCmp(x.v6[i].addr, x.v6[j].addr); c != 0 { + return c < 0 + } + return x.v6[i].bits < x.v6[j].bits + }) +} + +// lookup returns the value indexes of every prefix that matches the address at +// the longest matching prefix length. More than one is returned when several +// entries announce the same prefix, which happens with ASN data. +func (x *index) lookup(ip netip.Addr) []uint32 { + if ip.Is4() { + b := ip.As4() + return x.lookup4(binary.BigEndian.Uint32(b[:])) + } + if ip.Is4In6() { + b := ip.Unmap().As4() + return x.lookup4(binary.BigEndian.Uint32(b[:])) + } + return x.lookup6(ip.As16()) +} + +func (x *index) lookup4(ip uint32) []uint32 { + for bits := 32; bits >= 0; bits-- { + var mask uint32 = 0xffffffff + if bits < 32 { + mask <<= uint(32 - bits) + } + if bits == 0 { + mask = 0 + } + masked := ip & mask + lo := sort.Search(len(x.v4), func(i int) bool { return x.v4[i].addr >= masked }) + var vals []uint32 + for i := lo; i < len(x.v4) && x.v4[i].addr == masked; i++ { + if x.v4[i].bits == uint8(bits) { + vals = append(vals, x.v4[i].val) + } + } + if len(vals) > 0 { + return vals + } + } + return nil +} + +func (x *index) lookup6(ip [16]byte) []uint32 { + for bits := 128; bits >= 0; bits-- { + masked := maskV6(ip, bits) + lo := sort.Search(len(x.v6), func(i int) bool { return byteCmp(x.v6[i].addr, masked) >= 0 }) + var vals []uint32 + for i := lo; i < len(x.v6) && x.v6[i].addr == masked; i++ { + if x.v6[i].bits == uint8(bits) { + vals = append(vals, x.v6[i].val) + } + } + if len(vals) > 0 { + return vals + } + } + return nil +} + +func maskV6(a [16]byte, bits int) [16]byte { + var out [16]byte + full := bits / 8 + rem := bits % 8 + copy(out[:full], a[:full]) + if rem > 0 && full < 16 { + out[full] = a[full] & (byte(0xff) << uint(8-rem)) + } + return out +} + +func byteCmp(a, b [16]byte) int { + for i := 0; i < 16; i++ { + if a[i] != b[i] { + if a[i] < b[i] { + return -1 + } + return 1 + } + } + return 0 +} + +// Info describes a loaded dataset for the status endpoint. +type Info struct { + Name string `json:"name"` + Source string `json:"source"` + Version string `json:"version"` + Created string `json:"created"` + License string `json:"license"` + Entries int `json:"entries"` + IPv4Prefixes int `json:"ipv4Prefixes"` + IPv6Prefixes int `json:"ipv6Prefixes"` + ContentHash string `json:"contentHash"` + Downloaded bool `json:"downloaded"` +} + +// geoDataset maps prefixes to country codes. +type geoDataset struct { + idx index + codes []string + names []string + info Info +} + +// asnDataset maps prefixes to autonomous systems. +type asnDataset struct { + idx index + nums []uint32 + handles []string + names []string + countries []string + byNum map[uint32]int + info Info +} + +// Store holds the loaded datasets and swaps them safely on reload. +type Store struct { + mu sync.RWMutex + dataDir string + logger *zap.SugaredLogger + geo *geoDataset + asn *asnDataset +} + +var std *Store + +// Init loads the datasets from the embedded data and the data directory and +// installs the package level store. It is called once at startup. A failure to +// load a package is logged and leaves that dataset empty rather than stopping +// the server. +func Init(dataDir string, logger *zap.SugaredLogger) *Store { + s := &Store{dataDir: dataDir, logger: logger} + s.reloadGeo() + s.reloadASN() + std = s + return s +} + +// Get returns the package level store. Its methods are safe to call on a nil +// store and on empty datasets. +func Get() *Store { return std } + +// reloadGeo builds the country dataset from a downloaded package when present, +// otherwise from the embedded data. +func (s *Store) reloadGeo() { + if ds, err := s.loadGeoPackage(); err == nil { + s.setGeo(ds) + return + } else if s.logger != nil { + s.logger.Debugw("no downloaded geoip package, using embedded", "error", err) + } + ds, err := s.loadEmbeddedGeo() + if err != nil { + if s.logger != nil { + s.logger.Errorw("failed to load embedded geoip data", "error", err) + } + return + } + s.setGeo(ds) +} + +// reloadASN builds the ASN dataset from a downloaded package. There is no +// embedded fallback, so a missing package leaves ASN lookups empty. +func (s *Store) reloadASN() { + ds, err := s.loadASNPackage() + if err != nil { + s.setASN(nil) + if s.logger != nil { + s.logger.Debugw("no downloaded asn package", "error", err) + } + return + } + s.setASN(ds) +} + +func (s *Store) setGeo(ds *geoDataset) { + s.mu.Lock() + s.geo = ds + s.mu.Unlock() +} + +func (s *Store) setASN(ds *asnDataset) { + s.mu.Lock() + s.asn = ds + s.mu.Unlock() +} + +// LookupCountry returns the country code for an IP and whether one was found. +func (s *Store) LookupCountry(ipStr string) (string, bool) { + if s == nil { + return "", false + } + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return "", false + } + s.mu.RLock() + geo := s.geo + s.mu.RUnlock() + if geo == nil { + return "", false + } + vals := geo.idx.lookup(ip) + if len(vals) == 0 { + return "", false + } + return geo.codes[vals[0]], true +} + +// LookupASNs returns every autonomous system number that announces the longest +// prefix containing the IP. Usually one, sometimes several. +func (s *Store) LookupASNs(ipStr string) []uint32 { + if s == nil { + return nil + } + ip, err := netip.ParseAddr(ipStr) + if err != nil { + return nil + } + s.mu.RLock() + asn := s.asn + s.mu.RUnlock() + if asn == nil { + return nil + } + vals := asn.idx.lookup(ip) + if len(vals) == 0 { + return nil + } + out := make([]uint32, 0, len(vals)) + for _, v := range vals { + out = append(out, asn.nums[v]) + } + return out +} + +// LookupASNDetails returns the details of every ASN that announces the longest +// prefix containing the IP. Used by the ASN lookup tool. +func (s *Store) LookupASNDetails(ipStr string) []ASN { + nums := s.LookupASNs(ipStr) + if len(nums) == 0 { + return nil + } + return s.ResolveASNs(nums) +} + +// Country is one entry of the country list. +type Country struct { + Code string `json:"code"` + Name string `json:"name"` +} + +// Countries returns the available country codes and names, sorted by code. +func (s *Store) Countries() []Country { + if s == nil { + return nil + } + s.mu.RLock() + geo := s.geo + s.mu.RUnlock() + if geo == nil { + return nil + } + out := make([]Country, len(geo.codes)) + for i := range geo.codes { + out[i] = Country{Code: geo.codes[i], Name: geo.names[i]} + } + return out +} + +// CountryCodes returns just the available country codes, sorted. +func (s *Store) CountryCodes() []string { + if s == nil { + return nil + } + s.mu.RLock() + geo := s.geo + s.mu.RUnlock() + if geo == nil { + return nil + } + out := make([]string, len(geo.codes)) + copy(out, geo.codes) + return out +} + +// ASN is one entry of an ASN search or resolve result. +type ASN struct { + ASN uint32 `json:"asn"` + Handle string `json:"handle"` + Name string `json:"name"` + Country string `json:"country"` +} + +// ASNLoaded reports whether ASN data is currently loaded and usable. +func (s *Store) ASNLoaded() bool { + if s == nil { + return false + } + s.mu.RLock() + defer s.mu.RUnlock() + return s.asn != nil +} + +// HasASN reports whether the ASN exists in the loaded dataset. +func (s *Store) HasASN(num uint32) bool { + if s == nil { + return false + } + s.mu.RLock() + asn := s.asn + s.mu.RUnlock() + if asn == nil { + return false + } + _, ok := asn.byNum[num] + return ok +} + +// ResolveASNs returns the details of the given ASN numbers that exist. Numbers +// that are absent from the dataset are left out, which lets the caller show a +// warning for orphaned filter entries. +func (s *Store) ResolveASNs(nums []uint32) []ASN { + if s == nil { + return nil + } + s.mu.RLock() + asn := s.asn + s.mu.RUnlock() + if asn == nil { + return nil + } + out := []ASN{} + for _, n := range nums { + if i, ok := asn.byNum[n]; ok { + out = append(out, ASN{ASN: asn.nums[i], Handle: asn.handles[i], Name: asn.names[i], Country: asn.countries[i]}) + } + } + return out +} + +// SearchASN returns ASNs matching the query by number, name or handle, up to +// limit results. Matches are ranked so exact and prefix matches come before a +// match in the middle of a name, so a query like "M247" lists the M247 systems +// first. +func (s *Store) SearchASN(query string, limit int) []ASN { + if s == nil { + return nil + } + s.mu.RLock() + asn := s.asn + s.mu.RUnlock() + if asn == nil || query == "" { + return nil + } + if limit <= 0 || limit > 100 { + limit = 25 + } + q := strings.ToLower(strings.TrimSpace(query)) + + type scored struct { + a ASN + score int + } + var matches []scored + for i := range asn.nums { + numStr := fmt.Sprintf("%d", asn.nums[i]) + name := strings.ToLower(asn.names[i]) + handle := strings.ToLower(asn.handles[i]) + + score := -1 + switch { + case numStr == q || handle == q || name == q: + score = 0 + case strings.HasPrefix(numStr, q) || strings.HasPrefix(handle, q) || strings.HasPrefix(name, q): + score = 1 + case strings.Contains(name, q) || strings.Contains(handle, q): + score = 2 + } + if score >= 0 { + matches = append(matches, scored{ + a: ASN{ASN: asn.nums[i], Handle: asn.handles[i], Name: asn.names[i], Country: asn.countries[i]}, + score: score, + }) + } + } + + // best score first, then by ascending ASN number for a stable order + sort.SliceStable(matches, func(i, j int) bool { + if matches[i].score != matches[j].score { + return matches[i].score < matches[j].score + } + return matches[i].a.ASN < matches[j].a.ASN + }) + + out := []ASN{} + for i := 0; i < len(matches) && i < limit; i++ { + out = append(out, matches[i].a) + } + return out +} + +// Status returns the state of both packages for the settings screen. +func (s *Store) Status() map[string]Info { + out := map[string]Info{} + if s == nil { + return out + } + s.mu.RLock() + defer s.mu.RUnlock() + if s.geo != nil { + out[KindGeoIP] = s.geo.info + } + if s.asn != nil { + out[KindASN] = s.asn.info + } + return out +} + +// Reload rebuilds one dataset from disk after a download or a removal. +func (s *Store) Reload(kind string) { + if s == nil { + return + } + switch kind { + case KindGeoIP: + s.reloadGeo() + case KindASN: + s.reloadASN() + } +} diff --git a/backend/ipdata/ipdata_test.go b/backend/ipdata/ipdata_test.go new file mode 100644 index 00000000..352cb291 --- /dev/null +++ b/backend/ipdata/ipdata_test.go @@ -0,0 +1,124 @@ +package ipdata + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "go.uber.org/zap" +) + +func TestLongestPrefixAndMultiOrigin(t *testing.T) { + // two countries, one nested prefix to prove longest match wins + geo := buildGeo([]geoEntry{ + {Code: "US", Name: "United States", IPv4: []string{"8.0.0.0/8"}}, + {Code: "DE", Name: "Germany", IPv4: []string{"8.8.0.0/16"}, IPv6: []string{"2a01:4f8::/32"}}, + }) + s := &Store{geo: geo} + + if code, ok := s.LookupCountry("8.8.8.8"); !ok || code != "DE" { + t.Fatalf("8.8.8.8 got %q %v, want DE", code, ok) + } + if code, ok := s.LookupCountry("8.9.0.1"); !ok || code != "US" { + t.Fatalf("8.9.0.1 got %q %v, want US", code, ok) + } + if code, ok := s.LookupCountry("2a01:4f8::1"); !ok || code != "DE" { + t.Fatalf("ipv6 got %q %v, want DE", code, ok) + } + if _, ok := s.LookupCountry("1.2.3.4"); ok { + t.Fatal("1.2.3.4 should not match") + } + + // same prefix announced by two ASNs must return both + asn := buildASN([]asnEntry{ + {ASN: 64500, Handle: "A", Name: "Alpha", IPv4: []string{"203.0.113.0/24"}}, + {ASN: 64501, Handle: "B", Name: "Beta", IPv4: []string{"203.0.113.0/24"}}, + {ASN: 15169, Handle: "GOOGLE", Name: "Google LLC", IPv4: []string{"8.8.8.0/24"}}, + }) + s.asn = asn + + got := s.LookupASNs("203.0.113.9") + sort.Slice(got, func(i, j int) bool { return got[i] < got[j] }) + if !reflect.DeepEqual(got, []uint32{64500, 64501}) { + t.Fatalf("multi origin got %v, want [64500 64501]", got) + } + if got := s.LookupASNs("8.8.8.8"); len(got) != 1 || got[0] != 15169 { + t.Fatalf("8.8.8.8 asn got %v, want [15169]", got) + } + if !s.HasASN(15169) || s.HasASN(1) { + t.Fatal("HasASN wrong") + } + + // search and resolve + if res := s.SearchASN("goog", 10); len(res) != 1 || res[0].ASN != 15169 { + t.Fatalf("search got %v", res) + } + if res := s.ResolveASNs([]uint32{15169, 99999}); len(res) != 1 || res[0].ASN != 15169 { + t.Fatalf("resolve got %v, want only 15169", res) + } +} + +func TestNilStoreSafe(t *testing.T) { + var s *Store + if _, ok := s.LookupCountry("8.8.8.8"); ok { + t.Fatal("nil store should not match") + } + if s.LookupASNs("8.8.8.8") != nil { + t.Fatal("nil store asn should be nil") + } + if s.HasASN(1) { + t.Fatal("nil store HasASN should be false") + } +} + +func TestLoadPackageAndSearchRanking(t *testing.T) { + root := t.TempDir() + dir := filepath.Join(root, "ipdata", "asn") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + entries := []byte(`[ +{"asn":9009,"handle":"M247","name":"M247 Europe SRL","country":"RO","ipv4":["5.62.0.0/16"],"ipv6":[]}, +{"asn":329035,"handle":"M247AI-AS-US","name":"M247 LLC","country":"US","ipv4":["23.19.0.0/16"],"ipv6":[]}, +{"asn":15169,"handle":"GOOGLE","name":"Google LLC","country":"US","ipv4":["8.8.8.0/24"],"ipv6":[]} +]`) + sum := sha256.Sum256(entries) + info := map[string]any{ + "format": 1, "name": "asn", "version": "test", "created": "2026-01-01T00:00:00Z", + "source": "test", "license": "CC0-1.0", "entries": 3, + "ipv4_prefixes": 3, "ipv6_prefixes": 0, "content_hash": hex.EncodeToString(sum[:]), + } + infoBytes, _ := json.Marshal(info) + if err := os.WriteFile(filepath.Join(dir, "package.json"), infoBytes, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "entries.json"), entries, 0o644); err != nil { + t.Fatal(err) + } + + s := Init(root+"/", zap.NewNop().Sugar()) + + // a name search returns the matching systems, exact handle match first + res := s.SearchASN("M247", 10) + if len(res) != 2 { + t.Fatalf("search M247 got %d results, want 2: %+v", len(res), res) + } + if res[0].ASN != 9009 { + t.Fatalf("expected handle-exact AS9009 first, got %d", res[0].ASN) + } + + // number search + if r := s.SearchASN("15169", 10); len(r) != 1 || r[0].Handle != "GOOGLE" { + t.Fatalf("number search got %+v", r) + } + + // ip to asn still works from the loaded package + if got := s.LookupASNDetails("5.62.0.1"); len(got) != 1 || got[0].ASN != 9009 { + t.Fatalf("ip lookup got %+v", got) + } +} diff --git a/backend/ipdata/loaders.go b/backend/ipdata/loaders.go new file mode 100644 index 00000000..fd7dcac9 --- /dev/null +++ b/backend/ipdata/loaders.go @@ -0,0 +1,263 @@ +package ipdata + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/phishingclub/phishingclub/embedded" +) + +// packageFormat is the format version this reader understands. +const packageFormat = 1 + +// FilePackage and FileEntries are the two files inside a downloaded package. +const ( + filePackage = "package.json" + fileEntries = "entries.json" +) + +// packageInfo mirrors package.json in a downloaded package. +type packageInfo struct { + Format int `json:"format"` + Name string `json:"name"` + Version string `json:"version"` + Created string `json:"created"` + Source string `json:"source"` + License string `json:"license"` + Entries int `json:"entries"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` + ContentHash string `json:"content_hash"` +} + +type geoEntry struct { + Code string `json:"code"` + Name string `json:"name"` + IPv4 []string `json:"ipv4"` + IPv6 []string `json:"ipv6"` +} + +type asnEntry struct { + ASN uint32 `json:"asn"` + Handle string `json:"handle"` + Name string `json:"name"` + Country string `json:"country"` + IPv4 []string `json:"ipv4"` + IPv6 []string `json:"ipv6"` +} + +// packageDir is where a downloaded package of the given kind is extracted. +func (s *Store) packageDir(kind string) string { + return filepath.Join(s.dataDir, "ipdata", kind) +} + +// PackageDir returns the directory a package of the given kind is installed to. +func (s *Store) PackageDir(kind string) string { + return s.packageDir(kind) +} + +// DataRoot returns the directory that holds all installed packages. +func (s *Store) DataRoot() string { + return filepath.Join(s.dataDir, "ipdata") +} + +// Validate checks that a directory holds a readable package of the kind, so a +// download can be rejected before it is moved into place. +func Validate(dir, kind string) error { + _, entries, err := readPackage(dir, kind) + if err != nil { + return err + } + switch kind { + case KindGeoIP: + var l []geoEntry + if err := json.Unmarshal(entries, &l); err != nil { + return fmt.Errorf("parse %s: %w", fileEntries, err) + } + if len(l) == 0 { + return fmt.Errorf("geoip package has no entries") + } + case KindASN: + var l []asnEntry + if err := json.Unmarshal(entries, &l); err != nil { + return fmt.Errorf("parse %s: %w", fileEntries, err) + } + if len(l) == 0 { + return fmt.Errorf("asn package has no entries") + } + default: + return fmt.Errorf("unknown package kind %q", kind) + } + return nil +} + +// readPackage reads and verifies a downloaded package directory. +func readPackage(dir, wantName string) (*packageInfo, []byte, error) { + infoBytes, err := os.ReadFile(filepath.Join(dir, filePackage)) + if err != nil { + return nil, nil, err + } + var info packageInfo + if err := json.Unmarshal(infoBytes, &info); err != nil { + return nil, nil, fmt.Errorf("parse %s: %w", filePackage, err) + } + if info.Format != packageFormat { + return nil, nil, fmt.Errorf("package format %d, expected %d", info.Format, packageFormat) + } + if info.Name != wantName { + return nil, nil, fmt.Errorf("package is %q, expected %q", info.Name, wantName) + } + entries, err := os.ReadFile(filepath.Join(dir, fileEntries)) + if err != nil { + return nil, nil, err + } + sum := sha256.Sum256(entries) + if hex.EncodeToString(sum[:]) != info.ContentHash { + return nil, nil, fmt.Errorf("content hash does not match %s", filePackage) + } + return &info, entries, nil +} + +// loadGeoPackage builds the country dataset from a downloaded package. +func (s *Store) loadGeoPackage() (*geoDataset, error) { + info, entries, err := readPackage(s.packageDir(KindGeoIP), KindGeoIP) + if err != nil { + return nil, err + } + var list []geoEntry + if err := json.Unmarshal(entries, &list); err != nil { + return nil, fmt.Errorf("parse %s: %w", fileEntries, err) + } + ds := buildGeo(list) + ds.info = Info{ + Name: KindGeoIP, Source: info.Source, Version: info.Version, Created: info.Created, + License: info.License, Entries: len(list), + IPv4Prefixes: len(ds.idx.v4), IPv6Prefixes: len(ds.idx.v6), + ContentHash: info.ContentHash, Downloaded: true, + } + return ds, nil +} + +// loadASNPackage builds the ASN dataset from a downloaded package. +func (s *Store) loadASNPackage() (*asnDataset, error) { + info, entries, err := readPackage(s.packageDir(KindASN), KindASN) + if err != nil { + return nil, err + } + var list []asnEntry + if err := json.Unmarshal(entries, &list); err != nil { + return nil, fmt.Errorf("parse %s: %w", fileEntries, err) + } + ds := buildASN(list) + ds.info = Info{ + Name: KindASN, Source: info.Source, Version: info.Version, Created: info.Created, + License: info.License, Entries: len(list), + IPv4Prefixes: len(ds.idx.v4), IPv6Prefixes: len(ds.idx.v6), + ContentHash: info.ContentHash, Downloaded: true, + } + return ds, nil +} + +// embeddedMetadata mirrors the embedded geoip metadata.json. +type embeddedMetadata struct { + Generated string `json:"generated"` + Source string `json:"source"` + License string `json:"license"` + CountryCodes []string `json:"country_codes"` +} + +// embeddedCountry mirrors an embedded per country file. +type embeddedCountry struct { + Code string `json:"code"` + Name string `json:"name"` + IPv4 []string `json:"ipv4"` + IPv6 []string `json:"ipv6"` +} + +// loadEmbeddedGeo builds the country dataset from the data embedded in the +// binary. This keeps the current embedded format untouched. +func (s *Store) loadEmbeddedGeo() (*geoDataset, error) { + metaBytes, err := embedded.GeoIPData.ReadFile("geoip/metadata.json") + if err != nil { + return nil, err + } + var meta embeddedMetadata + if err := json.Unmarshal(metaBytes, &meta); err != nil { + return nil, fmt.Errorf("parse embedded metadata: %w", err) + } + list := make([]geoEntry, 0, len(meta.CountryCodes)) + for _, code := range meta.CountryCodes { + name := fmt.Sprintf("geoip/%s.json", strings.ToLower(code)) + data, err := embedded.GeoIPData.ReadFile(name) + if err != nil { + continue + } + var c embeddedCountry + if err := json.Unmarshal(data, &c); err != nil { + continue + } + list = append(list, geoEntry{Code: c.Code, Name: c.Name, IPv4: c.IPv4, IPv6: c.IPv6}) + } + ds := buildGeo(list) + ds.info = Info{ + Name: KindGeoIP, Source: meta.Source, Version: "embedded", Created: meta.Generated, + License: meta.License, Entries: len(list), + IPv4Prefixes: len(ds.idx.v4), IPv6Prefixes: len(ds.idx.v6), + Downloaded: false, + } + return ds, nil +} + +// buildGeo turns country entries into a sorted dataset. +func buildGeo(list []geoEntry) *geoDataset { + sort.Slice(list, func(i, j int) bool { return list[i].Code < list[j].Code }) + ds := &geoDataset{ + codes: make([]string, len(list)), + names: make([]string, len(list)), + } + for i, e := range list { + ds.codes[i] = e.Code + ds.names[i] = e.Name + for _, c := range e.IPv4 { + ds.idx.add(c, uint32(i)) + } + for _, c := range e.IPv6 { + ds.idx.add(c, uint32(i)) + } + } + ds.idx.sortRecords() + return ds +} + +// buildASN turns ASN entries into a sorted dataset. +func buildASN(list []asnEntry) *asnDataset { + sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN }) + ds := &asnDataset{ + nums: make([]uint32, len(list)), + handles: make([]string, len(list)), + names: make([]string, len(list)), + countries: make([]string, len(list)), + byNum: make(map[uint32]int, len(list)), + } + for i, e := range list { + ds.nums[i] = e.ASN + ds.handles[i] = e.Handle + ds.names[i] = e.Name + ds.countries[i] = e.Country + ds.byNum[e.ASN] = i + for _, c := range e.IPv4 { + ds.idx.add(c, uint32(i)) + } + for _, c := range e.IPv6 { + ds.idx.add(c, uint32(i)) + } + } + ds.idx.sortRecords() + return ds +} diff --git a/backend/main.go b/backend/main.go index a8725840..ca2822b4 100644 --- a/backend/main.go +++ b/backend/main.go @@ -27,6 +27,7 @@ import ( "github.com/phishingclub/phishingclub/database" "github.com/phishingclub/phishingclub/errs" "github.com/phishingclub/phishingclub/install" + "github.com/phishingclub/phishingclub/ipdata" "github.com/phishingclub/phishingclub/middleware" "github.com/phishingclub/phishingclub/model" "github.com/phishingclub/phishingclub/repository" @@ -235,6 +236,9 @@ func main() { logger.Errorw("failed to setup certmagic", "error", err) return } + // load the country and ASN data into memory. a downloaded package in the + // data directory wins over the embedded country data. + ipdata.Init(*flagFilePath, logger) // setup services, middleware and controllers services := app.NewServices( db, diff --git a/backend/model/allowDeny.go b/backend/model/allowDeny.go index 827feba7..fc8b4450 100644 --- a/backend/model/allowDeny.go +++ b/backend/model/allowDeny.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "regexp" + "strconv" "strings" "time" @@ -26,6 +27,7 @@ type AllowDeny struct { Cidrs nullable.Nullable[vo.IPNetSlice] `json:"cidrs"` JA4Fingerprints nullable.Nullable[string] `json:"ja4Fingerprints"` CountryCodes nullable.Nullable[string] `json:"countryCodes"` + Asns nullable.Nullable[string] `json:"asns"` Headers nullable.Nullable[string] `json:"headers"` Allowed nullable.Nullable[bool] `json:"allowed"` CompanyID nullable.Nullable[uuid.UUID] `json:"companyID"` @@ -62,6 +64,13 @@ func (r *AllowDeny) Validate() error { } } + hasASNs := false + if r.Asns.IsSpecified() { + if asns, err := r.Asns.Get(); err == nil && asns != "" { + hasASNs = true + } + } + hasHeaders := false if r.Headers.IsSpecified() { if headers, err := r.Headers.Get(); err == nil && headers != "" { @@ -69,12 +78,24 @@ func (r *AllowDeny) Validate() error { } } - if !hasCidrs && !hasJA4 && !hasCountryCodes && !hasHeaders { + if !hasCidrs && !hasJA4 && !hasCountryCodes && !hasASNs && !hasHeaders { return errs.NewValidationError( - errors.New("at least one of CIDRs, JA4 fingerprints, country codes, or headers must be provided"), + errors.New("at least one of CIDRs, JA4 fingerprints, country codes, ASNs, or headers must be provided"), ) } + // each ASN line must be a number, optionally prefixed with AS + if hasASNs { + asns, _ := r.Asns.Get() + for _, line := range parseCountryCodes(asns) { + if _, ok := parseASN(line); !ok { + return errs.NewValidationError( + fmt.Errorf("invalid ASN: %s", line), + ) + } + } + } + return nil } @@ -117,6 +138,12 @@ func (r *AllowDeny) ToDBMap() map[string]any { m["country_codes"] = codes } } + if r.Asns.IsSpecified() { + m["asns"] = "" + if asns, err := r.Asns.Get(); err == nil { + m["asns"] = asns + } + } if r.Headers.IsSpecified() { m["headers"] = "" if headers, err := r.Headers.Get(); err == nil { @@ -327,28 +354,19 @@ func isSpace(b byte) bool { // IsCountryAllowed checks if a country code is allowed based on the filter rules func (r *AllowDeny) IsCountryAllowed(countryCode string) bool { - if countryCode == "" { - // if no country code available, skip country check - return true - } - isTypeAllowList := r.Allowed.MustGet() // get country codes list countryCodesStr, err := r.CountryCodes.Get() if err != nil || countryCodesStr == "" { - // if no country codes configured, skip country check + // no country filter configured, this dimension does not restrict return true } - // if country code is empty but we have country filters configured if countryCode == "" { - // in allow list mode: unknown country should be denied - // in deny list mode: unknown country should be allowed - if isTypeAllowList { - return false - } - return true + // a country filter is configured but the visitor country is unknown. + // allow list denies the unknown visitor, deny list allows it. + return !isTypeAllowList } // parse country codes (newline separated) @@ -382,6 +400,81 @@ func (r *AllowDeny) IsCountryAllowed(countryCode string) bool { return true } +// IsASNAllowed checks if the autonomous systems that announce the visitor IP +// are allowed based on the filter rules. asns is every ASN that announces the +// longest prefix containing the IP, usually one, sometimes several. +func (r *AllowDeny) IsASNAllowed(asns []uint32) bool { + isTypeAllowList := r.Allowed.MustGet() + + // get asn list + asnStr, err := r.Asns.Get() + if err != nil || asnStr == "" { + // no asn filter configured, this dimension does not restrict + return true + } + + if len(asns) == 0 { + // an asn filter is configured but the visitor ASN is unknown. + // allow list denies the unknown visitor, deny list allows it. + return !isTypeAllowList + } + + // parse configured asns into a set + configured := map[uint32]struct{}{} + for _, line := range parseCountryCodes(asnStr) { + if n, ok := parseASN(line); ok { + configured[n] = struct{}{} + } + } + + // a visitor matches if any of its announcing ASNs is configured + isMatch := false + for _, n := range asns { + if _, ok := configured[n]; ok { + isMatch = true + break + } + } + + // if allow list and asn matches + if isTypeAllowList && isMatch { + return true + } + // if deny list and asn matches + if !isTypeAllowList && isMatch { + return false + } + + // If this is an allow list and asn didn't match, not allowed + if isTypeAllowList { + return false + } + + // If this is a deny list and asn didn't match, it is allowed + return true +} + +// parseASN parses a single ASN line into a number. It accepts an optional AS +// or ASN prefix, so "AS15169", "asn15169" and "15169" are all valid. +func parseASN(line string) (uint32, bool) { + s := trimSpace(line) + if s == "" { + return 0, false + } + lower := strings.ToLower(s) + lower = strings.TrimPrefix(lower, "asn") + lower = strings.TrimPrefix(lower, "as") + lower = trimSpace(lower) + if lower == "" { + return 0, false + } + n, err := strconv.ParseUint(lower, 10, 32) + if err != nil { + return 0, false + } + return uint32(n), true +} + // parseCountryCodes splits newline-separated country codes and trims whitespace func parseCountryCodes(input string) []string { var result []string diff --git a/backend/model/allowDeny_test.go b/backend/model/allowDeny_test.go new file mode 100644 index 00000000..abded2c4 --- /dev/null +++ b/backend/model/allowDeny_test.go @@ -0,0 +1,92 @@ +package model + +import ( + "testing" + + "github.com/oapi-codegen/nullable" + "github.com/phishingclub/phishingclub/vo" +) + +func filter(allowed bool, country, asns string) *AllowDeny { + r := &AllowDeny{Allowed: nullable.NewNullableWithValue(allowed)} + if country != "" { + r.CountryCodes = nullable.NewNullableWithValue(country) + } + if asns != "" { + r.Asns = nullable.NewNullableWithValue(asns) + } + return r +} + +func TestIsCountryAllowed(t *testing.T) { + cases := []struct { + name string + allowed bool + country string + visitor string + want bool + }{ + {"no filter configured passes", true, "", "DK", true}, + {"allow list match", true, "DK\nUS", "DK", true}, + {"allow list no match denies", true, "DK\nUS", "FR", false}, + {"deny list match denies", false, "RU", "RU", false}, + {"deny list no match allows", false, "RU", "DK", true}, + // the fail open fix: an unknown visitor country must be denied by an + // allow list and allowed by a deny list + {"allow list unknown visitor denied", true, "DK", "", false}, + {"deny list unknown visitor allowed", false, "RU", "", true}, + {"case insensitive", true, "dk", "DK", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := filter(c.allowed, c.country, "").IsCountryAllowed(c.visitor) + if got != c.want { + t.Fatalf("got %v want %v", got, c.want) + } + }) + } +} + +func TestIsASNAllowed(t *testing.T) { + cases := []struct { + name string + allowed bool + asns string + visitor []uint32 + want bool + }{ + {"no filter configured passes", true, "", []uint32{15169}, true}, + {"allow list match", true, "15169\n13335", []uint32{15169}, true}, + {"allow list no match denies", true, "15169", []uint32{13335}, false}, + {"allow list matches any announcing asn", true, "13335", []uint32{15169, 13335}, true}, + {"deny list match denies", false, "13335", []uint32{13335}, false}, + {"deny list no match allows", false, "13335", []uint32{15169}, true}, + {"AS prefix accepted", true, "AS15169", []uint32{15169}, true}, + // unknown visitor asn: allow list denies, deny list allows + {"allow list unknown visitor denied", true, "15169", nil, false}, + {"deny list unknown visitor allowed", false, "15169", nil, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := filter(c.allowed, "", c.asns).IsASNAllowed(c.visitor) + if got != c.want { + t.Fatalf("got %v want %v", got, c.want) + } + }) + } +} + +func TestValidateRejectsBadASN(t *testing.T) { + r := &AllowDeny{ + Name: nullable.NewNullableWithValue(*vo.NewString127Must("test")), + Allowed: nullable.NewNullableWithValue(true), + Asns: nullable.NewNullableWithValue("15169\nnotanumber"), + } + if err := r.Validate(); err == nil { + t.Fatal("expected validation error for bad ASN") + } + r.Asns = nullable.NewNullableWithValue("15169\nAS13335") + if err := r.Validate(); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/backend/proxy/proxy.go b/backend/proxy/proxy.go index c53fe7da..08d98a1f 100644 --- a/backend/proxy/proxy.go +++ b/backend/proxy/proxy.go @@ -29,7 +29,7 @@ import ( "github.com/phishingclub/phishingclub/cache" "github.com/phishingclub/phishingclub/data" "github.com/phishingclub/phishingclub/database" - "github.com/phishingclub/phishingclub/geoip" + "github.com/phishingclub/phishingclub/ipdata" "github.com/phishingclub/phishingclub/model" "github.com/phishingclub/phishingclub/repository" "github.com/phishingclub/phishingclub/server" @@ -5333,14 +5333,14 @@ func (m *ProxyHandler) checkFilter(req *http.Request, reqCtx *RequestContext) (b // get ja4 fingerprint from request header (set by middleware) ja4 := req.Header.Get(HEADER_JA4) - // get country code from GeoIP lookup - var countryCode string - if geo, err := geoip.Instance(); err == nil { - countryCode, _ = geo.Lookup(ip) - } + // get country code and ASNs from the IP data lookup + store := ipdata.Get() + countryCode, _ := store.LookupCountry(ip) + asns := store.LookupASNs(ip) m.logger.Debugw("checking geo ip", "ip", ip, "country", countryCode, + "asns", asns, ) // check IP, JA4, and country code against allow/deny lists @@ -5371,6 +5371,9 @@ func (m *ProxyHandler) checkFilter(req *http.Request, reqCtx *RequestContext) (b // check country code filter countryOk := allowDeny.IsCountryAllowed(countryCode) + // check ASN filter + asnOk := allowDeny.IsASNAllowed(asns) + // check header filter headers := req.Header headerOk, err := allowDeny.IsHeaderAllowed(headers) @@ -5382,13 +5385,13 @@ func (m *ProxyHandler) checkFilter(req *http.Request, reqCtx *RequestContext) (b // for deny lists: any filter failing blocks the request if isAllowListing { // allow list: all must be allowed - if ipOk && ja4Ok && countryOk && headerOk { + if ipOk && ja4Ok && countryOk && asnOk && headerOk { allowed = true break } } else { // deny list: if any filter denies, block the request - if !ipOk || !ja4Ok || !countryOk || !headerOk { + if !ipOk || !ja4Ok || !countryOk || !asnOk || !headerOk { allowed = false break } diff --git a/backend/repository/allowDeny.go b/backend/repository/allowDeny.go index ea15c9ab..1d3209ec 100644 --- a/backend/repository/allowDeny.go +++ b/backend/repository/allowDeny.go @@ -188,6 +188,7 @@ func ToAllowDeny(row *database.AllowDeny) *model.AllowDeny { ja4Fingerprints := nullable.NewNullableWithValue(row.JA4Fingerprints) countryCodes := nullable.NewNullableWithValue(row.CountryCodes) + asns := nullable.NewNullableWithValue(row.Asns) headers := nullable.NewNullableWithValue(row.Headers) return &model.AllowDeny{ @@ -198,6 +199,7 @@ func ToAllowDeny(row *database.AllowDeny) *model.AllowDeny { Cidrs: cidrsNullable, JA4Fingerprints: ja4Fingerprints, CountryCodes: countryCodes, + Asns: asns, Headers: headers, Allowed: nullable.NewNullableWithValue(row.Allowed), CompanyID: companyID, diff --git a/backend/service/allowDeny.go b/backend/service/allowDeny.go index c4755d90..ee006f1a 100644 --- a/backend/service/allowDeny.go +++ b/backend/service/allowDeny.go @@ -133,6 +133,9 @@ func (s *AllowDeny) Update( if v, err := incoming.CountryCodes.Get(); err == nil { current.CountryCodes.Set(v) } + if v, err := incoming.Asns.Get(); err == nil { + current.Asns.Set(v) + } if v, err := incoming.Headers.Get(); err == nil { current.Headers.Set(v) } diff --git a/backend/service/ipdata.go b/backend/service/ipdata.go new file mode 100644 index 00000000..8a338529 --- /dev/null +++ b/backend/service/ipdata.go @@ -0,0 +1,373 @@ +package service + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/go-errors/errors" + + "github.com/phishingclub/phishingclub/build" + "github.com/phishingclub/phishingclub/data" + "github.com/phishingclub/phishingclub/errs" + "github.com/phishingclub/phishingclub/ipdata" + "github.com/phishingclub/phishingclub/model" +) + +const ( + // ipdataBaseURL is the fixed location of the latest data packages. It takes + // no user input, so it adds no request forgery surface. + ipdataBaseURL = "https://github.com/phishingclub/ipdata/releases/latest/download" + + // ipdataMaxDownload caps a package download. The packages are a few MB, so + // this only stops a runaway response. + ipdataMaxDownload = 64 << 20 + + // ipdataMaxFile caps a single extracted file. + ipdataMaxFile = 128 << 20 +) + +// IPData manages the downloadable country and ASN data packages. +type IPData struct { + Common + Store *ipdata.Store + // mu serializes install and remove so two operators cannot race on the + // package directory + mu sync.Mutex +} + +// manifestPackage mirrors one package entry in the ipdata manifest. +type manifestPackage struct { + File string `json:"file"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + ContentHash string `json:"content_hash"` + Entries int `json:"entries"` + IPv4Prefixes int `json:"ipv4_prefixes"` + IPv6Prefixes int `json:"ipv6_prefixes"` +} + +// manifest mirrors the ipdata manifest.json. +type manifest struct { + Format int `json:"format"` + Version string `json:"version"` + Created string `json:"created"` + Packages map[string]manifestPackage `json:"packages"` +} + +// PackageStatus is the state of one package for the settings screen. +type PackageStatus struct { + Kind string `json:"kind"` + Installed bool `json:"installed"` + Info *ipdata.Info `json:"info,omitempty"` + UpdateAvailable bool `json:"updateAvailable"` + LatestVersion string `json:"latestVersion"` + LatestCreated string `json:"latestCreated"` +} + +func validKind(kind string) bool { + return kind == ipdata.KindGeoIP || kind == ipdata.KindASN +} + +func (s *IPData) httpClient(timeout time.Duration) *http.Client { + client := &http.Client{Timeout: timeout} + if !build.Flags.Production { + client.Transport = &http.Transport{ + // #nosec + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + return client +} + +// fetchManifest downloads and parses the current manifest. +func (s *IPData) fetchManifest() (*manifest, error) { + req, err := http.NewRequest(http.MethodGet, ipdataBaseURL+"/manifest.json", nil) + if err != nil { + return nil, errs.Wrap(err) + } + req.Header.Set("User-Agent", "PhishingClub-Client") + resp, err := s.httpClient(20 * time.Second).Do(req) + if err != nil { + return nil, errs.Wrap(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, errors.New("unexpected response fetching ipdata manifest") + } + var m manifest + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&m); err != nil { + return nil, errs.Wrap(err) + } + return &m, nil +} + +// Status returns the state of both packages, including whether a newer version +// is available. The remote check is best effort and never fails the call. +func (s *IPData) Status( + ctx context.Context, + session *model.Session, +) ([]PackageStatus, error) { + isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL) + if err != nil { + s.LogAuthError(err) + return nil, errs.Wrap(err) + } + if !isAuthorized { + return nil, errors.New("unauthorized") + } + + installed := s.Store.Status() + var remote *manifest + if m, err := s.fetchManifest(); err == nil { + remote = m + } else { + s.Logger.Debugw("could not fetch ipdata manifest", "error", err) + } + + out := make([]PackageStatus, 0, 2) + for _, kind := range []string{ipdata.KindGeoIP, ipdata.KindASN} { + ps := PackageStatus{Kind: kind} + if info, ok := installed[kind]; ok { + infoCopy := info + ps.Installed = info.Downloaded + ps.Info = &infoCopy + } + if remote != nil { + if rp, ok := remote.Packages[kind]; ok { + ps.LatestVersion = remote.Version + ps.LatestCreated = remote.Created + current := "" + if ps.Info != nil { + current = ps.Info.ContentHash + } + ps.UpdateAvailable = current != rp.ContentHash + } + } + out = append(out, ps) + } + return out, nil +} + +// Download fetches, verifies and installs the package of the given kind, then +// reloads it into the running store without a restart. +func (s *IPData) Download( + ctx context.Context, + session *model.Session, + kind string, +) error { + s.mu.Lock() + defer s.mu.Unlock() + + ae := NewAuditEvent("IPData.Download", session) + ae.Details["kind"] = kind + + isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL) + if err != nil { + s.LogAuthError(err) + return errs.Wrap(err) + } + if !isAuthorized { + s.AuditLogNotAuthorized(ae) + return errors.New("unauthorized") + } + if !validKind(kind) { + return errs.NewValidationError(fmt.Errorf("unknown package kind: %s", kind)) + } + + m, err := s.fetchManifest() + if err != nil { + return errs.NewOperationalError( + "Could not reach the data server. The data packages may not be published yet, or this server has no outbound internet access.", + err, + ) + } + pkg, ok := m.Packages[kind] + if !ok { + return errs.NewOperationalError( + fmt.Sprintf("The %s package is not available for download yet.", kind), + fmt.Errorf("manifest has no package %q", kind), + ) + } + + // download into the ipdata directory so the final rename stays on one + // filesystem + root := s.Store.DataRoot() + if err := os.MkdirAll(root, 0o750); err != nil { + return errs.Wrap(err) + } + tmpArchive, err := os.CreateTemp(root, ".dl-*.tar.gz") + if err != nil { + return errs.Wrap(err) + } + tmpArchivePath := tmpArchive.Name() + defer os.Remove(tmpArchivePath) + + // build the URL from the fixed kind, not from a manifest field, so no part + // of the path is influenced by the fetched manifest + url := ipdataBaseURL + "/" + kind + ".tar.gz" + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + tmpArchive.Close() + return errs.Wrap(err) + } + req.Header.Set("User-Agent", "PhishingClub-Client") + resp, err := s.httpClient(3 * time.Minute).Do(req) + if err != nil { + tmpArchive.Close() + return errs.NewOperationalError("Could not reach the data server to download the package.", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + tmpArchive.Close() + return errs.NewOperationalError( + "The data server did not return the package.", + fmt.Errorf("http %d downloading %s", resp.StatusCode, pkg.File), + ) + } + + // hash while copying, cap the size + hasher := sha256.New() + n, err := io.Copy(io.MultiWriter(tmpArchive, hasher), io.LimitReader(resp.Body, ipdataMaxDownload+1)) + tmpArchive.Close() + if err != nil { + return errs.NewOperationalError("The package download was interrupted.", err) + } + if n > ipdataMaxDownload { + return errs.NewOperationalError("The package is larger than the allowed size.", nil) + } + if got := hex.EncodeToString(hasher.Sum(nil)); got != pkg.SHA256 { + return errs.NewOperationalError( + "The downloaded package failed its integrity check.", + fmt.Errorf("sha256 mismatch for %s: want %s got %s", kind, pkg.SHA256, got), + ) + } + + // extract into a temp directory, then validate before moving into place + tmpDir, err := os.MkdirTemp(root, ".extract-*") + if err != nil { + return errs.Wrap(err) + } + defer os.RemoveAll(tmpDir) + if err := extractPackage(tmpArchivePath, tmpDir); err != nil { + return errs.NewOperationalError("The downloaded package could not be read.", err) + } + if err := ipdata.Validate(tmpDir, kind); err != nil { + return errs.NewOperationalError("The downloaded package is not valid.", err) + } + + // swap into the final location + finalDir := s.Store.PackageDir(kind) + oldDir := finalDir + ".old" + _ = os.RemoveAll(oldDir) + if _, err := os.Stat(finalDir); err == nil { + if err := os.Rename(finalDir, oldDir); err != nil { + return errs.Wrap(err) + } + } + if err := os.Rename(tmpDir, finalDir); err != nil { + // try to restore the previous package + _ = os.Rename(oldDir, finalDir) + return errs.Wrap(err) + } + _ = os.RemoveAll(oldDir) + + s.Store.Reload(kind) + ae.Details["version"] = pkg.ContentHash + s.AuditLogAuthorized(ae) + return nil +} + +// Remove deletes the installed package of the given kind and reloads the +// store. Removing geoip falls back to the embedded data, removing asn turns +// ASN lookups off. +func (s *IPData) Remove( + ctx context.Context, + session *model.Session, + kind string, +) error { + s.mu.Lock() + defer s.mu.Unlock() + + ae := NewAuditEvent("IPData.Remove", session) + ae.Details["kind"] = kind + + isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL) + if err != nil { + s.LogAuthError(err) + return errs.Wrap(err) + } + if !isAuthorized { + s.AuditLogNotAuthorized(ae) + return errors.New("unauthorized") + } + if !validKind(kind) { + return errs.NewValidationError(fmt.Errorf("unknown package kind: %s", kind)) + } + + if err := os.RemoveAll(s.Store.PackageDir(kind)); err != nil { + return errs.Wrap(err) + } + s.Store.Reload(kind) + s.AuditLogAuthorized(ae) + return nil +} + +// extractPackage unpacks package.json and entries.json from a gzip tar into +// dir. It reads only those two files by base name and rejects anything else. +func extractPackage(archivePath, dir string) error { + f, err := os.Open(archivePath) + if err != nil { + return err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + allowed := map[string]bool{"package.json": true, "entries.json": true} + seen := map[string]bool{} + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if hdr.Typeflag != tar.TypeReg { + continue + } + name := filepath.Base(hdr.Name) + if !allowed[name] { + continue + } + out, err := os.Create(filepath.Join(dir, name)) + if err != nil { + return err + } + if _, err := io.Copy(out, io.LimitReader(tr, ipdataMaxFile)); err != nil { + out.Close() + return err + } + out.Close() + seen[name] = true + } + if !seen["package.json"] || !seen["entries.json"] { + return errors.New("package archive is missing package.json or entries.json") + } + return nil +} diff --git a/frontend/src/lib/api/api.js b/frontend/src/lib/api/api.js index 93a58067..75e4c165 100644 --- a/frontend/src/lib/api/api.js +++ b/frontend/src/lib/api/api.js @@ -900,9 +900,7 @@ export class API { * @returns {Promise} */ getGroupedResultStats: async (campaignID, by = 'position') => { - return await getJSON( - this.getPath(`/campaign/${campaignID}/grouped-statistics?by=${by}`) - ); + return await getJSON(this.getPath(`/campaign/${campaignID}/grouped-statistics?by=${by}`)); }, /** @@ -3170,17 +3168,28 @@ export class API { * @param {string} allowdeny.cidrs * @param {string} allowdeny.ja4Fingerprints * @param {string} allowdeny.countryCodes + * @param {string} allowdeny.asns * @param {string} allowdeny.headers * @param {boolean} allowdeny.allowed * @param {string} allowdeny.companyID * @returns {Promise} */ - create: async ({ name, cidrs, ja4Fingerprints, countryCodes, headers, allowed, companyID }) => { + create: async ({ + name, + cidrs, + ja4Fingerprints, + countryCodes, + asns, + headers, + allowed, + companyID + }) => { return await postJSON(this.getPath('/allow-deny'), { name: name, cidrs: cidrs, ja4Fingerprints: ja4Fingerprints, countryCodes: countryCodes, + asns: asns, headers: headers, allowed: allowed, companyID: companyID @@ -3238,12 +3247,22 @@ export class API { * @param {string} allowdeny.companyID * @returns {Promise} */ - update: async ({ id, name, cidrs, ja4Fingerprints, countryCodes, headers, companyID }) => { + update: async ({ + id, + name, + cidrs, + ja4Fingerprints, + countryCodes, + asns, + headers, + companyID + }) => { return await patchJSON(this.getPath(`/allow-deny/${id}`), { name: name, cidrs: cidrs, ja4Fingerprints: ja4Fingerprints, countryCodes: countryCodes, + asns: asns, headers: headers, companyID: companyID }); @@ -3274,6 +3293,65 @@ export class API { } }; + /** + * ipdata is the API for the downloadable country and ASN data packages. + */ + ipdata = { + /** + * Get the state of both data packages, including whether an update is + * available. + * + * @returns {Promise} + */ + status: async () => { + return await getJSON(this.getPath('/ipdata/status')); + }, + + /** + * Download and install a package. + * + * @param {string} kind - "geoip" or "asn" + * @returns {Promise} + */ + download: async (kind) => { + return await postJSON(this.getPath(`/ipdata/package/${kind}/download`), {}); + }, + + /** + * Remove an installed package. + * + * @param {string} kind - "geoip" or "asn" + * @returns {Promise} + */ + remove: async (kind) => { + return await deleteJSON(this.getPath(`/ipdata/package/${kind}`)); + }, + + /** + * Search ASNs by number, name or handle for the filter typeahead. + * + * @param {string} query + * @param {number} [limit] + * @returns {Promise} + */ + searchASN: async (query, limit = 25) => { + return await getJSON( + this.getPath(`/ipdata/asn/search?q=${encodeURIComponent(query)}&limit=${limit}`) + ); + }, + + /** + * Resolve configured ASN numbers to their details. ASNs absent from the + * dataset are left out of the result so the UI can flag them. + * + * @param {string[]} asns + * @returns {Promise} + */ + resolveASN: async (asns) => { + return await postJSON(this.getPath('/ipdata/asn/resolve'), { asns }); + } + }; + /** * webhook is the API for web hook related operations. */ diff --git a/frontend/src/lib/components/AsnSelect.svelte b/frontend/src/lib/components/AsnSelect.svelte new file mode 100644 index 00000000..ef6ec6ff --- /dev/null +++ b/frontend/src/lib/components/AsnSelect.svelte @@ -0,0 +1,362 @@ + + +
+
+
+

+ ASNs +

+ {#if toolTipText.length > 0} + + {toolTipText} + + {/if} +
+

optional

+
+
+
+ + {#if !installed} +

+ No ASN data downloaded. Enable it in + Settings, IP Data. +

+ {:else} +
+
+ 0 ? displayValue : placeholder} + /> + {#if showDropdown} + + {/if} + {#if value.length > 0} + + {/if} +
+ + {#if showDropdown} +
+
    + {#if results.length > 1} +
  • + +
    • + {/if} + {#each results as r (r.asn)} +
  • + +
    • + {/each} +
+
+ {/if} + + {#if value.length > 0} +
+ {#each value as raw (raw)} + + {/each} +
+ {/if} +
+ {/if} +
diff --git a/frontend/src/routes/filter/+page.svelte b/frontend/src/routes/filter/+page.svelte index 8429e74f..759ec1a7 100644 --- a/frontend/src/routes/filter/+page.svelte +++ b/frontend/src/routes/filter/+page.svelte @@ -42,6 +42,7 @@ runBulkDelete } from '$lib/service/tableSelection.js'; import TextFieldMultiSelect from '$lib/components/TextFieldMultiSelect.svelte'; + import AsnSelect from '$lib/components/AsnSelect.svelte'; // services const appStateService = AppStateService.instance; @@ -54,6 +55,7 @@ cidrs: null, ja4Fingerprints: null, countryCodes: [], + asns: [], headers: [], allowed: null }; @@ -83,6 +85,7 @@ let modalMode = null; let modalText = ''; let availableCountryCodes = []; + let asnAvailable = false; let isDeleteAlertVisible = false; let deleteValues = { @@ -121,6 +124,7 @@ const res = await api.geoip.getMetadata(); if (res.success && res.data) { availableCountryCodes = res.data.country_codes || []; + asnAvailable = res.data.asn_available || false; } } catch (e) { console.error('failed to load geoip metadata', e); @@ -175,15 +179,16 @@ }; const onClickSubmit = async () => { - // validate that at least one of cidrs, ja4Fingerprints, countryCodes, or headers is provided + // validate that at least one of cidrs, ja4Fingerprints, countryCodes, asns, or headers is provided const hasCidrs = formValues.cidrs && formValues.cidrs.trim().length > 0; const hasJA4 = formValues.ja4Fingerprints && formValues.ja4Fingerprints.trim().length > 0; const hasCountryCodes = formValues.countryCodes && formValues.countryCodes.length > 0; + const hasAsns = formValues.asns && formValues.asns.length > 0; const hasHeaders = formValues.headers && formValues.headers.length > 0; - if (!hasCidrs && !hasJA4 && !hasCountryCodes && !hasHeaders) { + if (!hasCidrs && !hasJA4 && !hasCountryCodes && !hasAsns && !hasHeaders) { formError = - 'At least one of CIDRs, JA4 fingerprints, Country Codes, or Headers must be provided'; + 'At least one of CIDRs, JA4 fingerprints, Country Codes, ASNs, or Headers must be provided'; return; } @@ -226,6 +231,7 @@ cidrs: formValues.cidrs, ja4Fingerprints: formValues.ja4Fingerprints || '', countryCodes: formValues.countryCodes.join('\n'), + asns: formValues.asns.join('\n'), headers: headersStr, allowed: formValues.allowed, companyID: contextCompanyID @@ -269,6 +275,7 @@ cidrs: formValues.cidrs, ja4Fingerprints: formValues.ja4Fingerprints || '', countryCodes: formValues.countryCodes.join('\n'), + asns: formValues.asns.join('\n'), headers: headersStr, companyID: formValues.companyID }); @@ -312,6 +319,16 @@ const openCreateModal = () => { modalMode = 'create'; + formValues = { + id: null, + name: null, + cidrs: null, + ja4Fingerprints: null, + countryCodes: [], + asns: [], + headers: [], + allowed: null + }; isModalVisible = true; }; @@ -373,6 +390,15 @@ .filter((code) => code.length > 0); } + // parse asns from newline-separated string to array + let asnsArray = []; + if (allowDeny.asns) { + asnsArray = allowDeny.asns + .split('\n') + .map((a) => a.trim()) + .filter((a) => a.length > 0); + } + // parse headers from json string to array let headersArray = []; if (allowDeny.headers) { @@ -390,6 +416,7 @@ cidrs: allowDeny.cidrs, ja4Fingerprints: allowDeny.ja4Fingerprints || '', countryCodes: countryCodesArray, + asns: asnsArray, headers: headersArray, allowed: allowDeny.allowed, companyID: allowDeny.companyID @@ -597,6 +624,11 @@ > GeoIP Country Codes + diff --git a/frontend/src/routes/settings/+page.svelte b/frontend/src/routes/settings/+page.svelte index 0c608a14..95e018f2 100644 --- a/frontend/src/routes/settings/+page.svelte +++ b/frontend/src/routes/settings/+page.svelte @@ -12,6 +12,7 @@ import RedTeam from './panels/RedTeam.svelte'; import System from './panels/System.svelte'; import Branding from './panels/Branding.svelte'; + import IPData from './panels/IPData.svelte'; // Red Team panel is only relevant in red team phishing (blackbox) mode $: tabs = [ @@ -19,6 +20,7 @@ { id: 'access', label: 'Access', component: Access }, { id: 'scim', label: 'SCIM', component: Scim }, { id: 'data', label: 'Data', component: Data }, + { id: 'ipdata', label: 'IP Data', component: IPData }, { id: 'reports', label: 'Reports', component: Reports }, ...($displayMode === DISPLAY_MODE.BLACKBOX ? [{ id: 'redteam', label: 'Red Team', component: RedTeam }] diff --git a/frontend/src/routes/settings/panels/IPData.svelte b/frontend/src/routes/settings/panels/IPData.svelte new file mode 100644 index 00000000..cad8f687 --- /dev/null +++ b/frontend/src/routes/settings/panels/IPData.svelte @@ -0,0 +1,154 @@ + + +{#if loaded} +
+ {#each packages as p (p.kind)} + +
+

+ {meta[p.kind]?.blurb || ''} +

+
+

+ {#if p.info && p.info.downloaded} + Downloaded + {:else if p.info} + Built in + {:else} + Not downloaded + {/if} +

+ {#if p.info} +

+ {#if p.info.created}{fmtDate(p.info.created)} · + {/if}{(p.info.ipv4Prefixes + p.info.ipv6Prefixes).toLocaleString()} prefixes +

+ {/if} + {#if p.updateAvailable} +

+ Update available +

+ {/if} +
+
+ +
+ {#if p.info && p.info.downloaded} + + {/if} + +
+ + + {/each} +
+{/if} diff --git a/frontend/src/routes/tools/+page.svelte b/frontend/src/routes/tools/+page.svelte index 42baa743..d5fdb80d 100644 --- a/frontend/src/routes/tools/+page.svelte +++ b/frontend/src/routes/tools/+page.svelte @@ -5,11 +5,13 @@ import JA4Builder from './panels/JA4Builder.svelte'; import CalendarBuilder from './panels/CalendarBuilder.svelte'; import GeoIP from './panels/GeoIP.svelte'; + import ASN from './panels/ASN.svelte'; const tabs = [ { id: 'calendar', label: 'Calendar Invitation Builder', component: CalendarBuilder }, { id: 'ja4', label: 'JA4 Fingerprint Builder', component: JA4Builder }, - { id: 'geoip', label: 'GeoIP Lookup', component: GeoIP } + { id: 'geoip', label: 'GeoIP Lookup', component: GeoIP }, + { id: 'asn', label: 'ASN Lookup', component: ASN } ]; let active = 'calendar'; diff --git a/frontend/src/routes/tools/panels/ASN.svelte b/frontend/src/routes/tools/panels/ASN.svelte new file mode 100644 index 00000000..0db643f8 --- /dev/null +++ b/frontend/src/routes/tools/panels/ASN.svelte @@ -0,0 +1,162 @@ + + +
+ +
+ + Search + + + + +
+ Data from + + ipverse/asn-ip + +
+ + {#if !asnAvailable} +
+

+ No ASN data downloaded. Enable it in + Settings, IP Data. +

+
+ {:else if results !== null} + {#if results.length > 0} +
+ {#each results as r (r.asn)} +

+ AS{r.asn} + {r.name || r.handle}{#if r.country} + · {r.country}{/if}{#if r.handle} + · {r.handle}{/if} +

+ {/each} +
+ {:else} +
+

+ {mode === 'ip' ? 'No match' : 'No results'} +

+
+ {/if} + {/if} +
+ + + + +