ip filtering becomes filtering with ja4 and cidrs

Signed-off-by: Ronni Skansing <rskansing@gmail.com>
This commit is contained in:
Ronni Skansing
2025-11-05 21:10:33 +01:00
parent ff5921ec02
commit 691cff9659
12 changed files with 291 additions and 87 deletions
+28 -8
View File
@@ -1815,6 +1815,9 @@ func (s *Server) checkIPFilter(
domain *database.Domain,
campaignID *uuid.UUID,
) (bool, error) {
// get ja4 fingerprint from context
ja4 := middleware.GetJA4FromContext(ctx)
allowDenyLEntries, err := s.repositories.Campaign.GetAllDenyByCampaignID(ctx, campaignID)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
s.logger.Debugw("failed to get deny list for campaign",
@@ -1823,9 +1826,9 @@ func (s *Server) checkIPFilter(
)
return false, fmt.Errorf("failed to get deny list for campaign: %s", err)
}
// if there is a deny list, check if the IP allowed / denied
// when allow listing we must check all entries to see if we have a allowed IP
// when deny listing only a single entry needs to deny the IP
// if there is a deny list, check if the IP and JA4 allowed / denied
// when allow listing we must check all entries to see if we have a allowed IP/JA4
// when deny listing only a single entry needs to deny the IP/JA4
isAllowListing := false
allowed := len(allowDenyLEntries) == 0
for i, allowDeny := range allowDenyLEntries {
@@ -1836,22 +1839,38 @@ func (s *Server) checkIPFilter(
allowed = true
}
}
ok, err := allowDeny.IsIPAllowed(ip)
// check IP filter
ipOk, err := allowDeny.IsIPAllowed(ip)
if err != nil {
return false, errs.Wrap(err)
}
// check JA4 filter
ja4Ok, err := allowDeny.IsJA4Allowed(ja4)
if err != nil {
return false, errs.Wrap(err)
}
// both IP and JA4 must pass for the filter to allow
ok := ipOk && ja4Ok
if isAllowListing && ok {
s.logger.Debugw("IP is allow listed",
s.logger.Debugw("IP and JA4 are allow listed",
"ip", ip,
"ja4", ja4,
"list name", allowDeny.Name.MustGet().String(),
"list id", allowDeny.ID.MustGet().String(),
)
allowed = true
break
// if it is a deny list and a IP is not ok, we can break
// if it is a deny list and a IP/JA4 is not ok, we can break
} else if !isAllowListing && !ok {
s.logger.Debugw("IP is deny listed",
s.logger.Debugw("IP or JA4 is deny listed",
"ip", ip,
"ja4", ja4,
"ipOk", ipOk,
"ja4Ok", ja4Ok,
"list name", allowDeny.Name.MustGet().String(),
"list id", allowDeny.ID.MustGet().String(),
)
@@ -1860,8 +1879,9 @@ func (s *Server) checkIPFilter(
}
}
if !allowed {
s.logger.Debugw("IP is not allowed",
s.logger.Debugw("IP or JA4 is not allowed",
"ip", ip,
"ja4", ja4,
)
if denyPageID, err := campaign.DenyPageID.Get(); err == nil {
err = s.renderDenyPage(ctx, domain, &denyPageID)
+8 -7
View File
@@ -13,13 +13,14 @@ const (
// AllowDeny is a gorm data model for allow deny listing
type AllowDeny struct {
ID *uuid.UUID `gorm:"primary_key;not null;unique;type:uuid"`
CreatedAt *time.Time `gorm:"not null;index;"`
UpdatedAt *time.Time `gorm:"not null;index"`
CompanyID *uuid.UUID `gorm:"uniqueIndex:idx_allow_denies_unique_name_and_company_id;type:uuid"`
Name string `gorm:"not null;uniqueIndex:idx_allow_denies_unique_name_and_company_id;"`
Cidrs string `gorm:"not null;"`
Allowed bool `gorm:"not null;"`
ID *uuid.UUID `gorm:"primary_key;not null;unique;type:uuid"`
CreatedAt *time.Time `gorm:"not null;index;"`
UpdatedAt *time.Time `gorm:"not null;index"`
CompanyID *uuid.UUID `gorm:"uniqueIndex:idx_allow_denies_unique_name_and_company_id;type:uuid"`
Name string `gorm:"not null;uniqueIndex:idx_allow_denies_unique_name_and_company_id;"`
Cidrs string `gorm:"not null;default:''"`
JA4Fingerprints string `gorm:"not null;default:''"`
Allowed bool `gorm:"not null;"`
}
func (AllowDeny) TableName() string {
+132 -14
View File
@@ -16,13 +16,14 @@ import (
// AllowDeny is a model for allow deny listing
type AllowDeny struct {
ID nullable.Nullable[uuid.UUID] `json:"id"`
CreatedAt *time.Time `json:"createdAt"`
UpdatedAt *time.Time `json:"updatedAt"`
Name nullable.Nullable[vo.String127] `json:"name"`
Cidrs nullable.Nullable[vo.IPNetSlice] `json:"cidrs"`
Allowed nullable.Nullable[bool] `json:"allowed"`
CompanyID nullable.Nullable[uuid.UUID] `json:"companyID"`
ID nullable.Nullable[uuid.UUID] `json:"id"`
CreatedAt *time.Time `json:"createdAt"`
UpdatedAt *time.Time `json:"updatedAt"`
Name nullable.Nullable[vo.String127] `json:"name"`
Cidrs nullable.Nullable[vo.IPNetSlice] `json:"cidrs"`
JA4Fingerprints nullable.Nullable[string] `json:"ja4Fingerprints"`
Allowed nullable.Nullable[bool] `json:"allowed"`
CompanyID nullable.Nullable[uuid.UUID] `json:"companyID"`
}
// Validate checks if the allow deny list has a valid state
@@ -30,17 +31,31 @@ func (r *AllowDeny) Validate() error {
if err := validate.NullableFieldRequired("name", r.Name); err != nil {
return err
}
if err := validate.NullableFieldRequired("cidrs", r.Cidrs); err != nil {
return err
}
if err := validate.NullableFieldRequired("filter type", r.Allowed); err != nil {
return err
}
if v := r.Cidrs.MustGet(); len(v) == 0 {
// at least one of cidrs or ja4 fingerprints must be provided
hasCidrs := false
if r.Cidrs.IsSpecified() {
if cidrs, err := r.Cidrs.Get(); err == nil && len(cidrs) > 0 {
hasCidrs = true
}
}
hasJA4 := false
if r.JA4Fingerprints.IsSpecified() {
if ja4, err := r.JA4Fingerprints.Get(); err == nil && ja4 != "" {
hasJA4 = true
}
}
if !hasCidrs && !hasJA4 {
return errs.NewValidationError(
errors.New("cidrs must include atleast one CIDR"),
errors.New("at least one of CIDRs or JA4 fingerprints must be provided"),
)
}
return nil
}
@@ -71,6 +86,12 @@ func (r *AllowDeny) ToDBMap() map[string]any {
m["cidrs"] = cidrsStr
}
}
if r.JA4Fingerprints.IsSpecified() {
m["ja4_fingerprints"] = ""
if ja4, err := r.JA4Fingerprints.Get(); err == nil {
m["ja4_fingerprints"] = ja4
}
}
if r.Allowed.IsSpecified() {
m["allowed"] = nil
if allowed, err := r.Allowed.Get(); err == nil {
@@ -90,9 +111,11 @@ func (r *AllowDeny) ToDBMap() map[string]any {
func (r *AllowDeny) IsIPAllowed(ip string) (bool, error) {
isTypeAllowList := r.Allowed.MustGet()
// if no cidrs configured, skip ip check (always pass)
cidrs, err := r.Cidrs.Get()
if err != nil {
return false, errs.Wrap(err)
if err != nil || len(cidrs) == 0 {
return true, nil
}
netIP := net.ParseIP(ip)
@@ -120,3 +143,98 @@ func (r *AllowDeny) IsIPAllowed(ip string) (bool, error) {
// If this is a deny list and we didn't find the IP, it is allowed
return true, nil
}
// IsJA4Allowed checks if a JA4 fingerprint is allowed based on the filter rules
func (r *AllowDeny) IsJA4Allowed(ja4 string) (bool, error) {
if ja4 == "" {
// if no ja4 fingerprint available, skip ja4 check
return true, nil
}
isTypeAllowList := r.Allowed.MustGet()
// get ja4 fingerprints list
ja4FingerprintsStr, err := r.JA4Fingerprints.Get()
if err != nil || ja4FingerprintsStr == "" {
// if no ja4 fingerprints configured, skip ja4 check
return true, nil
}
// parse fingerprints (newline separated)
fingerprints := parseFingerprints(ja4FingerprintsStr)
// check if ja4 matches any fingerprint
isMatch := false
for _, fp := range fingerprints {
if fp == ja4 {
isMatch = true
break
}
}
// if allow list and ja4 matches
if isTypeAllowList && isMatch {
return true, nil
}
// if deny list and ja4 matches
if !isTypeAllowList && isMatch {
return false, nil
}
// If this is an allow list and ja4 didn't match, not allowed
if isTypeAllowList {
return false, nil
}
// If this is a deny list and ja4 didn't match, it is allowed
return true, nil
}
// parseFingerprints splits newline-separated fingerprints and trims whitespace
func parseFingerprints(input string) []string {
var result []string
lines := splitLines(input)
for _, line := range lines {
trimmed := trimSpace(line)
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
// splitLines splits a string by newlines
func splitLines(s string) []string {
var lines []string
start := 0
for i := 0; i < len(s); i++ {
if s[i] == '\n' {
lines = append(lines, s[start:i])
start = i + 1
}
}
if start < len(s) {
lines = append(lines, s[start:])
}
return lines
}
// trimSpace removes leading and trailing whitespace
func trimSpace(s string) string {
start := 0
end := len(s)
for start < end && isSpace(s[start]) {
start++
}
for end > start && isSpace(s[end-1]) {
end--
}
return s[start:end]
}
// isSpace checks if a byte is whitespace
func isSpace(b byte) bool {
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
}
+21 -3
View File
@@ -3631,7 +3631,7 @@ func (m *ProxyHandler) registerEvasionPageVisitEventDirect(req *http.Request, re
}
}
// checkIPFilter checks if the IP is allowed for proxy requests
// checkIPFilter checks if the IP and JA4 are allowed for proxy requests
// returns (blocked, response) where blocked=true means the IP should be blocked
func (m *ProxyHandler) checkIPFilter(req *http.Request, reqCtx *RequestContext) (bool, *http.Response) {
// use cached campaign info
@@ -3658,7 +3658,15 @@ func (m *ProxyHandler) checkIPFilter(req *http.Request, reqCtx *RequestContext)
return false, nil
}
// check IP against allow/deny lists
// get ja4 fingerprint from request context
ja4 := ""
if val := req.Context().Value("ja4_fingerprint"); val != nil {
if fp, ok := val.(string); ok {
ja4 = fp
}
}
// check IP and JA4 against allow/deny lists
isAllowListing := false
allowed := false // for allow lists, default is deny
@@ -3671,11 +3679,21 @@ func (m *ProxyHandler) checkIPFilter(req *http.Request, reqCtx *RequestContext)
}
}
ok, err := allowDeny.IsIPAllowed(ip)
// check IP filter
ipOk, err := allowDeny.IsIPAllowed(ip)
if err != nil {
continue
}
// check JA4 filter
ja4Ok, err := allowDeny.IsJA4Allowed(ja4)
if err != nil {
continue
}
// both IP and JA4 must pass for the filter to allow
ok := ipOk && ja4Ok
if isAllowListing && ok {
allowed = true
break
+10 -7
View File
@@ -186,13 +186,16 @@ func ToAllowDeny(row *database.AllowDeny) *model.AllowDeny {
}
cidrsNullable := nullable.NewNullableWithValue(cidrs)
ja4Fingerprints := nullable.NewNullableWithValue(row.JA4Fingerprints)
return &model.AllowDeny{
ID: id,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
Name: name,
Cidrs: cidrsNullable,
Allowed: nullable.NewNullableWithValue(row.Allowed),
CompanyID: companyID,
ID: id,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
Name: name,
Cidrs: cidrsNullable,
JA4Fingerprints: ja4Fingerprints,
Allowed: nullable.NewNullableWithValue(row.Allowed),
CompanyID: companyID,
}
}
+14 -8
View File
@@ -5,7 +5,6 @@ import (
"net"
"strings"
"github.com/go-errors/errors"
"github.com/phishingclub/phishingclub/errs"
)
@@ -13,23 +12,30 @@ type IPNetSlice []IPNet
// UnmarshalJSON implements custom unmarshaling for IPNetSlice
func (s *IPNetSlice) UnmarshalJSON(data []byte) error {
// if empty string
if strings.TrimSpace(string(data)) == "\"\"" {
return errors.New("CIDRs is empty.")
}
var str string
if err := json.Unmarshal(data, &str); err != nil {
return err
}
// if empty string, return empty slice (allow empty cidrs for ja4-only filters)
if strings.TrimSpace(str) == "" {
*s = IPNetSlice{}
return nil
}
strs := strings.Split(str, "\n")
// Convert each string to IPNet
result := make(IPNetSlice, len(strs))
for i, cidr := range strs {
result := make(IPNetSlice, 0, len(strs))
for _, cidr := range strs {
// skip empty lines
if strings.TrimSpace(cidr) == "" {
continue
}
ipnet, err := NewIPNet(cidr)
if err != nil {
return unwrapError(err)
}
result[i] = *ipnet
result = append(result, *ipnet)
}
*s = result
+6 -2
View File
@@ -2657,14 +2657,16 @@ export class API {
* @param {Object} allowdeny
* @param {string} allowdeny.name
* @param {string} allowdeny.cidrs
* @param {string} allowdeny.ja4Fingerprints
* @param {boolean} allowdeny.allowed
* @param {string} allowdeny.companyID
* @returns {Promise<ApiResponse>}
*/
create: async ({ name, cidrs, allowed, companyID }) => {
create: async ({ name, cidrs, ja4Fingerprints, allowed, companyID }) => {
return await postJSON(this.getPath('/allow-deny'), {
name: name,
cidrs: cidrs,
ja4Fingerprints: ja4Fingerprints,
allowed: allowed,
companyID: companyID
});
@@ -2715,13 +2717,15 @@ export class API {
* @param {string} allowdeny.id
* @param {string} allowdeny.name
* @param {string} allowdeny.cidrs
* @param {string} allowdeny.ja4Fingerprints
* @param {string} allowdeny.companyID
* @returns {Promise<ApiResponse>}
*/
update: async ({ id, name, cidrs, companyID }) => {
update: async ({ id, name, cidrs, ja4Fingerprints, companyID }) => {
return await patchJSON(this.getPath(`/allow-deny/${id}`), {
name: name,
cidrs: cidrs,
ja4Fingerprints: ja4Fingerprints,
companyID: companyID
});
},
@@ -80,7 +80,7 @@
<path stroke-linecap="round" stroke-linejoin="round" d="M19.5 14.25v-2.625a3.375 3.375 0 0 0-3.375-3.375h-1.5A1.125 1.125 0 0 1 13.5 7.125v-1.5a3.375 3.375 0 0 0-3.375-3.375H8.25m0 12.75h7.5m-7.5 3H12M10.5 2.25H5.625c-.621 0-1.125.504-1.125 1.125v17.25c0 .621.504 1.125 1.125 1.125h12.75c.621 0 1.125-.504 1.125-1.125V11.25a9 9 0 0 0-9-9Z" />
</svg>`,
ip_filters: `<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-6">
filters: `<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-6">
<path stroke-linecap="round" stroke-linejoin="round" d="M12 3c2.755 0 5.455.232 8.083.678.533.09.917.556.917 1.096v1.044a2.25 2.25 0 0 1-.659 1.591l-5.432 5.432a2.25 2.25 0 0 0-.659 1.591v2.927a2.25 2.25 0 0 1-1.244 2.013L9.75 21v-6.568a2.25 2.25 0 0 0-.659-1.591L3.659 7.409A2.25 2.25 0 0 1 3 5.818V4.774c0-.54.384-1.006.917-1.096A48.32 48.32 0 0 1 12 3Z" />
</svg>
`,
@@ -138,7 +138,7 @@
'/dashboard/': 'dashboard',
'/campaign/': 'campaigns_overview',
'/campaign-template/': 'campaign_templates',
'/ip-filter/': 'ip_filters',
'/filter/': 'filters',
'/webhook/': 'webhooks',
'/recipient/': 'recipients_overview',
'/recipient/group/': 'recipient_groups',
@@ -107,7 +107,7 @@
'/dashboard/': 'dashboard',
'/campaign/': 'campaigns_overview',
'/campaign-template/': 'campaign_templates',
'/ip-filter/': 'ip_filters',
'/filter/': 'filters',
'/webhook/': 'webhooks',
'/recipient/': 'recipients_overview',
'/recipient/group/': 'recipient_groups',
+2 -2
View File
@@ -78,8 +78,8 @@ export const route = {
route: '/api-sender/'
},
allowDeny: {
label: 'IP filters',
route: '/ip-filter/'
label: 'Filters',
route: '/filter/'
},
webhook: {
label: 'Webhooks',
@@ -1274,7 +1274,7 @@
{#if campaign.allowDeny?.length}
{#each campaign.allowDeny as allowDeny, i}
<a
href="/ip-filter/?edit={allowDeny.id}"
href="/filter/?edit={allowDeny.id}"
class="text-cta-blue dark:text-white hover:underline"
target="_blank"
>
@@ -43,6 +43,7 @@
id: null,
name: null,
cidrs: null,
ja4Fingerprints: null,
allowed: null
};
let allowDenyList = [];
@@ -63,7 +64,7 @@
};
$: {
modalText = getModalText('IP filter', modalMode);
modalText = getModalText('Filter', modalMode);
}
// hooks
@@ -94,7 +95,7 @@
allowDenyList = data.rows;
allowDenyListHasNextPage = data.hasNextPage;
} catch (e) {
addToast('Failed to get IP filters', 'Error');
addToast('Failed to get filters', 'Error');
console.error(e);
} finally {
isTableLoading = false;
@@ -113,8 +114,8 @@
throw res.error;
}
} catch (e) {
addToast('Failed to get IP filter', 'Error');
console.error('failed to get IP filter', e);
addToast('Failed to get filter', 'Error');
console.error('failed to get filter', e);
}
};
@@ -126,13 +127,22 @@
}
return res.data;
} catch (e) {
addToast('Failed to get IP filters', 'Error');
console.error('failed to get IP filters', e);
addToast('Failed to get filters', 'Error');
console.error('failed to get filters', e);
}
return [];
};
const onClickSubmit = async () => {
// validate that at least one of cidrs or ja4Fingerprints is provided
const hasCidrs = formValues.cidrs && formValues.cidrs.trim().length > 0;
const hasJA4 = formValues.ja4Fingerprints && formValues.ja4Fingerprints.trim().length > 0;
if (!hasCidrs && !hasJA4) {
formError = 'At least one of CIDRs or JA4 fingerprints must be provided';
return;
}
try {
isSubmitting = true;
if (modalMode === 'create' || modalMode === 'copy') {
@@ -149,16 +159,23 @@
const create = async () => {
formError = '';
formValues.cidrs = formValues.cidrs
.split('\n')
.map((line) => singleIPToCIDR(line))
.filter((line) => line.length > 0)
.join('\n');
// process cidrs only if provided
if (formValues.cidrs) {
formValues.cidrs = formValues.cidrs
.split('\n')
.map((line) => singleIPToCIDR(line))
.filter((line) => line.length > 0)
.join('\n');
} else {
formValues.cidrs = '';
}
try {
const res = await api.allowDeny.create({
name: formValues.name,
cidrs: formValues.cidrs,
ja4Fingerprints: formValues.ja4Fingerprints || '',
allowed: formValues.allowed,
companyID: contextCompanyID
});
@@ -166,39 +183,46 @@
formError = res.error;
return;
}
addToast('Created IP filter', 'Success');
addToast('Created filter', 'Success');
closeModal();
} catch (err) {
addToast('Failed to create IP filter', 'Error');
console.error('failed to create IP filter:', err);
addToast('Failed to create filter', 'Error');
console.error('failed to create filter:', err);
}
refreshAllowDenies();
};
const update = async () => {
formError = '';
formValues.cidrs = formValues.cidrs
.split('\n')
.map((line) => singleIPToCIDR(line))
.filter((line) => line.length > 0)
.join('\n');
// process cidrs only if provided
if (formValues.cidrs) {
formValues.cidrs = formValues.cidrs
.split('\n')
.map((line) => singleIPToCIDR(line))
.filter((line) => line.length > 0)
.join('\n');
} else {
formValues.cidrs = '';
}
try {
const res = await api.allowDeny.update({
id: formValues.id,
name: formValues.name,
cidrs: formValues.cidrs,
ja4Fingerprints: formValues.ja4Fingerprints || '',
companyID: formValues.companyID
});
if (res.success) {
addToast('Updated IP filter', 'Success');
addToast('Updated filter', 'Success');
closeModal();
} else {
formError = res.error;
}
} catch (e) {
addToast('Failed to update IP filter', 'Error');
console.error('failed to update IP filter', e);
addToast('Failed to update filter', 'Error');
console.error('failed to update filter', e);
}
refreshAllowDenies();
};
@@ -223,7 +247,7 @@
throw res.error;
})
.catch((e) => {
console.error('failed to delete IP filter:', e);
console.error('failed to delete filter:', e);
});
return action;
};
@@ -257,8 +281,8 @@
assignAllowDeny(allowDeny);
isModalVisible = true;
} catch (e) {
addToast('Failed to get IP filter', 'Error');
console.error('failed to get IP filter', e);
addToast('Failed to get filter', 'Error');
console.error('failed to get filter', e);
} finally {
hideIsLoading();
}
@@ -274,8 +298,8 @@
allowDeny.id = null;
isModalVisible = true;
} catch (e) {
addToast('Failed to get IP filter', 'Error');
console.error('failed to get IP filter', e);
addToast('Failed to get filter', 'Error');
console.error('failed to get filter', e);
} finally {
hideIsLoading();
}
@@ -286,6 +310,7 @@
id: allowDeny.id,
name: allowDeny.name,
cidrs: allowDeny.cidrs,
ja4Fingerprints: allowDeny.ja4Fingerprints || '',
allowed: allowDeny.allowed,
companyID: allowDeny.companyID
};
@@ -320,9 +345,9 @@
};
</script>
<HeadTitle title="IP filter" />
<HeadTitle title="Filter" />
<main>
<Headline>IP filters</Headline>
<Headline>Filters</Headline>
<BigButton on:click={openCreateModal}>New filter</BigButton>
<Table
columns={[
@@ -402,12 +427,21 @@
/>
{/if}
<TextareaField
required
minLength="1"
optional
bind:value={formValues.cidrs}
placeholder="8.8.8.8/16"
toolTipText="Newlines seperated CIDRs">CIDRs</TextareaField
toolTipText="Newlines seperated CIDRs (optional)">CIDRs</TextareaField
>
<TextareaField
optional
bind:value={formValues.ja4Fingerprints}
placeholder="t13d1715h2_8daaf6152771_02713d6af862"
toolTipText="Newlines separated JA4 fingerprints (optional)"
>JA4 Fingerprints</TextareaField
>
<p style="font-size: 0.875rem; color: #666; margin-top: 0.5rem;">
Note: At least one of CIDRs or JA4 Fingerprints must be provided.
</p>
</FormColumn>
</FormColumns>
<FormError message={formError} />