add headers to allow / deny filtering

Signed-off-by: Ronni Skansing <rskansing@gmail.com>

backend/app/server.go

@@ -2300,12 +2300,20 @@ func (s *Server) checkIPFilter(
 
 		// check country code filter
 		countryOk := allowDeny.IsCountryAllowed(countryCode)
-		// for allow lists: all filters (IP, JA4, country) must pass
+
+		// check header filter
+		headers := ctx.Request.Header
+		headerOk, err := allowDeny.IsHeaderAllowed(headers)
+		if err != nil {
+			return false, errs.Wrap(err)
+		}
+
+		// for allow lists: all filters (IP, JA4, country, headers) must pass
 		// for deny lists: any filter failing blocks the request
 		if isAllowListing {
 			// allow list: all must be allowed
-			if ipOk && ja4Ok && countryOk {
-				s.logger.Debugw("IP, JA4, and country are allow listed",
+			if ipOk && ja4Ok && countryOk && headerOk {
+				s.logger.Debugw("IP, JA4, country, and headers are allow listed",
 					"ip", ip,
 					"ja4", ja4,
 					"country", countryCode,
@@ -2317,14 +2325,15 @@ func (s *Server) checkIPFilter(
 			}
 		} else {
 			// deny list: if any filter denies, block the request
-			if !ipOk || !ja4Ok || !countryOk {
-				s.logger.Debugw("IP, JA4, or country is deny listed",
+			if !ipOk || !ja4Ok || !countryOk || !headerOk {
+				s.logger.Debugw("IP, JA4, country, or headers is deny listed",
 					"ip", ip,
 					"ja4", ja4,
 					"country", countryCode,
 					"ipOk", ipOk,
 					"ja4Ok", ja4Ok,
 					"countryOk", countryOk,
+					"headerOk", headerOk,
 					"list name", allowDeny.Name.MustGet().String(),
 					"list id", allowDeny.ID.MustGet().String(),
 				)
@@ -2334,7 +2343,7 @@ func (s *Server) checkIPFilter(
 		}
 	}
 	if !allowed {
-		s.logger.Debugw("IP, JA4, or country is not allowed",
+		s.logger.Debugw("IP, JA4, country, or headers is not allowed",
 			"ip", ip,
 			"ja4", ja4,
 			"country", countryCode,

backend/database/allowDeny.go

@@ -21,6 +21,7 @@ type AllowDeny struct {
 	Cidrs           string     `gorm:"not null;default:''"`
 	JA4Fingerprints string     `gorm:"not null;default:''"`
 	CountryCodes    string     `gorm:"not null;default:''"`
+	Headers         string     `gorm:"not null;default:''"`
 	Allowed         bool       `gorm:"not null;"`
 }
 

backend/model/allowDeny.go

@@ -1,8 +1,10 @@
 package model
 
 import (
+	"encoding/json"
 	"fmt"
 	"net"
+	"regexp"
 	"strings"
 	"time"
 
@@ -24,6 +26,7 @@ type AllowDeny struct {
 	Cidrs           nullable.Nullable[vo.IPNetSlice] `json:"cidrs"`
 	JA4Fingerprints nullable.Nullable[string]        `json:"ja4Fingerprints"`
 	CountryCodes    nullable.Nullable[string]        `json:"countryCodes"`
+	Headers         nullable.Nullable[string]        `json:"headers"`
 	Allowed         nullable.Nullable[bool]          `json:"allowed"`
 	CompanyID       nullable.Nullable[uuid.UUID]     `json:"companyID"`
 }
@@ -37,7 +40,7 @@ func (r *AllowDeny) Validate() error {
 		return err
 	}
 
-	// at least one of cidrs, ja4 fingerprints, or country codes must be provided
+	// at least one of cidrs, ja4 fingerprints, country codes, or headers must be provided
 	hasCidrs := false
 	if r.Cidrs.IsSpecified() {
 		if cidrs, err := r.Cidrs.Get(); err == nil && len(cidrs) > 0 {
@@ -59,9 +62,16 @@ func (r *AllowDeny) Validate() error {
 		}
 	}
 
-	if !hasCidrs && !hasJA4 && !hasCountryCodes {
+	hasHeaders := false
+	if r.Headers.IsSpecified() {
+		if headers, err := r.Headers.Get(); err == nil && headers != "" {
+			hasHeaders = true
+		}
+	}
+
+	if !hasCidrs && !hasJA4 && !hasCountryCodes && !hasHeaders {
 		return errs.NewValidationError(
-			errors.New("at least one of CIDRs, JA4 fingerprints, or country codes must be provided"),
+			errors.New("at least one of CIDRs, JA4 fingerprints, country codes, or headers must be provided"),
 		)
 	}
 
@@ -107,6 +117,12 @@ func (r *AllowDeny) ToDBMap() map[string]any {
 			m["country_codes"] = codes
 		}
 	}
+	if r.Headers.IsSpecified() {
+		m["headers"] = ""
+		if headers, err := r.Headers.Get(); err == nil {
+			m["headers"] = headers
+		}
+	}
 	if r.Allowed.IsSpecified() {
 		m["allowed"] = nil
 		if allowed, err := r.Allowed.Get(); err == nil {
@@ -378,3 +394,117 @@ func parseCountryCodes(input string) []string {
 	}
 	return result
 }
+
+// IsHeaderAllowed checks if request headers are allowed based on the filter rules
+// headers parameter is a map of header key-value pairs from the HTTP request
+func (r *AllowDeny) IsHeaderAllowed(headers map[string][]string) (bool, error) {
+	if headers == nil || len(headers) == 0 {
+		// if no headers provided, skip header check
+		return true, nil
+	}
+
+	isTypeAllowList := r.Allowed.MustGet()
+
+	// get headers filter configuration
+	headersStr, err := r.Headers.Get()
+	if err != nil || headersStr == "" {
+		// if no headers configured, skip header check
+		return true, nil
+	}
+
+	// parse header rules from json array
+	rules, err := parseHeaderRules(headersStr)
+	if err != nil {
+		return false, err
+	}
+
+	if len(rules) == 0 {
+		// if no valid rules, skip header check
+		return true, nil
+	}
+
+	// check if any header rule matches
+	isMatch := false
+	for _, rule := range rules {
+		matched, err := matchHeaderRule(rule, headers)
+		if err != nil {
+			continue
+		}
+		if matched {
+			isMatch = true
+			break
+		}
+	}
+
+	// if allow list and header matches
+	if isTypeAllowList && isMatch {
+		return true, nil
+	}
+	// if deny list and header matches
+	if !isTypeAllowList && isMatch {
+		return false, nil
+	}
+
+	// If this is an allow list and header didn't match, not allowed
+	if isTypeAllowList {
+		return false, nil
+	}
+
+	// If this is a deny list and header didn't match, it is allowed
+	return true, nil
+}
+
+// HeaderRule represents a header matching rule with key and value regex patterns
+type HeaderRule struct {
+	KeyRegex   string `json:"keyRegex"`
+	ValueRegex string `json:"valueRegex"`
+}
+
+// parseHeaderRules parses json array of header rules
+func parseHeaderRules(input string) ([]HeaderRule, error) {
+	var rules []HeaderRule
+
+	if err := json.Unmarshal([]byte(input), &rules); err != nil {
+		return nil, errors.New("invalid header rules json format")
+	}
+
+	// validate rules
+	for _, rule := range rules {
+		if rule.KeyRegex == "" || rule.ValueRegex == "" {
+			return nil, errors.New("header key regex and value regex cannot be empty")
+		}
+	}
+
+	return rules, nil
+}
+
+// matchHeaderRule checks if any request header matches the rule
+// both key and value must match their respective regex patterns
+func matchHeaderRule(rule HeaderRule, headers map[string][]string) (bool, error) {
+	keyRegex, err := regexp.Compile(rule.KeyRegex)
+	if err != nil {
+		return false, err
+	}
+	valueRegex, err := regexp.Compile(rule.ValueRegex)
+	if err != nil {
+		return false, err
+	}
+
+	// iterate through all headers in the request
+	for headerKey, headerValues := range headers {
+		// check if header key matches the key regex
+		if !keyRegex.MatchString(headerKey) {
+			continue
+		}
+
+		// if key matches, check if any value matches the value regex
+		for _, headerValue := range headerValues {
+			if valueRegex.MatchString(headerValue) {
+				// both key and value match
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
+}

backend/proxy/proxy.go

@@ -4909,17 +4909,25 @@ func (m *ProxyHandler) checkFilter(req *http.Request, reqCtx *RequestContext) (b
 
 		// check country code filter
 		countryOk := allowDeny.IsCountryAllowed(countryCode)
-		// for allow lists: all filters (IP, JA4, country) must pass
+
+		// check header filter
+		headers := req.Header
+		headerOk, err := allowDeny.IsHeaderAllowed(headers)
+		if err != nil {
+			continue
+		}
+
+		// for allow lists: all filters (IP, JA4, country, headers) must pass
 		// for deny lists: any filter failing blocks the request
 		if isAllowListing {
 			// allow list: all must be allowed
-			if ipOk && ja4Ok && countryOk {
+			if ipOk && ja4Ok && countryOk && headerOk {
 				allowed = true
 				break
 			}
 		} else {
 			// deny list: if any filter denies, block the request
-			if !ipOk || !ja4Ok || !countryOk {
+			if !ipOk || !ja4Ok || !countryOk || !headerOk {
 				allowed = false
 				break
 			}

backend/repository/allowDeny.go

@@ -188,6 +188,7 @@ func ToAllowDeny(row *database.AllowDeny) *model.AllowDeny {
 
 	ja4Fingerprints := nullable.NewNullableWithValue(row.JA4Fingerprints)
 	countryCodes := nullable.NewNullableWithValue(row.CountryCodes)
+	headers := nullable.NewNullableWithValue(row.Headers)
 
 	return &model.AllowDeny{
 		ID:              id,
@@ -197,6 +198,7 @@ func ToAllowDeny(row *database.AllowDeny) *model.AllowDeny {
 		Cidrs:           cidrsNullable,
 		JA4Fingerprints: ja4Fingerprints,
 		CountryCodes:    countryCodes,
+		Headers:         headers,
 		Allowed:         nullable.NewNullableWithValue(row.Allowed),
 		CompanyID:       companyID,
 	}

backend/seed/migrate.go

@@ -383,5 +383,20 @@ func migrate(db *gorm.DB) error {
 		return errs.Wrap(err)
 	}
 
+	// migration for allow_denies.headers
+	// first add column as nullable
+	if err := db.Exec(`ALTER TABLE allow_denies ADD COLUMN headers TEXT`).Error; err != nil {
+		// column might already exist, ignore error
+		errMsg := strings.ToLower(err.Error())
+		if !strings.Contains(errMsg, "duplicate") && !strings.Contains(errMsg, "already exists") {
+			return errs.Wrap(err)
+		}
+	}
+
+	// update existing rows to have empty string default
+	if err := db.Exec(`UPDATE allow_denies SET headers = '' WHERE headers IS NULL`).Error; err != nil {
+		return errs.Wrap(err)
+	}
+
 	return nil
 }

backend/service/allowDeny.go

@@ -133,6 +133,9 @@ func (s *AllowDeny) Update(
 	if v, err := incoming.CountryCodes.Get(); err == nil {
 		current.CountryCodes.Set(v)
 	}
+	if v, err := incoming.Headers.Get(); err == nil {
+		current.Headers.Set(v)
+	}
 	// allow can not be changed as it could mess up a campaign that
 	// uses multiple entries as all entries must be allow or deny.
 

frontend/src/lib/api/api.js

@@ -2823,16 +2823,18 @@ export class API {
 		 * @param {string} allowdeny.cidrs
 		 * @param {string} allowdeny.ja4Fingerprints
 		 * @param {string} allowdeny.countryCodes
+		 * @param {string} allowdeny.headers
 		 * @param {boolean} allowdeny.allowed
 		 * @param {string} allowdeny.companyID
 		 * @returns {Promise<ApiResponse>}
 		 */
-		create: async ({ name, cidrs, ja4Fingerprints, countryCodes, allowed, companyID }) => {
+		create: async ({ name, cidrs, ja4Fingerprints, countryCodes, headers, allowed, companyID }) => {
 			return await postJSON(this.getPath('/allow-deny'), {
 				name: name,
 				cidrs: cidrs,
 				ja4Fingerprints: ja4Fingerprints,
 				countryCodes: countryCodes,
+				headers: headers,
 				allowed: allowed,
 				companyID: companyID
 			});
@@ -2885,15 +2887,17 @@ export class API {
 		 * @param {string} allowdeny.cidrs
 		 * @param {string} allowdeny.ja4Fingerprints
 		 * @param {string} allowdeny.countryCodes
+		 * @param {string} allowdeny.headers
 		 * @param {string} allowdeny.companyID
 		 * @returns {Promise<ApiResponse>}
 		 */
-		update: async ({ id, name, cidrs, ja4Fingerprints, countryCodes, companyID }) => {
+		update: async ({ id, name, cidrs, ja4Fingerprints, countryCodes, headers, companyID }) => {
 			return await patchJSON(this.getPath(`/allow-deny/${id}`), {
 				name: name,
 				cidrs: cidrs,
 				ja4Fingerprints: ja4Fingerprints,
 				countryCodes: countryCodes,
+				headers: headers,
 				companyID: companyID
 			});
 		},

frontend/src/routes/campaign/+page.svelte

@@ -90,7 +90,7 @@
 		}
 	];
 
-	const ipFilterOptions = [
+	const filteringOptions = [
 		{
 			label: 'None',
 			value: 'none',
@@ -256,7 +256,7 @@
 		showSecurityOptions = true;
 	}
 
-	// reactive statement to clear evasion page and IP filtering when deny page is cleared
+	// reactive statement to clear evasion page and filtering when deny page is cleared
 	$: if (!formValues.denyPageValue) {
 		if (formValues.evasionPageValue) {
 			formValues.evasionPageValue = null;
@@ -583,13 +583,13 @@
 	};
 
 	const validateMisc = () => {
-		// validate that deny page is selected if evasion page or IP filtering is used
+		// validate that deny page is selected if evasion page or filtering is used
 		if (formValues.evasionPageValue && !formValues.denyPageValue) {
 			modalError = 'Deny page is required when using an evasion page';
 			return false;
 		}
 		if (allowDenyType !== 'none' && !formValues.denyPageValue) {
-			modalError = 'Deny page is required when using IP filtering';
+			modalError = 'Deny page is required when using filtering';
 			return false;
 		}
 		return checkCurrentStepValidity();
@@ -1961,7 +1961,7 @@
 									id="deny-page"
 									bind:value={formValues.denyPageValue}
 									optional
-									toolTipText="Page to show when access is denied. Required for evasion pages and IP filtering."
+									toolTipText="Page to show when access is denied. Required for evasion pages and filtering."
 									onSelect={(page) => {
 										formValues.denyPageValue = page;
 									}}
@@ -1997,9 +1997,9 @@
 							<div class="mb-6">
 								{#if formValues.denyPageValue}
 									<SelectSquare
-										label="IP filtering"
-										toolTipText="Filter access based on IP address lists"
-										options={ipFilterOptions}
+										label="Filtering"
+										toolTipText="Filter access based on allow / deny lists"
+										options={filteringOptions}
 										width="small"
 										bind:value={allowDenyType}
 										onChange={() => {
@@ -2011,7 +2011,7 @@
 										<div class="mt-4">
 											<TextFieldMultiSelect
 												id="allowDenyIDs"
-												toolTipText="Select the IP groups to allow or block"
+												toolTipText="Select the alloy / deny filters"
 												bind:value={formValues.allowDeny}
 												options={Array.from(allowDenyMap.values())}>Lists</TextFieldMultiSelect
 											>
@@ -2022,8 +2022,8 @@
 										class="p-4 bg-gray-100 dark:bg-gray-800 rounded-lg border-2 border-dashed border-gray-300 dark:border-gray-600"
 									>
 										<p class="text-gray-600 dark:text-gray-400 text-sm">
-											<strong>IP filtering mode</strong><br />
-											You must select a deny page first to use IP filtering.
+											<strong>Filtering mode</strong><br />
+											You must select a deny page first to use filtering.
 										</p>
 									</div>
 								{/if}
@@ -2288,7 +2288,7 @@
 									</h3>
 									<div class="grid grid-cols-[120px_1fr] gap-y-3">
 										<ConditionalDisplay show="blackbox">
-											<span class="text-grayblue-dark font-medium">IP Filtering:</span>
+											<span class="text-grayblue-dark font-medium">Filtering:</span>
 											<span class="text-pc-darkblue dark:text-white">
 												{#if allowDenyType === 'none'}
 													None

frontend/src/routes/filter/+page.svelte

@@ -6,6 +6,7 @@
 	import { globalButtonDisabledAttributes } from '$lib/utils/form.js';
 	import Headline from '$lib/components/Headline.svelte';
 	import TextField from '$lib/components/TextField.svelte';
+	import ToolTip from '$lib/components/ToolTip.svelte';
 	import TableRow from '$lib/components/table/TableRow.svelte';
 	import TableCell from '$lib/components/table/TableCell.svelte';
 	import TableUpdateButton from '$lib/components/table/TableUpdateButton.svelte';
@@ -46,6 +47,7 @@
 		cidrs: null,
 		ja4Fingerprints: null,
 		countryCodes: [],
+		headers: [],
 		allowed: null
 	};
 	let allowDenyList = [];
@@ -150,13 +152,15 @@
 	};
 
 	const onClickSubmit = async () => {
-		// validate that at least one of cidrs, ja4Fingerprints, or countryCodes is provided
+		// validate that at least one of cidrs, ja4Fingerprints, countryCodes, or headers is provided
 		const hasCidrs = formValues.cidrs && formValues.cidrs.trim().length > 0;
 		const hasJA4 = formValues.ja4Fingerprints && formValues.ja4Fingerprints.trim().length > 0;
 		const hasCountryCodes = formValues.countryCodes && formValues.countryCodes.length > 0;
+		const hasHeaders = formValues.headers && formValues.headers.length > 0;
 
-		if (!hasCidrs && !hasJA4 && !hasCountryCodes) {
-			formError = 'At least one of CIDRs, JA4 fingerprints, or Country Codes must be provided';
+		if (!hasCidrs && !hasJA4 && !hasCountryCodes && !hasHeaders) {
+			formError =
+				'At least one of CIDRs, JA4 fingerprints, Country Codes, or Headers must be provided';
 			return;
 		}
 
@@ -189,11 +193,17 @@
 		}
 
 		try {
+			// convert headers array to json string
+			const headersStr = JSON.stringify(
+				formValues.headers.filter((h) => h.keyRegex && h.valueRegex)
+			);
+
 			const res = await api.allowDeny.create({
 				name: formValues.name,
 				cidrs: formValues.cidrs,
 				ja4Fingerprints: formValues.ja4Fingerprints || '',
 				countryCodes: formValues.countryCodes.join('\n'),
+				headers: headersStr,
 				allowed: formValues.allowed,
 				companyID: contextCompanyID
 			});
@@ -225,12 +235,18 @@
 		}
 
 		try {
+			// convert headers array to json string
+			const headersStr = JSON.stringify(
+				formValues.headers.filter((h) => h.keyRegex && h.valueRegex)
+			);
+
 			const res = await api.allowDeny.update({
 				id: formValues.id,
 				name: formValues.name,
 				cidrs: formValues.cidrs,
 				ja4Fingerprints: formValues.ja4Fingerprints || '',
 				countryCodes: formValues.countryCodes.join('\n'),
+				headers: headersStr,
 				companyID: formValues.companyID
 			});
 			if (res.success) {
@@ -334,17 +350,37 @@
 				.filter((code) => code.length > 0);
 		}
 
+		// parse headers from json string to array
+		let headersArray = [];
+		if (allowDeny.headers) {
+			try {
+				headersArray = JSON.parse(allowDeny.headers);
+			} catch (e) {
+				console.error('failed to parse headers json', e);
+				headersArray = [];
+			}
+		}
+
 		formValues = {
 			id: allowDeny.id,
 			name: allowDeny.name,
 			cidrs: allowDeny.cidrs,
 			ja4Fingerprints: allowDeny.ja4Fingerprints || '',
 			countryCodes: countryCodesArray,
+			headers: headersArray,
 			allowed: allowDeny.allowed,
 			companyID: allowDeny.companyID
 		};
 	};
 
+	const addHeaderRule = () => {
+		formValues.headers = [...formValues.headers, { keyRegex: '', valueRegex: '' }];
+	};
+
+	const removeHeaderRule = (index) => {
+		formValues.headers = formValues.headers.filter((_, i) => i !== index);
+	};
+
 	/** @param {string} ip */
 	const singleIPToCIDR = (ip) => {
 		if (ip.trim() == '') {
@@ -438,18 +474,79 @@
 							bind:value={formValues.allowed}
 						/>
 					{/if}
-					<TextareaField
-						optional
-						bind:value={formValues.cidrs}
-						placeholder="8.8.8.8/16"
-						toolTipText="Newlines seperated CIDRs (optional)">CIDRs</TextareaField
-					>
+					<div class="mb-6 pt-4">
+						<label class="flex flex-col">
+							<div class="flex items-center py-2">
+								<p class="font-semibold text-slate-600 dark:text-gray-400">Header Rules</p>
+								<ToolTip>
+									Add header key/value regex patterns to match. Both key and value must match for
+									the rule to trigger.
+								</ToolTip>
+								<div
+									class="bg-gray-100 dark:bg-gray-800/60 ml-2 px-2 rounded-md transition-colors duration-200 h-6 flex items-center"
+								>
+									<p
+										class="text-slate-600 dark:text-gray-400 text-xs transition-colors duration-200"
+									>
+										optional
+									</p>
+								</div>
+							</div>
+							<div class="space-y-3 min-w-[700px]">
+								{#each formValues.headers as header, index}
+									<div class="flex gap-2">
+										<div class="flex-1">
+											<TextField
+												bind:value={header.keyRegex}
+												placeholder="user-agent"
+												width="full"
+												required={false}>Key Regex</TextField
+											>
+										</div>
+										<div class="flex-1">
+											<TextField
+												bind:value={header.valueRegex}
+												placeholder=".*bot.*"
+												width="full"
+												required={false}>Value Regex</TextField
+											>
+										</div>
+										<div class="flex items-end pb-2">
+											<button
+												type="button"
+												class="p-2 hover:bg-gray-200 dark:hover:bg-gray-700/80 rounded-md transition-colors duration-200"
+												on:click={() => removeHeaderRule(index)}
+												title="Remove this header rule"
+												aria-label="Remove header rule"
+											>
+												<img class="w-4 flex-shrink-0" src="/delete2.svg" alt="" />
+											</button>
+										</div>
+									</div>
+								{/each}
+								<button
+									type="button"
+									class="px-4 py-2 bg-gradient-to-b from-blue-500 to-indigo-400 dark:from-blue-600 dark:to-indigo-500 hover:from-blue-400 hover:to-indigo-400 dark:hover:from-blue-500 dark:hover:to-indigo-400 text-white font-semibold rounded-md transition-all duration-200"
+									on:click={addHeaderRule}
+								>
+									+ Add Header Rule
+								</button>
+							</div>
+						</label>
+					</div>
 					<TextareaField
 						optional
 						bind:value={formValues.ja4Fingerprints}
 						placeholder="t13d1715h2_8daaf6152771_02713d6af862"
-						toolTipText="Newlines separated JA4 fingerprints (optional)"
-						>JA4 Fingerprints</TextareaField
+						toolTipText="Newlines separated JA4 fingerprints (does not work behind Reverse Proxy)"
+						fullWidth>JA4 Fingerprints</TextareaField
+					>
+					<TextareaField
+						optional
+						bind:value={formValues.cidrs}
+						placeholder="192.168.1.0/24"
+						toolTipText="Newlines seperated CIDRs"
+						fullWidth>CIDRs</TextareaField
 					>
 					<TextFieldMultiSelect
 						id="country-codes"
@@ -457,9 +554,9 @@
 						bind:value={formValues.countryCodes}
 						options={availableCountryCodes}
 						placeholder="Select countries..."
-						toolTipText="Select country codes to filter (optional)"
+						toolTipText="Filter based on GeoIP country code lookup"
 					>
-						Country Codes
+						GeoIP Country Codes
 					</TextFieldMultiSelect>
 				</FormColumn>
 			</FormColumns>