add anonymize data feature

Signed-off-by: Ronni Skansing <rskansing@gmail.com>

backend/app/administration.go

@@ -73,14 +73,14 @@ const (
 	ROUTE_V1_SESSION_ID   = "/api/v1/session/:id"
 	ROUTE_V1_SESSION_PING = "/api/v1/session/ping"
 	// company
-	ROUTE_V1_COMPANY                  = "/api/v1/company"
-	ROUTE_V1_COMPANY_ID               = "/api/v1/company/:id"
-	ROUTE_V1_COMPANY_ID_EXPORT        = "/api/v1/company/:id/export"
-	ROUTE_V1_COMPANY_ID_EXPORT_SHARED = "/api/v1/company/shared/export"
-	ROUTE_V1_COMPANY_SCIM             = "/api/v1/company/scim/:companyID"
-	ROUTE_V1_COMPANY_SCIM_TOKEN       = "/api/v1/company/scim/:companyID/token"
-	ROUTE_V1_COMPANY_SCIM_PRUNE       = "/api/v1/company/scim/:companyID/prune"
-	ROUTE_V1_COMPANY_SCIM_RESTORE     = "/api/v1/company/scim/:companyID/restore"
+	ROUTE_V1_COMPANY                   = "/api/v1/company"
+	ROUTE_V1_COMPANY_ID                = "/api/v1/company/:id"
+	ROUTE_V1_COMPANY_ID_EXPORT         = "/api/v1/company/:id/export"
+	ROUTE_V1_COMPANY_ID_EXPORT_SHARED  = "/api/v1/company/shared/export"
+	ROUTE_V1_COMPANY_SCIM              = "/api/v1/company/scim/:companyID"
+	ROUTE_V1_COMPANY_SCIM_TOKEN        = "/api/v1/company/scim/:companyID/token"
+	ROUTE_V1_COMPANY_SCIM_PRUNE        = "/api/v1/company/scim/:companyID/prune"
+	ROUTE_V1_COMPANY_SCIM_RESTORE      = "/api/v1/company/scim/:companyID/restore"
 	ROUTE_V1_COMPANY_REPORT_CONFIG     = "/api/v1/company/report-config/:companyID"
 	ROUTE_V1_COMPANY_REPORT_CONFIG_LOG = "/api/v1/company/report-config/:companyID/log"
 	ROUTE_V1_REPORT_CONFIG_GLOBAL      = "/api/v1/report-config"
@@ -165,6 +165,7 @@ const (
 	ROUTE_V1_CAMPAIGN_EXPORT_EVENTS      = "/api/v1/campaign/:id/export/events"
 	ROUTE_V1_CAMPAIGN_EXPORT_SUBMISSIONS = "/api/v1/campaign/:id/export/submissions"
 	ROUTE_V1_CAMPAIGN_ANONYMIZE          = "/api/v1/campaign/:id/anonymize"
+	ROUTE_V1_CAMPAIGN_ANONYMIZE_DATA     = "/api/v1/campaign/:id/anonymize-data"
 	ROUTE_V1_CAMPAIGN_ID                 = "/api/v1/campaign/:id"
 	ROUTE_V1_CAMPAIGN_NAME               = "/api/v1/campaign/name/:name"
 	ROUTE_V1_CAMPAIGN_RECIPIENTS         = "/api/v1/campaign/:id/recipients"
@@ -515,6 +516,7 @@ func setupRoutes(
 		GET(ROUTE_V1_CAMPAIGN_EXPORT_SUBMISSIONS, middleware.SessionHandler, controllers.Campaign.ExportSubmissionsAsCSV).
 		POST(ROUTE_V1_CAMPAIGN_UPLOAD_REPORTED, middleware.SessionHandler, controllers.Campaign.UploadReportedCSV).
 		POST(ROUTE_V1_CAMPAIGN_ANONYMIZE, middleware.SessionHandler, controllers.Campaign.AnonymizeByID).
+		POST(ROUTE_V1_CAMPAIGN_ANONYMIZE_DATA, middleware.SessionHandler, controllers.Campaign.AnonymizeDataByID).
 		DELETE(ROUTE_V1_CAMPAIGN_DEVICE_CODES, middleware.SessionHandler, controllers.Campaign.DeleteDeviceCodesByCampaignID).
 		DELETE(ROUTE_V1_CAMPAIGN_ID, middleware.SessionHandler, controllers.Campaign.DeleteByID).
 		// campaign PDF report — ExtendedTimeout required for headless browser rendering

backend/controller/campaign.go

@@ -29,16 +29,17 @@ import (
 // to the database schema
 // this is tied to a slice in the repository package
 var allowedCampaignColumns = map[string]string{
-	"created_at":    repository.TableColumn(database.CAMPAIGN_TABLE, "created_at"),
-	"updated_at":    repository.TableColumn(database.CAMPAIGN_TABLE, "updated_at"),
-	"closed_at":     repository.TableColumn(database.CAMPAIGN_TABLE, "closed_at"),
-	"close_at":      repository.TableColumn(database.CAMPAIGN_TABLE, "close_at"),
-	"anonymized_at": repository.TableColumn(database.CAMPAIGN_TABLE, "anonymized_at"),
-	"is_test":       repository.TableColumn(database.CAMPAIGN_TABLE, "is_test"),
-	"send_start_at": repository.TableColumn(database.CAMPAIGN_TABLE, "send_start_at"),
-	"send_end_at":   repository.TableColumn(database.CAMPAIGN_TABLE, "send_end_at"),
-	"template":      repository.TableColumn(database.CAMPAIGN_TEMPLATE_TABLE, "name"),
-	"name":          repository.TableColumn(database.CAMPAIGN_TABLE, "name"),
+	"created_at":         repository.TableColumn(database.CAMPAIGN_TABLE, "created_at"),
+	"updated_at":         repository.TableColumn(database.CAMPAIGN_TABLE, "updated_at"),
+	"closed_at":          repository.TableColumn(database.CAMPAIGN_TABLE, "closed_at"),
+	"close_at":           repository.TableColumn(database.CAMPAIGN_TABLE, "close_at"),
+	"anonymized_at":      repository.TableColumn(database.CAMPAIGN_TABLE, "anonymized_at"),
+	"data_anonymized_at": repository.TableColumn(database.CAMPAIGN_TABLE, "data_anonymized_at"),
+	"is_test":            repository.TableColumn(database.CAMPAIGN_TABLE, "is_test"),
+	"send_start_at":      repository.TableColumn(database.CAMPAIGN_TABLE, "send_start_at"),
+	"send_end_at":        repository.TableColumn(database.CAMPAIGN_TABLE, "send_end_at"),
+	"template":           repository.TableColumn(database.CAMPAIGN_TEMPLATE_TABLE, "name"),
+	"name":               repository.TableColumn(database.CAMPAIGN_TABLE, "name"),
 }
 
 // campaignEventColumns is a map between the frontend and the backend
@@ -1001,6 +1002,26 @@ func (c *Campaign) AnonymizeByID(g *gin.Context) {
 	c.Response.OK(g, gin.H{})
 }
 
+// AnonymizeDataByID anonymizes the submitted data of a campaign by its id
+func (c *Campaign) AnonymizeDataByID(g *gin.Context) {
+	// handle session
+	session, _, ok := c.handleSession(g)
+	if !ok {
+		return
+	}
+	// parse request
+	id, ok := c.handleParseIDParam(g)
+	if !ok {
+		return
+	}
+	// anonymize submitted data
+	err := c.CampaignService.AnonymizeDataByID(g, session, id)
+	if ok := c.handleErrors(g, err); !ok {
+		return
+	}
+	c.Response.OK(g, gin.H{})
+}
+
 // GetCampaignStats gets campaign statistics by campaign ID
 func (c *Campaign) GetCampaignStats(g *gin.Context) {
 	// handle session
@@ -1228,4 +1249,3 @@ func (c *Campaign) UploadReportedCSV(g *gin.Context) {
 		"message":   "CSV processed successfully",
 	})
 }
-

backend/database/campaign.go

@@ -21,10 +21,15 @@ type Campaign struct {
 	ClosedAt     *time.Time `gorm:"index;"`
 	AnonymizeAt  *time.Time `gorm:"index;"`
 	AnonymizedAt *time.Time `gorm:"index;"`
-	SortField    string     `gorm:";"`
-	SortOrder    string     `gorm:";"` // 'asc,desc,random'
-	SendStartAt  *time.Time `gorm:"index;"`
-	SendEndAt    *time.Time `gorm:"index;"`
+	// DataAnonymizeAt schedules anonymization of submitted data, user agent, ip
+	// and browser metadata while keeping the recipient relation intact.
+	DataAnonymizeAt *time.Time `gorm:"index;"`
+	// DataAnonymizedAt is set when submitted data has been anonymized.
+	DataAnonymizedAt *time.Time `gorm:"index;"`
+	SortField        string     `gorm:";"`
+	SortOrder        string     `gorm:";"` // 'asc,desc,random'
+	SendStartAt      *time.Time `gorm:"index;"`
+	SendEndAt        *time.Time `gorm:"index;"`
 	// ScheduleAt is set when the campaign uses late-scheduling.
 	// the task runner will call schedule() when now >= ScheduleAt.
 	// null means the campaign was scheduled immediately at creation.

backend/errs/all.go

@@ -33,9 +33,10 @@ var (
 	ErrSessionCookieNotFound = goerrors.New("session cookie not found")
 
 	// campaign
-	ErrCampaignAlreadySetToClose = goerrors.New("campaign already set to closed")
-	ErrCampaignAlreadyClosed     = goerrors.New("campaign already closed")
-	ErrCampaignAlreadyAnonymized = goerrors.New("campaign already anonymized")
+	ErrCampaignAlreadySetToClose     = goerrors.New("campaign already set to closed")
+	ErrCampaignAlreadyClosed         = goerrors.New("campaign already closed")
+	ErrCampaignAlreadyAnonymized     = goerrors.New("campaign already anonymized")
+	ErrCampaignDataAlreadyAnonymized = goerrors.New("campaign data already anonymized")
 
 	// validation err
 	ErrValidationFailed = goerrors.New("validation error")

backend/model/campaign.go

@@ -24,6 +24,8 @@ type Campaign struct {
 	ClosedAt            nullable.Nullable[time.Time]                 `json:"closedAt"`
 	AnonymizeAt         nullable.Nullable[time.Time]                 `json:"anonymizeAt"`
 	AnonymizedAt        nullable.Nullable[time.Time]                 `json:"anonymizedAt"`
+	DataAnonymizeAt     nullable.Nullable[time.Time]                 `json:"dataAnonymizeAt"`
+	DataAnonymizedAt    nullable.Nullable[time.Time]                 `json:"dataAnonymizedAt"`
 	SortField           nullable.Nullable[vo.CampaignSortField]      `json:"sortField"`
 	SortOrder           nullable.Nullable[vo.CampaignSendingOrder]   `json:"sortOrder"`
 	SendStartAt         nullable.Nullable[time.Time]                 `json:"sendStartAt"`
@@ -227,6 +229,24 @@ func (c *Campaign) Validate() error {
 			}
 		}
 	}
+	if c.DataAnonymizeAt.IsSpecified() && !c.DataAnonymizeAt.IsNull() {
+		dataAnonymizeAt := c.DataAnonymizeAt.MustGet()
+		if v, err := c.CloseAt.Get(); err == nil {
+			if dataAnonymizeAt.Before(v) {
+				return validate.WrapErrorWithField(errors.New("anonymize submitted data at must be after close date"), "DataAnonymizeAt")
+			}
+		}
+		if v, err := c.SendEndAt.Get(); err == nil {
+			if dataAnonymizeAt.Before(v) {
+				return validate.WrapErrorWithField(errors.New("anonymize submitted data at must be after end date"), "DataAnonymizeAt")
+			}
+		}
+		if v, err := c.SendStartAt.Get(); err == nil {
+			if dataAnonymizeAt.Before(v) {
+				return validate.WrapErrorWithField(errors.New("anonymize submitted data at must be after start date"), "DataAnonymizeAt")
+			}
+		}
+	}
 	// validate jitter values if specified
 	// negative values are allowed for asymmetric jitter (e.g., -10 to 20 means send 10 min early to 20 min late)
 	jitterMinSpecified := c.JitterMin.IsSpecified() && !c.JitterMin.IsNull()
@@ -437,6 +457,12 @@ func (c *Campaign) ToDBMap() map[string]any {
 			m["anonymize_at"] = utils.RFC3339UTC(v)
 		}
 	}
+	if c.DataAnonymizeAt.IsSpecified() {
+		m["data_anonymize_at"] = nil
+		if v, err := c.DataAnonymizeAt.Get(); err == nil {
+			m["data_anonymize_at"] = utils.RFC3339UTC(v)
+		}
+	}
 	if c.SaveSubmittedData.IsSpecified() {
 		m["save_submitted_data"] = false
 		if v, err := c.SaveSubmittedData.Get(); err == nil {

backend/repository/campaign.go

@@ -25,6 +25,8 @@ var allowedCampaignColumns = []string{
 	TableColumn(database.CAMPAIGN_TABLE, "closed_at"),
 	TableColumn(database.CAMPAIGN_TABLE, "anonymize_at"),
 	TableColumn(database.CAMPAIGN_TABLE, "anonymized_at"),
+	TableColumn(database.CAMPAIGN_TABLE, "data_anonymize_at"),
+	TableColumn(database.CAMPAIGN_TABLE, "data_anonymized_at"),
 	TableColumn(database.CAMPAIGN_TABLE, "send_start_at"),
 	TableColumn(database.CAMPAIGN_TABLE, "send_end_at"),
 	TableColumn(database.CAMPAIGN_TABLE, "notable_event_id"),
@@ -1366,6 +1368,43 @@ func (r *Campaign) GetReadyToAnonymize(
 	return result, nil
 }
 
+// GetReadyToDataAnonymize gets all campaigns that are ready to have their
+// submitted data anonymized while keeping the recipient relation intact.
+// Fully anonymized campaigns are excluded as their data is already scrubbed.
+func (r *Campaign) GetReadyToDataAnonymize(
+	ctx context.Context,
+	options *CampaignOption,
+) (*model.Result[model.Campaign], error) {
+	result := model.NewEmptyResult[model.Campaign]()
+	db := r.load(r.DB, options)
+	db, err := useQuery(db, database.CAMPAIGN_TABLE, options.QueryArgs)
+	if err != nil {
+		return result, errs.Wrap(err)
+	}
+	var dbCampaigns []database.Campaign
+	res := db.
+		Where("data_anonymize_at <= ? AND data_anonymized_at IS NULL AND anonymized_at IS NULL", utils.NowRFC3339UTC()).
+		Find(&dbCampaigns)
+	if res.Error != nil {
+		return result, res.Error
+	}
+
+	hasNextPage, err := useHasNextPage(db, database.CAMPAIGN_TABLE, options.QueryArgs)
+	if err != nil {
+		return result, errs.Wrap(err)
+	}
+	result.HasNextPage = hasNextPage
+
+	for _, dbCampaign := range dbCampaigns {
+		campaign, err := ToCampaign(&dbCampaign)
+		if err != nil {
+			return nil, errs.Wrap(err)
+		}
+		result.Rows = append(result.Rows, campaign)
+	}
+	return result, nil
+}
+
 // GetReadyToLateSchedule returns campaigns where schedule_at <= now and
 // the campaign has not yet been scheduled (notable event is still pending_schedule).
 func (r *Campaign) GetReadyToLateSchedule(
@@ -1413,7 +1452,12 @@ func (r *Campaign) SaveEvent(
 		"ip_address":  campaignEvent.IP.String(),
 		"user_agent":  campaignEvent.UserAgent.String(),
 		"data":        campaignEvent.Data.String(),
-		"metadata":    func() string { if campaignEvent.Metadata == nil { return "" }; return campaignEvent.Metadata.String() }(),
+		"metadata": func() string {
+			if campaignEvent.Metadata == nil {
+				return ""
+			}
+			return campaignEvent.Metadata.String()
+		}(),
 	}
 	if campaignEvent.RecipientID != nil {
 		row["recipient_id"] = campaignEvent.RecipientID.String()
@@ -1662,6 +1706,51 @@ func (r *Campaign) AddAnonymizedAt(
 	return nil
 }
 
+// AddDataAnonymizedAt adds a data anonymized at time to a campaign
+func (r *Campaign) AddDataAnonymizedAt(
+	ctx context.Context,
+	id *uuid.UUID,
+) error {
+	row := map[string]interface{}{
+		"data_anonymized_at": utils.NowRFC3339UTC(),
+	}
+	AddUpdatedAt(row)
+	res := r.DB.
+		Model(&database.Campaign{}).
+		Where("id = ?", id).
+		Updates(row)
+
+	if res.Error != nil {
+		return res.Error
+	}
+	return nil
+}
+
+// AnonymizeCampaignEventData anonymizes the submitted data, user agent, ip
+// address and browser metadata of all events in a campaign while keeping the
+// recipient relation intact
+func (r *Campaign) AnonymizeCampaignEventData(
+	ctx context.Context,
+	campaignID *uuid.UUID,
+) error {
+	row := map[string]any{
+		"user_agent": "anonymized",
+		"ip_address": nil,
+		"data":       "anonymized",
+		"metadata":   "",
+	}
+	AddUpdatedAt(row)
+	res := r.DB.
+		Model(&database.CampaignEvent{}).
+		Where("campaign_id = ?", campaignID).
+		Updates(row)
+
+	if res.Error != nil {
+		return res.Error
+	}
+	return nil
+}
+
 // AnonymizeCampaignEvent anonymizes a campaign event
 func (r *Campaign) AnonymizeCampaignEvent(
 	ctx context.Context,
@@ -1956,6 +2045,14 @@ func ToCampaign(row *database.Campaign) (*model.Campaign, error) {
 	if row.AnonymizedAt != nil {
 		anonymizedAt.Set(*row.AnonymizedAt)
 	}
+	dataAnonymizeAt := nullable.NewNullNullable[time.Time]()
+	if row.DataAnonymizeAt != nil {
+		dataAnonymizeAt.Set(*row.DataAnonymizeAt)
+	}
+	dataAnonymizedAt := nullable.NewNullNullable[time.Time]()
+	if row.DataAnonymizedAt != nil {
+		dataAnonymizedAt.Set(*row.DataAnonymizedAt)
+	}
 
 	var notableEventName string
 	var notableEventID nullable.Nullable[uuid.UUID]
@@ -1976,6 +2073,8 @@ func ToCampaign(row *database.Campaign) (*model.Campaign, error) {
 		ClosedAt:            closedAt,
 		AnonymizeAt:         anonymizeAt,
 		AnonymizedAt:        anonymizedAt,
+		DataAnonymizeAt:     dataAnonymizeAt,
+		DataAnonymizedAt:    dataAnonymizedAt,
 		SortField:           sortField,
 		SortOrder:           sortOrder,
 		SendStartAt:         sendStartAt,

backend/service/campaign.go

@@ -1540,12 +1540,18 @@ func (c *Campaign) UpdateByID(
 	if v, err := incoming.AnonymizeAt.Get(); err == nil {
 		current.AnonymizeAt.Set(v.Truncate(time.Minute))
 	}
+	if v, err := incoming.DataAnonymizeAt.Get(); err == nil {
+		current.DataAnonymizeAt.Set(v.Truncate(time.Minute))
+	}
 	if v, err := incoming.ClosedAt.Get(); err == nil {
 		current.ClosedAt.Set(v.Truncate(time.Minute))
 	}
 	if v, err := incoming.AnonymizedAt.Get(); err == nil {
 		current.AnonymizedAt.Set(v.Truncate(time.Minute))
 	}
+	if v, err := incoming.DataAnonymizedAt.Get(); err == nil {
+		current.DataAnonymizedAt.Set(v.Truncate(time.Minute))
+	}
 	if v, err := incoming.ScheduleAt.Get(); err == nil {
 		current.ScheduleAt.Set(v)
 	} else if incoming.ScheduleAt.IsSpecified() {
@@ -3077,6 +3083,56 @@ func (c *Campaign) HandleAnonymizeCampaigns(
 	return nil
 }
 
+// HandleDataAnonymizeCampaigns anonymizes the submitted data of campaigns that
+// are ready for data anonymization
+func (c *Campaign) HandleDataAnonymizeCampaigns(
+	ctx context.Context,
+	session *model.Session,
+) error {
+	ae := NewAuditEvent("Campaign.HandleDataAnonymizeCampaigns", session)
+	// check permissions
+	isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL)
+	if err != nil && !errors.Is(err, errs.ErrAuthorizationFailed) {
+		c.LogAuthError(err)
+		return errs.Wrap(err)
+	}
+	if !isAuthorized {
+		c.AuditLogNotAuthorized(ae)
+		return errs.ErrAuthorizationFailed
+	}
+	// get all campaigns that are ready to have their submitted data anonymized
+	campaigns, err := c.CampaignRepository.GetReadyToDataAnonymize(
+		ctx,
+		&repository.CampaignOption{},
+	)
+	if err != nil {
+		c.Logger.Errorw("failed to get ready to data anonymize campaigns", "error", err)
+		return errs.Wrap(err)
+	}
+	// anonymize the submitted data of the campaigns
+	affectedIds := []string{}
+	for _, campaign := range campaigns.Rows {
+		campaignID := campaign.ID.MustGet()
+		affectedIds = append(affectedIds, campaignID.String())
+		c.Logger.Debugw("anonymizing campaign data with id", "campaignID", campaignID)
+		err = c.AnonymizeDataByID(ctx, session, &campaignID)
+		if errors.Is(err, errs.ErrCampaignDataAlreadyAnonymized) {
+			c.Logger.Debugw("campaign data already anonymized", "error", err)
+			continue
+		}
+		if err != nil {
+			c.Logger.Errorw("failed to anonymize campaign data by id", "error", err)
+			continue
+		}
+		c.Logger.Debugw("anonymized campaign data with id", "campaignID", campaignID)
+	}
+	if len(affectedIds) > 0 {
+		ae.Details["dataAnonymizedCampaignIds"] = affectedIds
+		c.AuditLogAuthorized(ae)
+	}
+	return nil
+}
+
 // SchedulePendingCampaigns is called by the task runner. It finds all campaigns
 // whose schedule_at time has passed and triggers schedule() for each.
 func (c *Campaign) SchedulePendingCampaigns(
@@ -3993,6 +4049,64 @@ func (c *Campaign) AnonymizeByID(
 	return nil
 }
 
+// AnonymizeDataByID anonymizes the submitted data, user agent, ip address and
+// browser metadata of a campaign while keeping the recipient relation intact
+func (c *Campaign) AnonymizeDataByID(
+	ctx context.Context,
+	session *model.Session,
+	id *uuid.UUID,
+) error {
+	ae := NewAuditEvent("Campaign.AnonymizeDataByID", session)
+	ae.Details["id"] = id.String()
+	// check permissions
+	isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL)
+	if err != nil && !errors.Is(err, errs.ErrAuthorizationFailed) {
+		c.LogAuthError(err)
+		return errs.Wrap(err)
+	}
+	if !isAuthorized {
+		c.AuditLogNotAuthorized(ae)
+		return errs.ErrAuthorizationFailed
+	}
+	// get campaign to check it exists and its anonymization state
+	campaign, err := c.CampaignRepository.GetByID(
+		ctx,
+		id,
+		&repository.CampaignOption{},
+	)
+	if err != nil {
+		c.Logger.Errorw("failed to get campaign by id", "error", err)
+		return errs.Wrap(err)
+	}
+	// a fully anonymized campaign has already had its data scrubbed
+	if campaign.AnonymizedAt.IsSpecified() && !campaign.AnonymizedAt.IsNull() {
+		return errs.Wrap(errs.ErrCampaignAlreadyAnonymized)
+	}
+	// once the campaign is closed and its data is anonymized there is no new
+	// data to scrub, while it is open it can be re-run to scrub newer events
+	isClosed := campaign.ClosedAt.IsSpecified() && !campaign.ClosedAt.IsNull()
+	isDataAnonymized := campaign.DataAnonymizedAt.IsSpecified() && !campaign.DataAnonymizedAt.IsNull()
+	if isClosed && isDataAnonymized {
+		return errs.Wrap(errs.ErrCampaignDataAlreadyAnonymized)
+	}
+	// scrub the submitted data, user agent, ip and browser metadata of all events
+	// while keeping the recipient relation intact
+	err = c.CampaignRepository.AnonymizeCampaignEventData(ctx, id)
+	if err != nil {
+		c.Logger.Errorw("failed to anonymize campaign event data", "error", err)
+		return errs.Wrap(err)
+	}
+	// add a timestamp to the campaign to indicate when the data was anonymized
+	err = c.CampaignRepository.AddDataAnonymizedAt(ctx, id)
+	if err != nil {
+		c.Logger.Errorw("failed to add data anonymized at to campaign", "error", err)
+		return errs.Wrap(err)
+	}
+	c.AuditLogAuthorized(ae)
+
+	return nil
+}
+
 // SendEmailByCampaignRecipientID sends an email to a specific campaign recipient
 // Multiple sends to the same recipient are allowed to support retry scenarios and follow-ups.
 func (c *Campaign) SendEmailByCampaignRecipientID(

backend/task/runner.go

@@ -150,6 +150,13 @@ func (d *Runner) Process(
 			session,
 		)
 	})
+	// anonymize submitted data of campaigns that are ready for data anonymization
+	d.runTask("anonymize campaign data", func() error {
+		return d.CampaignService.HandleDataAnonymizeCampaigns(
+			ctx,
+			session,
+		)
+	})
 	// send the next batch of messages
 	d.runTask("send messages", func() error {
 		err := d.CampaignService.SendNextBatch(

docker/dnsmasq/startup.sh

@@ -4,8 +4,8 @@ BACKEND_IP=$(getent hosts backend | awk '{ print $1 }')
 
 # Update the dnsmasq configuration file
 echo "address=/.test/$BACKEND_IP" > /etc/dnsmasq.conf
-# labs can use *.training.test
-echo "address=/training.test/172.20.0.150" >> /etc/dnsmasq.conf
+# labs can use *.labs.test
+echo "address=/labs.test/172.20.0.150" >> /etc/dnsmasq.conf
 echo "BACKEND_IP=$BACKEND_IP"
 echo "Loaded configuration (/etc/dnsmasq.conf):"
 cat /etc/dnsmasq.conf

frontend/src/lib/api/api.js

@@ -473,6 +473,14 @@ export class API {
 			return postJSON(this.getPath(`/campaign/${campaignID}/anonymize`));
 		},
 
+		/**
+		 * @param {string} campaignID
+		 * @returns {Promise<ApiResponse>}
+		 */
+		anonymizeData: async (campaignID) => {
+			return postJSON(this.getPath(`/campaign/${campaignID}/anonymize-data`));
+		},
+
 		/**
 		 * @param {string} campaignID
 		 * @returns {Promise<ApiResponse>}
@@ -523,6 +531,7 @@ export class API {
 		 * @param {string} campaign.sendEndAt
 		 * @param {string} [campaign.closeAt]
 		 * @param {string} [campaign.anonymizeAt]
+		 * @param {string} [campaign.dataAnonymizeAt]
 		 * @param {string} [campaign.scheduleAt]
 		 * @param {string[]} campaign.recipientGroupIDs []uuid
 		 * @param {string[]} campaign.allowDenyIDs []uuid
@@ -551,6 +560,7 @@ export class API {
 			sendEndAt,
 			closeAt,
 			anonymizeAt,
+			dataAnonymizeAt,
 			scheduleAt,
 			recipientGroupIDs,
 			allowDenyIDs,
@@ -578,6 +588,7 @@ export class API {
 				sendEndAt,
 				closeAt,
 				anonymizeAt,
+				dataAnonymizeAt,
 				scheduleAt,
 				recipientGroupIDs,
 				allowDenyIDs,
@@ -607,6 +618,7 @@ export class API {
 		 * @param {string} campaign.sendEndAt
 		 * @param {string} [campaign.closeAt]
 		 * @param {string} [campaign.anonymizeAt]
+		 * @param {string} [campaign.dataAnonymizeAt]
 		 * @param {string} [campaign.scheduleAt]
 		 * @param {string} campaign.templateID uuid
 		 * @param {string[]} campaign.recipientGroupIDs []uuid
@@ -636,6 +648,7 @@ export class API {
 			sendEndAt,
 			closeAt,
 			anonymizeAt,
+			dataAnonymizeAt,
 			scheduleAt,
 			recipientGroupIDs,
 			allowDenyIDs,
@@ -662,6 +675,7 @@ export class API {
 				sendEndAt,
 				closeAt,
 				anonymizeAt,
+				dataAnonymizeAt,
 				scheduleAt,
 				recipientGroupIDs,
 				allowDenyIDs,

frontend/src/routes/campaign/+page.svelte

@@ -465,6 +465,7 @@
 		scheduledEndAt: null,
 		closeAt: null,
 		anonymizeAt: null,
+		dataAnonymizeAt: null,
 		template: null,
 		sortField: defaultSendField,
 		sortOrder: defaultSendOrder,
@@ -867,6 +868,9 @@
 			const anonymizeAtUTC = formValues.anonymizeAt
 				? new Date(formValues.anonymizeAt).toISOString()
 				: null;
+			const dataAnonymizeAtUTC = formValues.dataAnonymizeAt
+				? new Date(formValues.dataAnonymizeAt).toISOString()
+				: null;
 			const contraintStartTimeUTC = formValues.contraintStartTime
 				? localTimeToUTC(formValues.contraintStartTime)
 				: null;
@@ -887,6 +891,7 @@
 				sortOrder: sortOrder.byKey(formValues.sortOrder),
 				closeAt: closeAtUTC,
 				anonymizeAt: anonymizeAtUTC,
+				dataAnonymizeAt: dataAnonymizeAtUTC,
 				saveSubmittedData: formValues.saveSubmittedData,
 				saveBrowserMetadata: formValues.saveBrowserMetadata,
 				isAnonymous: formValues.isAnonymous,
@@ -942,6 +947,9 @@
 			const anonymizeAtUTC = formValues.anonymizeAt
 				? new Date(formValues.anonymizeAt).toISOString()
 				: null;
+			const dataAnonymizeAtUTC = formValues.dataAnonymizeAt
+				? new Date(formValues.dataAnonymizeAt).toISOString()
+				: null;
 			const contraintStartTimeUTC = formValues.contraintStartTime
 				? localTimeToUTC(formValues.contraintStartTime)
 				: null;
@@ -970,6 +978,7 @@
 				sendEndAt: sendEndAtUTC,
 				closeAt: closeAtUTC,
 				anonymizeAt: anonymizeAtUTC,
+				dataAnonymizeAt: dataAnonymizeAtUTC,
 				recipientGroupIDs: recipientGroupIDs,
 				allowDenyIDs: allowDenyIDs,
 				denyPageID: denyPageMap.byValueOrNull(formValues.denyPageValue),
@@ -1114,6 +1123,7 @@
 			scheduledEndAt: null,
 			closeAt: null,
 			anonymizeAt: null,
+			dataAnonymizeAt: null,
 			template: null,
 			sortField: defaultSendField,
 			sortOrder: defaultSendOrder,
@@ -1240,6 +1250,7 @@
 			contraintEndTime: copyMode ? null : utcTimeToLocal(campaign.constraintEndTime),
 			closeAt: copyMode ? null : campaign.closeAt,
 			anonymizeAt: copyMode ? null : campaign.anonymizeAt,
+			dataAnonymizeAt: copyMode ? null : campaign.dataAnonymizeAt,
 			saveSubmittedData: campaign.saveSubmittedData,
 			saveBrowserMetadata: campaign.saveBrowserMetadata ?? false,
 			isAnonymous: campaign.isAnonymous,
@@ -2132,7 +2143,19 @@
 												? new Date(formValues.sendEndAt)
 												: new Date()}
 										optional
-										toolTipText="When reached, the campaign will close and a"
+										toolTipText="When reached, the campaign closes and all data is anonymized, including the recipient relation."
+										>Anonymize All Data</DateTimeField
+									>
+
+									<DateTimeField
+										bind:value={formValues.dataAnonymizeAt}
+										min={formValues.closeAt
+											? new Date(formValues.closeAt)
+											: formValues.sendEndAt
+												? new Date(formValues.sendEndAt)
+												: new Date()}
+										optional
+										toolTipText="When reached, only the submitted data, user agent, ip and browser metadata are anonymized. The recipient relation is kept."
 										>Anonymize Data</DateTimeField
 									>
 								{/if}
@@ -2739,12 +2762,22 @@
 										{/if}
 
 										{#if formValues.anonymizeAt}
-											<span class="text-grayblue-dark font-medium">Anonymize at:</span>
+											<span class="text-grayblue-dark font-medium">Anonymize all data at:</span>
 											<span class="text-pc-darkblue dark:text-white">
 												<Datetime value={formValues.anonymizeAt} />
 												<RelativeTime value={formValues.anonymizeAt} />
 											</span>
 										{/if}
+
+										{#if formValues.dataAnonymizeAt}
+											<span class="text-grayblue-dark font-medium"
+												>Anonymize data at:</span
+											>
+											<span class="text-pc-darkblue dark:text-white">
+												<Datetime value={formValues.dataAnonymizeAt} />
+												<RelativeTime value={formValues.dataAnonymizeAt} />
+											</span>
+										{/if}
 									</div>
 								</div>
 							</div>

frontend/src/routes/campaign/[id]/+page.svelte

@@ -62,6 +62,8 @@
 		sendStartAt: null,
 		sendEndAt: null,
 		anonymizedAt: null,
+		dataAnonymizeAt: null,
+		dataAnonymizedAt: null,
 		closeAt: null,
 		closedAt: null,
 		template: null,
@@ -143,6 +145,7 @@
 	let isRecipientTableLoading = false;
 	let isCloseModalVisible = false;
 	let isAnonymizeModalVisible = false;
+	let isAnonymizeDataModalVisible = false;
 	let isSendMessageModalVisible = false;
 	let isSetAsSentModalVisible = false;
 	let isSessionSushiModalVisible = false;
@@ -301,6 +304,8 @@
 			campaign.sendEndAt = t.sendEndAt;
 			campaign.anonymizedAt = t.anonymizedAt;
 			campaign.anonymizeAt = t.anonymizeAt;
+			campaign.dataAnonymizedAt = t.dataAnonymizedAt;
+			campaign.dataAnonymizeAt = t.dataAnonymizeAt;
 			campaign.closeAt = t.closeAt;
 			campaign.closedAt = t.closedAt;
 			campaign.isTest = t.isTest;
@@ -731,6 +736,14 @@
 		isAnonymizeModalVisible = false;
 	};
 
+	const showAnonymizeDataModal = () => {
+		isAnonymizeDataModalVisible = true;
+	};
+
+	const closeAnonymizeDataModal = () => {
+		isAnonymizeDataModalVisible = false;
+	};
+
 	const closeSessionSushiModal = () => {
 		isSessionSushiModalVisible = false;
 		storedCookieData = '';
@@ -884,9 +897,10 @@
 			await setCampaign();
 			await getEvents();
 			await refreshCampaignRecipients();
-			// bug: have to clear ref and wait a tick or svelte does not re-render
+			// reset timeline and refetch all events (timeline only fetches new events incrementally)
 			timelineEventsMap = new Map();
 			timelineEvents = [];
+			lastPoll3399Nano = '';
 			await tick();
 			await refreshRecipientsTimes();
 			await refreshCampaignEventsSince();
@@ -901,6 +915,35 @@
 		return res;
 	};
 
+	const onConfirmAnonymizeData = async () => {
+		let res;
+		try {
+			showIsLoading();
+			res = await api.campaign.anonymizeData($page.params.id);
+			if (!res.success) {
+				throw res.error;
+			}
+			await setCampaign();
+			await getEvents();
+			await refreshCampaignRecipients();
+			// reset timeline and refetch all events (timeline only fetches new events incrementally)
+			timelineEventsMap = new Map();
+			timelineEvents = [];
+			lastPoll3399Nano = '';
+			await tick();
+			await refreshRecipientsTimes();
+			await refreshCampaignEventsSince();
+			closeAnonymizeDataModal();
+			addToast('Data anonymized', 'Success');
+		} catch (e) {
+			addToast('Failed to anonymize data', 'Error');
+			console.error('failed to anonymize data', e);
+		} finally {
+			hideIsLoading();
+		}
+		return res;
+	};
+
 	const onClickExportEvents = async () => {
 		try {
 			showIsLoading();
@@ -1817,6 +1860,20 @@
 							><Datetime value={campaign.anonymizedAt} /></span
 						>
 					</div>
+
+					<div class="flex justify-between">
+						<span class="text-gray-600 dark:text-gray-400">Anonymize Data At:</span>
+						<span class="text-pc-darkblue dark:text-white text-right"
+							><Datetime value={campaign.dataAnonymizeAt} /></span
+						>
+					</div>
+
+					<div class="flex justify-between">
+						<span class="text-gray-600 dark:text-gray-400">Data Anonymized:</span>
+						<span class="text-pc-darkblue dark:text-white text-right"
+							><Datetime value={campaign.dataAnonymizedAt} /></span
+						>
+					</div>
 				</div>
 
 				{#if campaign.constraintWeekDays}
@@ -1933,7 +1990,29 @@
 									disabled={!!campaign.anonymizedAt || isContextMismatch()}
 									on:click={showAnonymizeModal}
 								>
-									Anonymize
+									Anonymize All
+								</IconButton>
+							</div>
+							<div
+								title={isContextMismatch()
+									? campaign.companyID
+										? 'Switch to company view to perform this action'
+										: 'Switch to global view to perform this action'
+									: campaign.anonymizedAt
+										? 'Campaign is already anonymized'
+										: campaign.dataAnonymizedAt && campaign.closedAt
+											? 'Data is already anonymized'
+											: ''}
+							>
+								<IconButton
+									variant="red"
+									icon="anonymize"
+									disabled={!!campaign.anonymizedAt ||
+										(!!campaign.dataAnonymizedAt && !!campaign.closedAt) ||
+										isContextMismatch()}
+									on:click={showAnonymizeDataModal}
+								>
+									Anonymize Data
 								</IconButton>
 							</div>
 						</div>
@@ -2741,6 +2820,28 @@
 		</div>
 	</Alert>
 
+	<Alert
+		headline="anonymize data"
+		bind:visible={isAnonymizeDataModalVisible}
+		onConfirm={onConfirmAnonymizeData}
+		verification="confirm"
+	>
+		<div>
+			<ul
+				class="list-disc ml-8 mt-4 mb-4 text-gray-700 dark:text-gray-200 transition-colors duration-200"
+			>
+				<li class="list-tem">
+					Submitted data, user agent, ip and browser metadata will be anonymized
+				</li>
+				<li class="list-tem">The recipient relation is kept</li>
+				<li class="list-tem">Anonymization is permanent and not reversable</li>
+				{#if !campaign.closedAt}
+					<li class="list-tem">The campaign stays open and keeps tracking events</li>
+				{/if}
+			</ul>
+		</div>
+	</Alert>
+
 	<Alert
 		headline={`Send message`}
 		bind:visible={isSendMessageModalVisible}