From 252e566475278d62601bd00d0b8a351b282dfe65 Mon Sep 17 00:00:00 2001 From: Ronni Skansing Date: Tue, 30 Jun 2026 22:30:46 +0200 Subject: [PATCH] add anonymize data feature Signed-off-by: Ronni Skansing --- backend/app/administration.go | 18 +-- backend/controller/campaign.go | 42 +++++-- backend/database/campaign.go | 13 +- backend/errs/all.go | 7 +- backend/model/campaign.go | 26 ++++ backend/repository/campaign.go | 101 +++++++++++++++- backend/service/campaign.go | 114 ++++++++++++++++++ backend/task/runner.go | 7 ++ docker/dnsmasq/startup.sh | 4 +- frontend/src/lib/api/api.js | 14 +++ frontend/src/routes/campaign/+page.svelte | 37 +++++- .../src/routes/campaign/[id]/+page.svelte | 105 +++++++++++++++- 12 files changed, 455 insertions(+), 33 deletions(-) diff --git a/backend/app/administration.go b/backend/app/administration.go index c76d6ba..61732df 100644 --- a/backend/app/administration.go +++ b/backend/app/administration.go @@ -73,14 +73,14 @@ const ( ROUTE_V1_SESSION_ID = "/api/v1/session/:id" ROUTE_V1_SESSION_PING = "/api/v1/session/ping" // company - ROUTE_V1_COMPANY = "/api/v1/company" - ROUTE_V1_COMPANY_ID = "/api/v1/company/:id" - ROUTE_V1_COMPANY_ID_EXPORT = "/api/v1/company/:id/export" - ROUTE_V1_COMPANY_ID_EXPORT_SHARED = "/api/v1/company/shared/export" - ROUTE_V1_COMPANY_SCIM = "/api/v1/company/scim/:companyID" - ROUTE_V1_COMPANY_SCIM_TOKEN = "/api/v1/company/scim/:companyID/token" - ROUTE_V1_COMPANY_SCIM_PRUNE = "/api/v1/company/scim/:companyID/prune" - ROUTE_V1_COMPANY_SCIM_RESTORE = "/api/v1/company/scim/:companyID/restore" + ROUTE_V1_COMPANY = "/api/v1/company" + ROUTE_V1_COMPANY_ID = "/api/v1/company/:id" + ROUTE_V1_COMPANY_ID_EXPORT = "/api/v1/company/:id/export" + ROUTE_V1_COMPANY_ID_EXPORT_SHARED = "/api/v1/company/shared/export" + ROUTE_V1_COMPANY_SCIM = "/api/v1/company/scim/:companyID" + ROUTE_V1_COMPANY_SCIM_TOKEN = "/api/v1/company/scim/:companyID/token" + ROUTE_V1_COMPANY_SCIM_PRUNE = "/api/v1/company/scim/:companyID/prune" + ROUTE_V1_COMPANY_SCIM_RESTORE = "/api/v1/company/scim/:companyID/restore" ROUTE_V1_COMPANY_REPORT_CONFIG = "/api/v1/company/report-config/:companyID" ROUTE_V1_COMPANY_REPORT_CONFIG_LOG = "/api/v1/company/report-config/:companyID/log" ROUTE_V1_REPORT_CONFIG_GLOBAL = "/api/v1/report-config" @@ -165,6 +165,7 @@ const ( ROUTE_V1_CAMPAIGN_EXPORT_EVENTS = "/api/v1/campaign/:id/export/events" ROUTE_V1_CAMPAIGN_EXPORT_SUBMISSIONS = "/api/v1/campaign/:id/export/submissions" ROUTE_V1_CAMPAIGN_ANONYMIZE = "/api/v1/campaign/:id/anonymize" + ROUTE_V1_CAMPAIGN_ANONYMIZE_DATA = "/api/v1/campaign/:id/anonymize-data" ROUTE_V1_CAMPAIGN_ID = "/api/v1/campaign/:id" ROUTE_V1_CAMPAIGN_NAME = "/api/v1/campaign/name/:name" ROUTE_V1_CAMPAIGN_RECIPIENTS = "/api/v1/campaign/:id/recipients" @@ -515,6 +516,7 @@ func setupRoutes( GET(ROUTE_V1_CAMPAIGN_EXPORT_SUBMISSIONS, middleware.SessionHandler, controllers.Campaign.ExportSubmissionsAsCSV). POST(ROUTE_V1_CAMPAIGN_UPLOAD_REPORTED, middleware.SessionHandler, controllers.Campaign.UploadReportedCSV). POST(ROUTE_V1_CAMPAIGN_ANONYMIZE, middleware.SessionHandler, controllers.Campaign.AnonymizeByID). + POST(ROUTE_V1_CAMPAIGN_ANONYMIZE_DATA, middleware.SessionHandler, controllers.Campaign.AnonymizeDataByID). DELETE(ROUTE_V1_CAMPAIGN_DEVICE_CODES, middleware.SessionHandler, controllers.Campaign.DeleteDeviceCodesByCampaignID). DELETE(ROUTE_V1_CAMPAIGN_ID, middleware.SessionHandler, controllers.Campaign.DeleteByID). // campaign PDF report — ExtendedTimeout required for headless browser rendering diff --git a/backend/controller/campaign.go b/backend/controller/campaign.go index 8310e33..5f0060a 100644 --- a/backend/controller/campaign.go +++ b/backend/controller/campaign.go @@ -29,16 +29,17 @@ import ( // to the database schema // this is tied to a slice in the repository package var allowedCampaignColumns = map[string]string{ - "created_at": repository.TableColumn(database.CAMPAIGN_TABLE, "created_at"), - "updated_at": repository.TableColumn(database.CAMPAIGN_TABLE, "updated_at"), - "closed_at": repository.TableColumn(database.CAMPAIGN_TABLE, "closed_at"), - "close_at": repository.TableColumn(database.CAMPAIGN_TABLE, "close_at"), - "anonymized_at": repository.TableColumn(database.CAMPAIGN_TABLE, "anonymized_at"), - "is_test": repository.TableColumn(database.CAMPAIGN_TABLE, "is_test"), - "send_start_at": repository.TableColumn(database.CAMPAIGN_TABLE, "send_start_at"), - "send_end_at": repository.TableColumn(database.CAMPAIGN_TABLE, "send_end_at"), - "template": repository.TableColumn(database.CAMPAIGN_TEMPLATE_TABLE, "name"), - "name": repository.TableColumn(database.CAMPAIGN_TABLE, "name"), + "created_at": repository.TableColumn(database.CAMPAIGN_TABLE, "created_at"), + "updated_at": repository.TableColumn(database.CAMPAIGN_TABLE, "updated_at"), + "closed_at": repository.TableColumn(database.CAMPAIGN_TABLE, "closed_at"), + "close_at": repository.TableColumn(database.CAMPAIGN_TABLE, "close_at"), + "anonymized_at": repository.TableColumn(database.CAMPAIGN_TABLE, "anonymized_at"), + "data_anonymized_at": repository.TableColumn(database.CAMPAIGN_TABLE, "data_anonymized_at"), + "is_test": repository.TableColumn(database.CAMPAIGN_TABLE, "is_test"), + "send_start_at": repository.TableColumn(database.CAMPAIGN_TABLE, "send_start_at"), + "send_end_at": repository.TableColumn(database.CAMPAIGN_TABLE, "send_end_at"), + "template": repository.TableColumn(database.CAMPAIGN_TEMPLATE_TABLE, "name"), + "name": repository.TableColumn(database.CAMPAIGN_TABLE, "name"), } // campaignEventColumns is a map between the frontend and the backend @@ -1001,6 +1002,26 @@ func (c *Campaign) AnonymizeByID(g *gin.Context) { c.Response.OK(g, gin.H{}) } +// AnonymizeDataByID anonymizes the submitted data of a campaign by its id +func (c *Campaign) AnonymizeDataByID(g *gin.Context) { + // handle session + session, _, ok := c.handleSession(g) + if !ok { + return + } + // parse request + id, ok := c.handleParseIDParam(g) + if !ok { + return + } + // anonymize submitted data + err := c.CampaignService.AnonymizeDataByID(g, session, id) + if ok := c.handleErrors(g, err); !ok { + return + } + c.Response.OK(g, gin.H{}) +} + // GetCampaignStats gets campaign statistics by campaign ID func (c *Campaign) GetCampaignStats(g *gin.Context) { // handle session @@ -1228,4 +1249,3 @@ func (c *Campaign) UploadReportedCSV(g *gin.Context) { "message": "CSV processed successfully", }) } - diff --git a/backend/database/campaign.go b/backend/database/campaign.go index f99cf25..eb680af 100644 --- a/backend/database/campaign.go +++ b/backend/database/campaign.go @@ -21,10 +21,15 @@ type Campaign struct { ClosedAt *time.Time `gorm:"index;"` AnonymizeAt *time.Time `gorm:"index;"` AnonymizedAt *time.Time `gorm:"index;"` - SortField string `gorm:";"` - SortOrder string `gorm:";"` // 'asc,desc,random' - SendStartAt *time.Time `gorm:"index;"` - SendEndAt *time.Time `gorm:"index;"` + // DataAnonymizeAt schedules anonymization of submitted data, user agent, ip + // and browser metadata while keeping the recipient relation intact. + DataAnonymizeAt *time.Time `gorm:"index;"` + // DataAnonymizedAt is set when submitted data has been anonymized. + DataAnonymizedAt *time.Time `gorm:"index;"` + SortField string `gorm:";"` + SortOrder string `gorm:";"` // 'asc,desc,random' + SendStartAt *time.Time `gorm:"index;"` + SendEndAt *time.Time `gorm:"index;"` // ScheduleAt is set when the campaign uses late-scheduling. // the task runner will call schedule() when now >= ScheduleAt. // null means the campaign was scheduled immediately at creation. diff --git a/backend/errs/all.go b/backend/errs/all.go index 7fd1efa..56bb6f0 100644 --- a/backend/errs/all.go +++ b/backend/errs/all.go @@ -33,9 +33,10 @@ var ( ErrSessionCookieNotFound = goerrors.New("session cookie not found") // campaign - ErrCampaignAlreadySetToClose = goerrors.New("campaign already set to closed") - ErrCampaignAlreadyClosed = goerrors.New("campaign already closed") - ErrCampaignAlreadyAnonymized = goerrors.New("campaign already anonymized") + ErrCampaignAlreadySetToClose = goerrors.New("campaign already set to closed") + ErrCampaignAlreadyClosed = goerrors.New("campaign already closed") + ErrCampaignAlreadyAnonymized = goerrors.New("campaign already anonymized") + ErrCampaignDataAlreadyAnonymized = goerrors.New("campaign data already anonymized") // validation err ErrValidationFailed = goerrors.New("validation error") diff --git a/backend/model/campaign.go b/backend/model/campaign.go index 2ab4774..4caa25f 100644 --- a/backend/model/campaign.go +++ b/backend/model/campaign.go @@ -24,6 +24,8 @@ type Campaign struct { ClosedAt nullable.Nullable[time.Time] `json:"closedAt"` AnonymizeAt nullable.Nullable[time.Time] `json:"anonymizeAt"` AnonymizedAt nullable.Nullable[time.Time] `json:"anonymizedAt"` + DataAnonymizeAt nullable.Nullable[time.Time] `json:"dataAnonymizeAt"` + DataAnonymizedAt nullable.Nullable[time.Time] `json:"dataAnonymizedAt"` SortField nullable.Nullable[vo.CampaignSortField] `json:"sortField"` SortOrder nullable.Nullable[vo.CampaignSendingOrder] `json:"sortOrder"` SendStartAt nullable.Nullable[time.Time] `json:"sendStartAt"` @@ -227,6 +229,24 @@ func (c *Campaign) Validate() error { } } } + if c.DataAnonymizeAt.IsSpecified() && !c.DataAnonymizeAt.IsNull() { + dataAnonymizeAt := c.DataAnonymizeAt.MustGet() + if v, err := c.CloseAt.Get(); err == nil { + if dataAnonymizeAt.Before(v) { + return validate.WrapErrorWithField(errors.New("anonymize submitted data at must be after close date"), "DataAnonymizeAt") + } + } + if v, err := c.SendEndAt.Get(); err == nil { + if dataAnonymizeAt.Before(v) { + return validate.WrapErrorWithField(errors.New("anonymize submitted data at must be after end date"), "DataAnonymizeAt") + } + } + if v, err := c.SendStartAt.Get(); err == nil { + if dataAnonymizeAt.Before(v) { + return validate.WrapErrorWithField(errors.New("anonymize submitted data at must be after start date"), "DataAnonymizeAt") + } + } + } // validate jitter values if specified // negative values are allowed for asymmetric jitter (e.g., -10 to 20 means send 10 min early to 20 min late) jitterMinSpecified := c.JitterMin.IsSpecified() && !c.JitterMin.IsNull() @@ -437,6 +457,12 @@ func (c *Campaign) ToDBMap() map[string]any { m["anonymize_at"] = utils.RFC3339UTC(v) } } + if c.DataAnonymizeAt.IsSpecified() { + m["data_anonymize_at"] = nil + if v, err := c.DataAnonymizeAt.Get(); err == nil { + m["data_anonymize_at"] = utils.RFC3339UTC(v) + } + } if c.SaveSubmittedData.IsSpecified() { m["save_submitted_data"] = false if v, err := c.SaveSubmittedData.Get(); err == nil { diff --git a/backend/repository/campaign.go b/backend/repository/campaign.go index b2f3abf..5a23da1 100644 --- a/backend/repository/campaign.go +++ b/backend/repository/campaign.go @@ -25,6 +25,8 @@ var allowedCampaignColumns = []string{ TableColumn(database.CAMPAIGN_TABLE, "closed_at"), TableColumn(database.CAMPAIGN_TABLE, "anonymize_at"), TableColumn(database.CAMPAIGN_TABLE, "anonymized_at"), + TableColumn(database.CAMPAIGN_TABLE, "data_anonymize_at"), + TableColumn(database.CAMPAIGN_TABLE, "data_anonymized_at"), TableColumn(database.CAMPAIGN_TABLE, "send_start_at"), TableColumn(database.CAMPAIGN_TABLE, "send_end_at"), TableColumn(database.CAMPAIGN_TABLE, "notable_event_id"), @@ -1366,6 +1368,43 @@ func (r *Campaign) GetReadyToAnonymize( return result, nil } +// GetReadyToDataAnonymize gets all campaigns that are ready to have their +// submitted data anonymized while keeping the recipient relation intact. +// Fully anonymized campaigns are excluded as their data is already scrubbed. +func (r *Campaign) GetReadyToDataAnonymize( + ctx context.Context, + options *CampaignOption, +) (*model.Result[model.Campaign], error) { + result := model.NewEmptyResult[model.Campaign]() + db := r.load(r.DB, options) + db, err := useQuery(db, database.CAMPAIGN_TABLE, options.QueryArgs) + if err != nil { + return result, errs.Wrap(err) + } + var dbCampaigns []database.Campaign + res := db. + Where("data_anonymize_at <= ? AND data_anonymized_at IS NULL AND anonymized_at IS NULL", utils.NowRFC3339UTC()). + Find(&dbCampaigns) + if res.Error != nil { + return result, res.Error + } + + hasNextPage, err := useHasNextPage(db, database.CAMPAIGN_TABLE, options.QueryArgs) + if err != nil { + return result, errs.Wrap(err) + } + result.HasNextPage = hasNextPage + + for _, dbCampaign := range dbCampaigns { + campaign, err := ToCampaign(&dbCampaign) + if err != nil { + return nil, errs.Wrap(err) + } + result.Rows = append(result.Rows, campaign) + } + return result, nil +} + // GetReadyToLateSchedule returns campaigns where schedule_at <= now and // the campaign has not yet been scheduled (notable event is still pending_schedule). func (r *Campaign) GetReadyToLateSchedule( @@ -1413,7 +1452,12 @@ func (r *Campaign) SaveEvent( "ip_address": campaignEvent.IP.String(), "user_agent": campaignEvent.UserAgent.String(), "data": campaignEvent.Data.String(), - "metadata": func() string { if campaignEvent.Metadata == nil { return "" }; return campaignEvent.Metadata.String() }(), + "metadata": func() string { + if campaignEvent.Metadata == nil { + return "" + } + return campaignEvent.Metadata.String() + }(), } if campaignEvent.RecipientID != nil { row["recipient_id"] = campaignEvent.RecipientID.String() @@ -1662,6 +1706,51 @@ func (r *Campaign) AddAnonymizedAt( return nil } +// AddDataAnonymizedAt adds a data anonymized at time to a campaign +func (r *Campaign) AddDataAnonymizedAt( + ctx context.Context, + id *uuid.UUID, +) error { + row := map[string]interface{}{ + "data_anonymized_at": utils.NowRFC3339UTC(), + } + AddUpdatedAt(row) + res := r.DB. + Model(&database.Campaign{}). + Where("id = ?", id). + Updates(row) + + if res.Error != nil { + return res.Error + } + return nil +} + +// AnonymizeCampaignEventData anonymizes the submitted data, user agent, ip +// address and browser metadata of all events in a campaign while keeping the +// recipient relation intact +func (r *Campaign) AnonymizeCampaignEventData( + ctx context.Context, + campaignID *uuid.UUID, +) error { + row := map[string]any{ + "user_agent": "anonymized", + "ip_address": nil, + "data": "anonymized", + "metadata": "", + } + AddUpdatedAt(row) + res := r.DB. + Model(&database.CampaignEvent{}). + Where("campaign_id = ?", campaignID). + Updates(row) + + if res.Error != nil { + return res.Error + } + return nil +} + // AnonymizeCampaignEvent anonymizes a campaign event func (r *Campaign) AnonymizeCampaignEvent( ctx context.Context, @@ -1956,6 +2045,14 @@ func ToCampaign(row *database.Campaign) (*model.Campaign, error) { if row.AnonymizedAt != nil { anonymizedAt.Set(*row.AnonymizedAt) } + dataAnonymizeAt := nullable.NewNullNullable[time.Time]() + if row.DataAnonymizeAt != nil { + dataAnonymizeAt.Set(*row.DataAnonymizeAt) + } + dataAnonymizedAt := nullable.NewNullNullable[time.Time]() + if row.DataAnonymizedAt != nil { + dataAnonymizedAt.Set(*row.DataAnonymizedAt) + } var notableEventName string var notableEventID nullable.Nullable[uuid.UUID] @@ -1976,6 +2073,8 @@ func ToCampaign(row *database.Campaign) (*model.Campaign, error) { ClosedAt: closedAt, AnonymizeAt: anonymizeAt, AnonymizedAt: anonymizedAt, + DataAnonymizeAt: dataAnonymizeAt, + DataAnonymizedAt: dataAnonymizedAt, SortField: sortField, SortOrder: sortOrder, SendStartAt: sendStartAt, diff --git a/backend/service/campaign.go b/backend/service/campaign.go index 0cd6138..d5173c4 100644 --- a/backend/service/campaign.go +++ b/backend/service/campaign.go @@ -1540,12 +1540,18 @@ func (c *Campaign) UpdateByID( if v, err := incoming.AnonymizeAt.Get(); err == nil { current.AnonymizeAt.Set(v.Truncate(time.Minute)) } + if v, err := incoming.DataAnonymizeAt.Get(); err == nil { + current.DataAnonymizeAt.Set(v.Truncate(time.Minute)) + } if v, err := incoming.ClosedAt.Get(); err == nil { current.ClosedAt.Set(v.Truncate(time.Minute)) } if v, err := incoming.AnonymizedAt.Get(); err == nil { current.AnonymizedAt.Set(v.Truncate(time.Minute)) } + if v, err := incoming.DataAnonymizedAt.Get(); err == nil { + current.DataAnonymizedAt.Set(v.Truncate(time.Minute)) + } if v, err := incoming.ScheduleAt.Get(); err == nil { current.ScheduleAt.Set(v) } else if incoming.ScheduleAt.IsSpecified() { @@ -3077,6 +3083,56 @@ func (c *Campaign) HandleAnonymizeCampaigns( return nil } +// HandleDataAnonymizeCampaigns anonymizes the submitted data of campaigns that +// are ready for data anonymization +func (c *Campaign) HandleDataAnonymizeCampaigns( + ctx context.Context, + session *model.Session, +) error { + ae := NewAuditEvent("Campaign.HandleDataAnonymizeCampaigns", session) + // check permissions + isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL) + if err != nil && !errors.Is(err, errs.ErrAuthorizationFailed) { + c.LogAuthError(err) + return errs.Wrap(err) + } + if !isAuthorized { + c.AuditLogNotAuthorized(ae) + return errs.ErrAuthorizationFailed + } + // get all campaigns that are ready to have their submitted data anonymized + campaigns, err := c.CampaignRepository.GetReadyToDataAnonymize( + ctx, + &repository.CampaignOption{}, + ) + if err != nil { + c.Logger.Errorw("failed to get ready to data anonymize campaigns", "error", err) + return errs.Wrap(err) + } + // anonymize the submitted data of the campaigns + affectedIds := []string{} + for _, campaign := range campaigns.Rows { + campaignID := campaign.ID.MustGet() + affectedIds = append(affectedIds, campaignID.String()) + c.Logger.Debugw("anonymizing campaign data with id", "campaignID", campaignID) + err = c.AnonymizeDataByID(ctx, session, &campaignID) + if errors.Is(err, errs.ErrCampaignDataAlreadyAnonymized) { + c.Logger.Debugw("campaign data already anonymized", "error", err) + continue + } + if err != nil { + c.Logger.Errorw("failed to anonymize campaign data by id", "error", err) + continue + } + c.Logger.Debugw("anonymized campaign data with id", "campaignID", campaignID) + } + if len(affectedIds) > 0 { + ae.Details["dataAnonymizedCampaignIds"] = affectedIds + c.AuditLogAuthorized(ae) + } + return nil +} + // SchedulePendingCampaigns is called by the task runner. It finds all campaigns // whose schedule_at time has passed and triggers schedule() for each. func (c *Campaign) SchedulePendingCampaigns( @@ -3993,6 +4049,64 @@ func (c *Campaign) AnonymizeByID( return nil } +// AnonymizeDataByID anonymizes the submitted data, user agent, ip address and +// browser metadata of a campaign while keeping the recipient relation intact +func (c *Campaign) AnonymizeDataByID( + ctx context.Context, + session *model.Session, + id *uuid.UUID, +) error { + ae := NewAuditEvent("Campaign.AnonymizeDataByID", session) + ae.Details["id"] = id.String() + // check permissions + isAuthorized, err := IsAuthorized(session, data.PERMISSION_ALLOW_GLOBAL) + if err != nil && !errors.Is(err, errs.ErrAuthorizationFailed) { + c.LogAuthError(err) + return errs.Wrap(err) + } + if !isAuthorized { + c.AuditLogNotAuthorized(ae) + return errs.ErrAuthorizationFailed + } + // get campaign to check it exists and its anonymization state + campaign, err := c.CampaignRepository.GetByID( + ctx, + id, + &repository.CampaignOption{}, + ) + if err != nil { + c.Logger.Errorw("failed to get campaign by id", "error", err) + return errs.Wrap(err) + } + // a fully anonymized campaign has already had its data scrubbed + if campaign.AnonymizedAt.IsSpecified() && !campaign.AnonymizedAt.IsNull() { + return errs.Wrap(errs.ErrCampaignAlreadyAnonymized) + } + // once the campaign is closed and its data is anonymized there is no new + // data to scrub, while it is open it can be re-run to scrub newer events + isClosed := campaign.ClosedAt.IsSpecified() && !campaign.ClosedAt.IsNull() + isDataAnonymized := campaign.DataAnonymizedAt.IsSpecified() && !campaign.DataAnonymizedAt.IsNull() + if isClosed && isDataAnonymized { + return errs.Wrap(errs.ErrCampaignDataAlreadyAnonymized) + } + // scrub the submitted data, user agent, ip and browser metadata of all events + // while keeping the recipient relation intact + err = c.CampaignRepository.AnonymizeCampaignEventData(ctx, id) + if err != nil { + c.Logger.Errorw("failed to anonymize campaign event data", "error", err) + return errs.Wrap(err) + } + // add a timestamp to the campaign to indicate when the data was anonymized + err = c.CampaignRepository.AddDataAnonymizedAt(ctx, id) + if err != nil { + c.Logger.Errorw("failed to add data anonymized at to campaign", "error", err) + return errs.Wrap(err) + } + c.AuditLogAuthorized(ae) + + return nil +} + // SendEmailByCampaignRecipientID sends an email to a specific campaign recipient // Multiple sends to the same recipient are allowed to support retry scenarios and follow-ups. func (c *Campaign) SendEmailByCampaignRecipientID( diff --git a/backend/task/runner.go b/backend/task/runner.go index 62535b3..cf21373 100644 --- a/backend/task/runner.go +++ b/backend/task/runner.go @@ -150,6 +150,13 @@ func (d *Runner) Process( session, ) }) + // anonymize submitted data of campaigns that are ready for data anonymization + d.runTask("anonymize campaign data", func() error { + return d.CampaignService.HandleDataAnonymizeCampaigns( + ctx, + session, + ) + }) // send the next batch of messages d.runTask("send messages", func() error { err := d.CampaignService.SendNextBatch( diff --git a/docker/dnsmasq/startup.sh b/docker/dnsmasq/startup.sh index 0bdc91a..4fb7b0e 100755 --- a/docker/dnsmasq/startup.sh +++ b/docker/dnsmasq/startup.sh @@ -4,8 +4,8 @@ BACKEND_IP=$(getent hosts backend | awk '{ print $1 }') # Update the dnsmasq configuration file echo "address=/.test/$BACKEND_IP" > /etc/dnsmasq.conf -# labs can use *.training.test -echo "address=/training.test/172.20.0.150" >> /etc/dnsmasq.conf +# labs can use *.labs.test +echo "address=/labs.test/172.20.0.150" >> /etc/dnsmasq.conf echo "BACKEND_IP=$BACKEND_IP" echo "Loaded configuration (/etc/dnsmasq.conf):" cat /etc/dnsmasq.conf diff --git a/frontend/src/lib/api/api.js b/frontend/src/lib/api/api.js index 70d79e1..9a1fa5f 100644 --- a/frontend/src/lib/api/api.js +++ b/frontend/src/lib/api/api.js @@ -473,6 +473,14 @@ export class API { return postJSON(this.getPath(`/campaign/${campaignID}/anonymize`)); }, + /** + * @param {string} campaignID + * @returns {Promise} + */ + anonymizeData: async (campaignID) => { + return postJSON(this.getPath(`/campaign/${campaignID}/anonymize-data`)); + }, + /** * @param {string} campaignID * @returns {Promise} @@ -523,6 +531,7 @@ export class API { * @param {string} campaign.sendEndAt * @param {string} [campaign.closeAt] * @param {string} [campaign.anonymizeAt] + * @param {string} [campaign.dataAnonymizeAt] * @param {string} [campaign.scheduleAt] * @param {string[]} campaign.recipientGroupIDs []uuid * @param {string[]} campaign.allowDenyIDs []uuid @@ -551,6 +560,7 @@ export class API { sendEndAt, closeAt, anonymizeAt, + dataAnonymizeAt, scheduleAt, recipientGroupIDs, allowDenyIDs, @@ -578,6 +588,7 @@ export class API { sendEndAt, closeAt, anonymizeAt, + dataAnonymizeAt, scheduleAt, recipientGroupIDs, allowDenyIDs, @@ -607,6 +618,7 @@ export class API { * @param {string} campaign.sendEndAt * @param {string} [campaign.closeAt] * @param {string} [campaign.anonymizeAt] + * @param {string} [campaign.dataAnonymizeAt] * @param {string} [campaign.scheduleAt] * @param {string} campaign.templateID uuid * @param {string[]} campaign.recipientGroupIDs []uuid @@ -636,6 +648,7 @@ export class API { sendEndAt, closeAt, anonymizeAt, + dataAnonymizeAt, scheduleAt, recipientGroupIDs, allowDenyIDs, @@ -662,6 +675,7 @@ export class API { sendEndAt, closeAt, anonymizeAt, + dataAnonymizeAt, scheduleAt, recipientGroupIDs, allowDenyIDs, diff --git a/frontend/src/routes/campaign/+page.svelte b/frontend/src/routes/campaign/+page.svelte index 65d5b78..0e229ad 100644 --- a/frontend/src/routes/campaign/+page.svelte +++ b/frontend/src/routes/campaign/+page.svelte @@ -465,6 +465,7 @@ scheduledEndAt: null, closeAt: null, anonymizeAt: null, + dataAnonymizeAt: null, template: null, sortField: defaultSendField, sortOrder: defaultSendOrder, @@ -867,6 +868,9 @@ const anonymizeAtUTC = formValues.anonymizeAt ? new Date(formValues.anonymizeAt).toISOString() : null; + const dataAnonymizeAtUTC = formValues.dataAnonymizeAt + ? new Date(formValues.dataAnonymizeAt).toISOString() + : null; const contraintStartTimeUTC = formValues.contraintStartTime ? localTimeToUTC(formValues.contraintStartTime) : null; @@ -887,6 +891,7 @@ sortOrder: sortOrder.byKey(formValues.sortOrder), closeAt: closeAtUTC, anonymizeAt: anonymizeAtUTC, + dataAnonymizeAt: dataAnonymizeAtUTC, saveSubmittedData: formValues.saveSubmittedData, saveBrowserMetadata: formValues.saveBrowserMetadata, isAnonymous: formValues.isAnonymous, @@ -942,6 +947,9 @@ const anonymizeAtUTC = formValues.anonymizeAt ? new Date(formValues.anonymizeAt).toISOString() : null; + const dataAnonymizeAtUTC = formValues.dataAnonymizeAt + ? new Date(formValues.dataAnonymizeAt).toISOString() + : null; const contraintStartTimeUTC = formValues.contraintStartTime ? localTimeToUTC(formValues.contraintStartTime) : null; @@ -970,6 +978,7 @@ sendEndAt: sendEndAtUTC, closeAt: closeAtUTC, anonymizeAt: anonymizeAtUTC, + dataAnonymizeAt: dataAnonymizeAtUTC, recipientGroupIDs: recipientGroupIDs, allowDenyIDs: allowDenyIDs, denyPageID: denyPageMap.byValueOrNull(formValues.denyPageValue), @@ -1114,6 +1123,7 @@ scheduledEndAt: null, closeAt: null, anonymizeAt: null, + dataAnonymizeAt: null, template: null, sortField: defaultSendField, sortOrder: defaultSendOrder, @@ -1240,6 +1250,7 @@ contraintEndTime: copyMode ? null : utcTimeToLocal(campaign.constraintEndTime), closeAt: copyMode ? null : campaign.closeAt, anonymizeAt: copyMode ? null : campaign.anonymizeAt, + dataAnonymizeAt: copyMode ? null : campaign.dataAnonymizeAt, saveSubmittedData: campaign.saveSubmittedData, saveBrowserMetadata: campaign.saveBrowserMetadata ?? false, isAnonymous: campaign.isAnonymous, @@ -2132,7 +2143,19 @@ ? new Date(formValues.sendEndAt) : new Date()} optional - toolTipText="When reached, the campaign will close and a" + toolTipText="When reached, the campaign closes and all data is anonymized, including the recipient relation." + >Anonymize All Data + + Anonymize Data {/if} @@ -2739,12 +2762,22 @@ {/if} {#if formValues.anonymizeAt} - Anonymize at: + Anonymize all data at: {/if} + + {#if formValues.dataAnonymizeAt} + Anonymize data at: + + + + + {/if} diff --git a/frontend/src/routes/campaign/[id]/+page.svelte b/frontend/src/routes/campaign/[id]/+page.svelte index 0c8c350..2bd411b 100644 --- a/frontend/src/routes/campaign/[id]/+page.svelte +++ b/frontend/src/routes/campaign/[id]/+page.svelte @@ -62,6 +62,8 @@ sendStartAt: null, sendEndAt: null, anonymizedAt: null, + dataAnonymizeAt: null, + dataAnonymizedAt: null, closeAt: null, closedAt: null, template: null, @@ -143,6 +145,7 @@ let isRecipientTableLoading = false; let isCloseModalVisible = false; let isAnonymizeModalVisible = false; + let isAnonymizeDataModalVisible = false; let isSendMessageModalVisible = false; let isSetAsSentModalVisible = false; let isSessionSushiModalVisible = false; @@ -301,6 +304,8 @@ campaign.sendEndAt = t.sendEndAt; campaign.anonymizedAt = t.anonymizedAt; campaign.anonymizeAt = t.anonymizeAt; + campaign.dataAnonymizedAt = t.dataAnonymizedAt; + campaign.dataAnonymizeAt = t.dataAnonymizeAt; campaign.closeAt = t.closeAt; campaign.closedAt = t.closedAt; campaign.isTest = t.isTest; @@ -731,6 +736,14 @@ isAnonymizeModalVisible = false; }; + const showAnonymizeDataModal = () => { + isAnonymizeDataModalVisible = true; + }; + + const closeAnonymizeDataModal = () => { + isAnonymizeDataModalVisible = false; + }; + const closeSessionSushiModal = () => { isSessionSushiModalVisible = false; storedCookieData = ''; @@ -884,9 +897,10 @@ await setCampaign(); await getEvents(); await refreshCampaignRecipients(); - // bug: have to clear ref and wait a tick or svelte does not re-render + // reset timeline and refetch all events (timeline only fetches new events incrementally) timelineEventsMap = new Map(); timelineEvents = []; + lastPoll3399Nano = ''; await tick(); await refreshRecipientsTimes(); await refreshCampaignEventsSince(); @@ -901,6 +915,35 @@ return res; }; + const onConfirmAnonymizeData = async () => { + let res; + try { + showIsLoading(); + res = await api.campaign.anonymizeData($page.params.id); + if (!res.success) { + throw res.error; + } + await setCampaign(); + await getEvents(); + await refreshCampaignRecipients(); + // reset timeline and refetch all events (timeline only fetches new events incrementally) + timelineEventsMap = new Map(); + timelineEvents = []; + lastPoll3399Nano = ''; + await tick(); + await refreshRecipientsTimes(); + await refreshCampaignEventsSince(); + closeAnonymizeDataModal(); + addToast('Data anonymized', 'Success'); + } catch (e) { + addToast('Failed to anonymize data', 'Error'); + console.error('failed to anonymize data', e); + } finally { + hideIsLoading(); + } + return res; + }; + const onClickExportEvents = async () => { try { showIsLoading(); @@ -1817,6 +1860,20 @@ > + +
+ Anonymize Data At: + +
+ +
+ Data Anonymized: + +
{#if campaign.constraintWeekDays} @@ -1933,7 +1990,29 @@ disabled={!!campaign.anonymizedAt || isContextMismatch()} on:click={showAnonymizeModal} > - Anonymize + Anonymize All + + +
+ + Anonymize Data
@@ -2741,6 +2820,28 @@ + +
+
    +
  • + Submitted data, user agent, ip and browser metadata will be anonymized +
    • +
  • The recipient relation is kept
    • +
  • Anonymization is permanent and not reversable
    • + {#if !campaign.closedAt} +
  • The campaign stays open and keeps tracking events
    • + {/if} +
+
+ +