From d5192aa792742778121e0dc60fee05072f87af11 Mon Sep 17 00:00:00 2001 From: Ronni Skansing Date: Mon, 22 Jun 2026 23:14:02 +0200 Subject: [PATCH] fix dub recipients migration script Signed-off-by: Ronni Skansing --- backend/main.go | 15 ++ backend/seed/README.md | 68 +++++++++ backend/seed/recipient_lowercase.go | 206 ++++++++++++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 backend/seed/README.md create mode 100644 backend/seed/recipient_lowercase.go diff --git a/backend/main.go b/backend/main.go index c9b818e..ebe6fef 100644 --- a/backend/main.go +++ b/backend/main.go @@ -59,6 +59,8 @@ var ( flagRecovery = flag.Bool("recover", false, "Used for interactive recovery of an account") flagConfigOnly = flag.Bool("config-only", false, "Run interactive installer and save config without installing") flagDebug = flag.Bool("debug", false, "Force debug logging on db and app logger, ignores db settings on startup") + flagMigrate = flag.String("migrate", "", "Run a named one-off data migration then exit (available: lowercase-recipient-emails)") + flagDryRun = flag.Bool("dry-run", false, "With --migrate, report what would change without writing any changes") ) func main() { @@ -201,6 +203,19 @@ func main() { if err != nil { logger.Fatalw("Failed to run migrations and seeding", "error", err) } + // run a one-off opt in data migration and exit, the schema is guaranteed + // by the seeding above + if *flagMigrate != "" { + switch *flagMigrate { + case "lowercase-recipient-emails": + if _, err := seed.LowercaseRecipientEmails(db, logger, *flagDryRun); err != nil { + logger.Fatalw("migration failed", "migration", *flagMigrate, "error", err) + } + default: + logger.Fatalf("unknown migration: %s", *flagMigrate) + } + return + } // setup logging again so it is according to the database err = setLogLevels(db, atomicLogger, *flagDebug) if err != nil { diff --git a/backend/seed/README.md b/backend/seed/README.md new file mode 100644 index 0000000..82cec40 --- /dev/null +++ b/backend/seed/README.md @@ -0,0 +1,68 @@ +# Description +The `seed` folder holds database seeding and migrations. + +- `migrate.go` runs the automatic, idempotent data migrations on every startup as part of `InitialInstallAndSeed`. These run unattended and must stay safe to repeat. +- The `*_dev.go` files seed development data. +- `recipient_lowercase.go` is an opt in, manual one off migration, described below. + +# Opt in migrations + +Some migrations are not safe to run unattended, so they are not part of startup. They are run explicitly from the CLI and exit once finished: + +``` +./phishingclub --migrate +``` + +Available names: + +- `lowercase-recipient-emails` + +## lowercase-recipient-emails + +### Background + +Before 1.39.0 recipient emails were stored exactly as entered and were not normalized to lowercase. A recipient email is the recipient identity, so this allowed two recipients that differ only by casing, for example `Foo@example.com` and `foo@example.com`, to exist as separate rows. They are the same person stored twice, a duplicate recipient. + +From 1.39.0 new recipients are stored lowercased and all lookups are case insensitive, so this duplicate can no longer be created. That fix applies to new data only and does not touch rows already in the database. + +This migration repairs data created before 1.39.0 by normalizing existing recipients to lowercase and merging any case variant duplicates into one. It is only needed if such duplicates exist. On an install that has no duplicate recipients it makes no changes, and running it is harmless. + +### What it does + +For every set of recipients that share the same lowercased email it: + +1. Picks the survivor. The row already stored lowercased is kept. If none is, the oldest row is kept and its email is lowercased. +2. Merges the others into the survivor. Every reference is repointed to the survivor across campaign membership, group membership, events and device codes. A membership that would become a duplicate, for example the same person already in a campaign or group, is collapsed to one. +3. Deletes the merged recipients. + +A recipient that is simply stored with mixed casing and has no duplicate is lowercased in place. + +The whole run is a single transaction, so it either completes fully or changes nothing. It is idempotent, so running it again does nothing once the data is clean. + +### Dry run first + +Add `--dry-run` to see what would change without writing anything: + +``` +./phishingclub --migrate lowercase-recipient-emails --dry-run +``` + +The dry run performs the full merge inside a transaction and then rolls it back, so the logged summary is exactly what a real run would do. Use it to confirm whether you have duplicates and how many recipients would be merged before running it for real. + +### Run only when no campaigns are active + +Run this only when no campaign is sending or collecting results. The merge is a real merge: when two case variants of the same person are in the same campaign, only one membership survives, and the removed membership row id is what appears in already sent tracking links and pixels, so those links stop resolving. Running it against a quiet system avoids losing in flight tracking. + +### What it logs + +On completion it logs a summary: + +- `groupsProcessed` lowercased emails that needed work +- `duplicateGroups` of those, how many had more than one recipient +- `recipientsMerged` recipients removed by merging +- `emailsLowercased` survivors whose stored email was changed to lowercase + +### Notes + +- The application database is SQLite. Matching uses SQL `LOWER()`, which folds ASCII `A` to `Z`. A non ASCII letter in the local part is the lone case it does not fold, consistent with the rest of the recipient code. +- It lowercases but does not trim whitespace. diff --git a/backend/seed/recipient_lowercase.go b/backend/seed/recipient_lowercase.go new file mode 100644 index 0000000..39843bd --- /dev/null +++ b/backend/seed/recipient_lowercase.go @@ -0,0 +1,206 @@ +package seed + +import ( + "github.com/go-errors/errors" + "github.com/phishingclub/phishingclub/database" + "github.com/phishingclub/phishingclub/errs" + "go.uber.org/zap" + "gorm.io/gorm" +) + +// RecipientLowercaseReport summarises a lowercase-recipient-emails run. +type RecipientLowercaseReport struct { + GroupsProcessed int + DuplicateGroups int + RecipientsMerged int + EmailsLowercased int +} + +// LowercaseRecipientEmails normalises every recipient email to lowercase and +// merges recipients that are the same address apart from casing into one +// canonical lowercased recipient. for each group the already lowercased row is +// kept, otherwise the oldest row is kept and lowercased. every reference to a +// merged recipient (campaign membership, group membership, events, device +// codes) is repointed to the survivor, and any membership that would become a +// duplicate is collapsed to one. the whole run is one transaction and is safe +// to run more than once. +// +// when dryRun is true the work is performed in a transaction that is rolled +// back, so the report reflects exactly what would change without writing any +// changes. +func LowercaseRecipientEmails(db *gorm.DB, logger *zap.SugaredLogger, dryRun bool) (*RecipientLowercaseReport, error) { + report := &RecipientLowercaseReport{} + tx := db.Begin() + if tx.Error != nil { + return nil, errs.Wrap(tx.Error) + } + if err := lowercaseRecipientEmailsInTx(tx, logger, report); err != nil { + tx.Rollback() + return nil, errs.Wrap(errors.Errorf("lowercase recipient emails migration failed: %w", err)) + } + if dryRun { + tx.Rollback() + logger.Infow("lowercase-recipient-emails dry run, no changes written", + "groupsProcessed", report.GroupsProcessed, + "duplicateGroups", report.DuplicateGroups, + "recipientsToMerge", report.RecipientsMerged, + "emailsToLowercase", report.EmailsLowercased, + ) + return report, nil + } + if err := tx.Commit().Error; err != nil { + return nil, errs.Wrap(err) + } + logger.Infow("lowercase-recipient-emails migration completed", + "groupsProcessed", report.GroupsProcessed, + "duplicateGroups", report.DuplicateGroups, + "recipientsMerged", report.RecipientsMerged, + "emailsLowercased", report.EmailsLowercased, + ) + return report, nil +} + +// lowercaseRecipientEmailsInTx performs the normalisation and merge work on the +// given transaction, filling in the report as it goes. +func lowercaseRecipientEmailsInTx(tx *gorm.DB, logger *zap.SugaredLogger, report *RecipientLowercaseReport) error { + // find every lowercased email that needs work, either because more than + // one recipient shares it or because the single recipient is not stored + // lowercased + var keys []struct { + LowerEmail string `gorm:"column:lower_email"` + } + err := tx.Raw( + "SELECT LOWER(email) AS lower_email FROM " + database.RECIPIENT_TABLE + + " WHERE email IS NOT NULL AND email != ''" + + " GROUP BY LOWER(email)" + + " HAVING COUNT(*) > 1 OR SUM(CASE WHEN email = LOWER(email) THEN 1 ELSE 0 END) < COUNT(*)", + ).Scan(&keys).Error + if err != nil { + return errs.Wrap(err) + } + + for _, k := range keys { + report.GroupsProcessed++ + // load the group, ordered so the survivor is first: an already + // lowercased row wins, then the oldest row + var rows []struct { + ID string `gorm:"column:id"` + Email string `gorm:"column:email"` + } + err := tx.Raw( + "SELECT id, email FROM "+database.RECIPIENT_TABLE+ + " WHERE email IS NOT NULL AND email != '' AND LOWER(email) = ?"+ + " ORDER BY CASE WHEN email = LOWER(email) THEN 0 ELSE 1 END ASC, created_at ASC, id ASC", + k.LowerEmail, + ).Scan(&rows).Error + if err != nil { + return errs.Wrap(err) + } + if len(rows) == 0 { + continue + } + survivor := rows[0] + if len(rows) > 1 { + report.DuplicateGroups++ + } + for _, nonSurvivor := range rows[1:] { + if err := mergeRecipient(tx, nonSurvivor.ID, survivor.ID); err != nil { + return errs.Wrap(err) + } + report.RecipientsMerged++ + } + if survivor.Email != k.LowerEmail { + if err := tx.Exec( + "UPDATE "+database.RECIPIENT_TABLE+" SET email = ? WHERE id = ?", + k.LowerEmail, survivor.ID, + ).Error; err != nil { + return errs.Wrap(err) + } + report.EmailsLowercased++ + } + logger.Debugw("processed recipient email group", + "email", k.LowerEmail, + "survivor", survivor.ID, + "merged", len(rows)-1, + ) + } + return nil +} + +// mergeRecipient repoints every reference from fromID to toID, collapsing any +// membership that would otherwise duplicate, then deletes the fromID recipient. +// references are repointed before the delete so no foreign key is left dangling. +func mergeRecipient(tx *gorm.DB, fromID string, toID string) error { + // campaign membership is unique on (campaign_id, recipient_id), so first + // drop the rows that the survivor already has for the same campaign + if err := tx.Exec( + "DELETE FROM "+database.CAMPAIGN_RECIPIENT_TABLE_NAME+ + " WHERE recipient_id = ? AND campaign_id IN"+ + " (SELECT campaign_id FROM "+database.CAMPAIGN_RECIPIENT_TABLE_NAME+ + " WHERE recipient_id = ?)", + fromID, toID, + ).Error; err != nil { + return err + } + if err := tx.Exec( + "UPDATE "+database.CAMPAIGN_RECIPIENT_TABLE_NAME+ + " SET recipient_id = ? WHERE recipient_id = ?", + toID, fromID, + ).Error; err != nil { + return err + } + + // group membership is unique on (recipient_id, recipient_group_id) + if err := tx.Exec( + "DELETE FROM "+database.RECIPIENT_GROUP_RECIPIENT_TABLE+ + " WHERE recipient_id = ? AND recipient_group_id IN"+ + " (SELECT recipient_group_id FROM "+database.RECIPIENT_GROUP_RECIPIENT_TABLE+ + " WHERE recipient_id = ?)", + fromID, toID, + ).Error; err != nil { + return err + } + if err := tx.Exec( + "UPDATE "+database.RECIPIENT_GROUP_RECIPIENT_TABLE+ + " SET recipient_id = ? WHERE recipient_id = ?", + toID, fromID, + ).Error; err != nil { + return err + } + + // device codes are unique on (campaign_id, recipient_id) + if err := tx.Exec( + "DELETE FROM "+database.MICROSOFT_DEVICE_CODE_TABLE+ + " WHERE recipient_id = ? AND campaign_id IN"+ + " (SELECT campaign_id FROM "+database.MICROSOFT_DEVICE_CODE_TABLE+ + " WHERE recipient_id = ?)", + fromID, toID, + ).Error; err != nil { + return err + } + if err := tx.Exec( + "UPDATE "+database.MICROSOFT_DEVICE_CODE_TABLE+ + " SET recipient_id = ? WHERE recipient_id = ?", + toID, fromID, + ).Error; err != nil { + return err + } + + // events have no uniqueness on the recipient, so keep them all and repoint + if err := tx.Exec( + "UPDATE "+database.CAMPAIGN_EVENT_TABLE+ + " SET recipient_id = ? WHERE recipient_id = ?", + toID, fromID, + ).Error; err != nil { + return err + } + + // the merged recipient now has no references and can be removed + if err := tx.Exec( + "DELETE FROM "+database.RECIPIENT_TABLE+" WHERE id = ?", + fromID, + ).Error; err != nil { + return err + } + return nil +}