fix broken upload reported csv with column selection

Signed-off-by: Ronni Skansing <rskansing@gmail.com>

backend/controller/campaign.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/csv"
 	"io"
+	"strconv"
 	"strings"
 	"time"
 
@@ -1104,8 +1105,35 @@ func (c *Campaign) UploadReportedCSV(g *gin.Context) {
 
 	c.Logger.Debugw("processing CSV", "rows", len(records), "headers", records[0])
 
+	// get required column indices from query params
+	emailColStr := g.Query("emailColumn")
+	if emailColStr == "" {
+		c.Response.ValidationFailed(g, "emailColumn", errors.New("emailColumn query parameter is required"))
+		return
+	}
+
+	dateColStr := g.Query("dateColumn")
+	if dateColStr == "" {
+		c.Response.ValidationFailed(g, "dateColumn", errors.New("dateColumn query parameter is required"))
+		return
+	}
+
+	emailColIdx, err := strconv.Atoi(emailColStr)
+	if err != nil || emailColIdx < 0 {
+		c.Response.ValidationFailed(g, "emailColumn", errors.New("emailColumn must be a valid non-negative integer"))
+		return
+	}
+
+	dateColIdx, err := strconv.Atoi(dateColStr)
+	if err != nil || dateColIdx < 0 {
+		c.Response.ValidationFailed(g, "dateColumn", errors.New("dateColumn must be a valid non-negative integer"))
+		return
+	}
+
+	c.Logger.Debugw("using provided column indices", "emailColumn", emailColIdx, "dateColumn", dateColIdx)
+
 	// process CSV
-	processed, skipped, err := c.CampaignService.ProcessReportedCSV(g.Request.Context(), session, id, records)
+	processed, skipped, err := c.CampaignService.ProcessReportedCSV(g.Request.Context(), session, id, records, emailColIdx, dateColIdx)
 	if err != nil {
 		c.Logger.Errorw("failed to process reported CSV", "error", err)
 		if ok := c.handleErrors(g, err); !ok {

backend/service/campaign.go

@@ -4186,6 +4186,8 @@ func (c *Campaign) ProcessReportedCSV(
 	session *model.Session,
 	campaignID *uuid.UUID,
 	records [][]string,
+	emailColumnIndex int,
+	dateColumnIndex int,
 ) (int, int, error) {
 	ae := NewAuditEvent("Campaign.ProcessReportedCSV", session)
 	ae.Details["campaignID"] = campaignID.String()
@@ -4208,33 +4210,25 @@ func (c *Campaign) ProcessReportedCSV(
 		return 0, 0, errs.Wrap(err)
 	}
 
-	// validate CSV headers
-	headers := records[0]
-	reportedByIndex := -1
-	dateReportedIndex := -1
-
-	c.Logger.Debugw("processing CSV headers", "headers", headers)
-
-	for i, header := range headers {
-		switch strings.ToLower(strings.TrimSpace(header)) {
-		case "reported by":
-			reportedByIndex = i
-			c.Logger.Debugw("found reported by column", "index", i)
-		case "date reporter (utc+02:00)", "date reported(utc+02:00)", "date reported", "date reporter":
-			dateReportedIndex = i
-			c.Logger.Debugw("found date column", "index", i, "header", header)
-		}
+	if len(records) < 2 {
+		return 0, 0, errs.NewValidationError(errors.New("CSV must have at least a header row and one data row"))
 	}
 
-	if reportedByIndex == -1 {
-		c.Logger.Errorw("CSV missing required column", "expected", "reported by", "headers", headers)
-		return 0, 0, errs.NewValidationError(errors.New("CSV must have 'reported by' column"))
+	// validate column indices
+	if emailColumnIndex < 0 || emailColumnIndex >= len(records[0]) {
+		c.Logger.Errorw("invalid email column index", "index", emailColumnIndex, "columnCount", len(records[0]))
+		return 0, 0, errs.NewValidationError(errors.New("invalid email column index"))
 	}
-	if dateReportedIndex == -1 {
-		c.Logger.Errorw("CSV missing required date column", "expected", "date reported(utc+02:00)", "headers", headers)
-		return 0, 0, errs.NewValidationError(errors.New("CSV must have 'date reporter (utc+02:00)' or similar date column"))
+	if dateColumnIndex < 0 || dateColumnIndex >= len(records[0]) {
+		c.Logger.Errorw("invalid date column index", "index", dateColumnIndex, "columnCount", len(records[0]))
+		return 0, 0, errs.NewValidationError(errors.New("invalid date column index"))
 	}
 
+	reportedByIndex := emailColumnIndex
+	dateReportedIndex := dateColumnIndex
+
+	c.Logger.Infow("processing CSV columns", "emailColumn", reportedByIndex, "dateColumn", dateReportedIndex)
+
 	processed := 0
 	skipped := 0
 	reportedEventID := cache.EventIDByName[data.EVENT_CAMPAIGN_RECIPIENT_REPORTED]
@@ -4258,30 +4252,31 @@ func (c *Campaign) ProcessReportedCSV(
 		// parse date - try multiple formats and handle timezone
 		var parsedDate time.Time
 		dateFormats := []string{
-			"2006-01-02T15:04:05",
-			"2006-01-02 15:04:05",
+			// iso 8601 with fractional seconds (microsoft format)
+			"2006-01-02T15:04:05.999999999Z",
+			"2006-01-02T15:04:05.999999999-07:00",
+			"2006-01-02T15:04:05.999999999+02:00",
+			"2006-01-02T15:04:05.999999999+01:00",
+			// iso 8601 without fractional seconds
 			"2006-01-02T15:04:05Z",
 			"2006-01-02T15:04:05-07:00",
 			"2006-01-02T15:04:05+02:00",
+			"2006-01-02T15:04:05+01:00",
+			"2006-01-02T15:04:05",
+			// other common formats
+			"2006-01-02 15:04:05",
 			"2006-01-02",
 			"01/02/2006 15:04:05",
 			"01/02/2006",
 			"02-01-2006 15:04:05",
 			"02-01-2006",
+			"2006/01/02 15:04:05",
+			"2006/01/02",
 		}
 
 		dateParseError := true
 		for _, format := range dateFormats {
 			if pd, err := time.Parse(format, dateReported); err == nil {
-				// if the parsed date has no timezone info and the header mentions UTC+02:00,
-				// assume the time is in UTC+02:00 and convert to UTC
-				if pd.Location() == time.UTC && strings.Contains(strings.ToLower(headers[dateReportedIndex]), "utc+02:00") {
-					// treat as UTC+02:00 and convert to UTC
-					loc, _ := time.LoadLocation("Europe/Berlin") // UTC+2 (or use FixedZone)
-					if loc != nil {
-						pd = time.Date(pd.Year(), pd.Month(), pd.Day(), pd.Hour(), pd.Minute(), pd.Second(), pd.Nanosecond(), loc).UTC()
-					}
-				}
 				parsedDate = pd
 				dateParseError = false
 				break

frontend/src/lib/components/Modal.svelte

@@ -10,6 +10,7 @@
 	export let onClose = () => {};
 	export let bindTo = null;
 	export let resetTabFocus = () => {};
+	export let noAutoFocus = false;
 
 	let modalElement;
 	let previousActiveElement;
@@ -250,7 +251,7 @@
 		updateFocusableElements();
 
 		// Focus the first focusable element (excluding close button)
-		if (firstFocusableElement) {
+		if (!noAutoFocus && firstFocusableElement) {
 			firstFocusableElement.focus();
 		}
 	};

frontend/src/routes/campaign/[id]/+page.svelte

@@ -46,6 +46,9 @@
 	import FileField from '$lib/components/FileField.svelte';
 	import ConditionalDisplay from '$lib/components/ConditionalDisplay.svelte';
 	import IconButton from '$lib/components/IconButton.svelte';
+	import papaparse from 'papaparse';
+	import FormFooter from '$lib/components/FormFooter.svelte';
+	import TextFieldSelect from '$lib/components/TextFieldSelect.svelte';
 
 	// services
 	const appStateService = AppStateService.instance;
@@ -135,6 +138,13 @@
 	let isSetAsSentModalVisible = false;
 	let isSessionSushiModalVisible = false;
 	let isTrackingPixelWarningVisible = false;
+	let isReportedCSVModalVisible = false;
+	let isReportedCSVSubmitting = false;
+	let reportedCSVFile = null;
+	let reportedCSVHeaders = [];
+	let reportedCSVPreview = [];
+	let reportedEmailColumn = '';
+	let reportedDateColumn = '';
 	let pendingEmailPreviewRecipient = null;
 	let storedCookieData = '';
 
@@ -843,12 +853,65 @@
 			return;
 		}
 
+		// parse csv file to extract headers and preview
+		papaparse.parse(file, {
+			preview: 5, // preview first 5 rows
+			skipEmptyLines: true,
+			complete: (results) => {
+				if (!results.data || results.data.length < 2) {
+					addToast('CSV file must have headers and at least one data row', 'Error');
+					event.target.value = '';
+					return;
+				}
+
+				reportedCSVFile = file;
+				reportedCSVHeaders = results.data[0]; // first row is headers
+				reportedCSVPreview = results.data.slice(1); // remaining rows are data preview
+				reportedEmailColumn = '';
+				reportedDateColumn = '';
+				isReportedCSVModalVisible = true;
+
+				// clear the file input
+				event.target.value = '';
+			},
+			error: (error) => {
+				addToast(`Failed to parse CSV: ${error.message}`, 'Error');
+				event.target.value = '';
+			}
+		});
+	};
+
+	const closeReportedCSVModal = () => {
+		isReportedCSVModalVisible = false;
+		reportedCSVFile = null;
+		reportedCSVHeaders = [];
+		reportedCSVPreview = [];
+		reportedEmailColumn = '';
+		reportedDateColumn = '';
+	};
+
+	const onConfirmUploadReportedCSV = async () => {
+		if (!reportedCSVFile) return;
+
+		// validate that columns are selected
+		if (!reportedEmailColumn || !reportedDateColumn) {
+			addToast('Please select both email and date columns', 'Error');
+			return;
+		}
+
 		const formData = new FormData();
-		formData.append('file', file);
+		formData.append('file', reportedCSVFile);
+
+		// build URL with column indices
+		const emailColIndex = reportedCSVHeaders.indexOf(reportedEmailColumn);
+		const dateColIndex = reportedCSVHeaders.indexOf(reportedDateColumn);
+
+		let url = `/api/v1/campaign/${$page.params.id}/upload/reported?emailColumn=${emailColIndex}&dateColumn=${dateColIndex}`;
 
 		try {
+			isReportedCSVSubmitting = true;
 			showIsLoading();
-			const response = await fetch(`/api/v1/campaign/${$page.params.id}/upload/reported`, {
+			const response = await fetch(url, {
 				method: 'POST',
 				body: formData,
 				credentials: 'include'
@@ -865,6 +928,7 @@
 				await setResults();
 				await refreshCampaignRecipients();
 				await getEvents();
+				closeReportedCSVModal();
 			} else {
 				// handle validation errors
 				const errorMessage = result.error || `HTTP ${response.status}`;
@@ -874,9 +938,8 @@
 			console.error('Upload error:', error);
 			addToast('Network error: Failed to upload CSV', 'Error');
 		} finally {
+			isReportedCSVSubmitting = false;
 			hideIsLoading();
-			// clear the file input
-			event.target.value = '';
 		}
 	};
 
@@ -1540,7 +1603,7 @@
 						</p>
 						<FileField accept=".csv" on:change={onUploadReportedCSV}>Reported CSV</FileField>
 						<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">
-							Format: "Reported by" (email), "Date reported(UTC+02:00)"
+							Auto-detects email and date columns. You can specify manually if needed.
 						</p>
 					</div>
 				</div>
@@ -2266,4 +2329,55 @@
 			</p>
 		</div>
 	</Alert>
+
+	<Modal
+		headerText="Upload Reported CSV"
+		visible={isReportedCSVModalVisible}
+		onClose={closeReportedCSVModal}
+		isSubmitting={isReportedCSVSubmitting}
+		noAutoFocus={true}
+	>
+		<FormGrid on:submit={onConfirmUploadReportedCSV} isSubmitting={isReportedCSVSubmitting}>
+			<FormColumns>
+				<FormColumn>
+					<p class="text-sm text-gray-600 dark:text-gray-400 mb-4">
+						Select which columns contain the email addresses and report dates:
+					</p>
+
+					<TextFieldSelect
+						required
+						id="reported-email-column"
+						bind:value={reportedEmailColumn}
+						options={reportedCSVHeaders}
+						placeholder="Select email column...">Email Column</TextFieldSelect
+					>
+
+					<TextFieldSelect
+						required
+						id="reported-date-column"
+						bind:value={reportedDateColumn}
+						options={reportedCSVHeaders}
+						placeholder="Select date column...">Date Column</TextFieldSelect
+					>
+				</FormColumn>
+				<FormColumn overflowX={true}>
+					<Table
+						columns={reportedCSVHeaders.map((h) => ({ column: h, size: 'small' }))}
+						hasData={!!reportedCSVPreview.length}
+						plural="rows"
+					>
+						{#each reportedCSVPreview as row}
+							<TableRow>
+								{#each row as cell}
+									<TableCell value={cell} />
+								{/each}
+								<TableCellEmpty />
+							</TableRow>
+						{/each}
+					</Table>
+				</FormColumn>
+			</FormColumns>
+			<FormFooter closeModal={closeReportedCSVModal} isSubmitting={isReportedCSVSubmitting} />
+		</FormGrid>
+	</Modal>
 </main>