fix(time): correct export data timestamp conversions (#586)

This commit is contained in:
Roger
2026-04-23 20:39:56 +08:00
committed by GitHub
parent 0c6c781567
commit 50c4ea84cb
14 changed files with 228 additions and 30 deletions
+11 -9
View File
@@ -348,17 +348,19 @@ func isSkippedDir(name string) bool {
return false
}
// timeEpoch converts a WebKit/Chromium epoch timestamp (microseconds since
// 1601-01-01) to a time.Time.
// Offset from the Chromium epoch (1601-01-01 UTC) to the Unix epoch,
// matching base::Time::kTimeTToMicrosecondsOffset in Chromium.
const chromiumEpochOffsetMicros int64 = 11644473600000000
// timeEpoch converts a Chromium base::Time (μs since 1601 UTC) to UTC.
// Returns zero for non-positive input or out-of-JSON-range values.
func timeEpoch(epoch int64) time.Time {
maxTime := int64(99633311740000000)
if epoch > maxTime {
return time.Date(2049, 1, 1, 1, 1, 1, 1, time.Local)
if epoch <= 0 {
return time.Time{}
}
t := time.Date(1601, 1, 1, 0, 0, 0, 0, time.Local)
d := time.Duration(epoch)
for i := 0; i < 1000; i++ {
t = t.Add(d)
t := time.UnixMicro(epoch - chromiumEpochOffsetMicros).UTC()
if t.Year() < 1 || t.Year() > 9999 {
return time.Time{}
}
return t
}
+45
View File
@@ -4,6 +4,7 @@ import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -720,3 +721,47 @@ func TestSetKeyRetrievers_SatisfiesInterface(t *testing.T) {
SetKeyRetrievers(keyretriever.Retrievers)
} = (*Browser)(nil)
}
// Anchor: 2024-01-15T10:30:00Z as Chromium microseconds since 1601 UTC.
const anchorUnixSeconds = int64(1705314600)
var anchorChromiumMicros = (anchorUnixSeconds + 11644473600) * 1_000_000
func TestTimeEpoch_AnchorDate(t *testing.T) {
got := timeEpoch(anchorChromiumMicros)
want := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
assert.Equal(t, want, got)
assert.Equal(t, anchorUnixSeconds, got.Unix())
}
func TestTimeEpoch_ZeroReturnsZeroTime(t *testing.T) {
assert.True(t, timeEpoch(0).IsZero())
}
func TestTimeEpoch_NegativeReturnsZeroTime(t *testing.T) {
assert.True(t, timeEpoch(-1).IsZero())
}
func TestTimeEpoch_AlwaysUTC(t *testing.T) {
// assert.Same checks pointer equality: time.UTC and time.Local are
// distinct *Location globals, so this catches any regression that
// drops .UTC() even when the runner's TZ happens to be UTC.
got := timeEpoch(anchorChromiumMicros)
assert.Same(t, time.UTC, got.Location())
}
func TestTimeEpoch_MicrosecondPrecisionPreserved(t *testing.T) {
got := timeEpoch(anchorChromiumMicros + 123456)
assert.Equal(t, 123456*int64(time.Microsecond), int64(got.Nanosecond()))
}
func TestTimeEpoch_UnixEpochBoundary(t *testing.T) {
got := timeEpoch(chromiumEpochOffsetMicros)
assert.Equal(t, time.Unix(0, 0).UTC(), got)
}
func TestTimeEpoch_OutOfJSONRangeReturnsZero(t *testing.T) {
jsonBytes, err := timeEpoch(1 << 62).MarshalJSON()
require.NoError(t, err)
assert.JSONEq(t, `"0001-01-01T00:00:00Z"`, string(jsonBytes))
}
+1 -1
View File
@@ -28,7 +28,7 @@ func extractBookmarks(path string) ([]types.BookmarkEntry, error) {
Name: title,
URL: url,
Folder: bookmarkType(bt),
CreatedAt: timestamp(dateAdded / 1000000),
CreatedAt: firefoxMicros(dateAdded),
}, nil
})
if err != nil {
+2 -2
View File
@@ -36,8 +36,8 @@ func extractCookies(path string) ([]types.CookieEntry, error) {
IsHTTPOnly: isHTTPOnly != 0,
HasExpire: hasExpire,
IsPersistent: hasExpire,
ExpireAt: timestamp(expiry),
CreatedAt: timestamp(createdAt / 1000000),
ExpireAt: firefoxSeconds(expiry),
CreatedAt: firefoxMicros(createdAt),
}, nil
})
if err != nil {
+2 -2
View File
@@ -32,7 +32,7 @@ func extractDownloads(path string) ([]types.DownloadEntry, error) {
entry := types.DownloadEntry{
URL: url,
StartTime: timestamp(dateAdded / 1000000),
StartTime: firefoxMicros(dateAdded),
}
// Firefox stores download metadata as: "target_path,{json}"
@@ -42,7 +42,7 @@ func extractDownloads(path string) ([]types.DownloadEntry, error) {
entry.TargetPath = contentList[0]
json := "{" + contentList[1]
entry.TotalBytes = gjson.Get(json, "fileSize").Int()
entry.EndTime = timestamp(gjson.Get(json, "endTime").Int() / 1000)
entry.EndTime = firefoxMillis(gjson.Get(json, "endTime").Int())
} else {
entry.TargetPath = content
}
+1 -1
View File
@@ -27,7 +27,7 @@ func extractHistories(path string) ([]types.HistoryEntry, error) {
URL: url,
Title: title,
VisitCount: visitCount,
LastVisit: timestamp(lastVisit / 1000000),
LastVisit: firefoxMicros(lastVisit),
}, nil
})
if err != nil {
+1 -1
View File
@@ -68,7 +68,7 @@ func extractPasswords(masterKey []byte, path string) ([]types.LoginEntry, error)
URL: url,
Username: string(user),
Password: string(pwd),
CreatedAt: timestamp(v.Get("timeCreated").Int() / 1000),
CreatedAt: firefoxMillis(v.Get("timeCreated").Int()),
})
}
if decryptFails > 0 {
+33 -6
View File
@@ -288,11 +288,38 @@ func resolveSourcePaths(sources map[types.Category][]sourcePath, profileDir stri
return resolved
}
// timestamp converts a Unix epoch timestamp (seconds) to a time.Time.
func timestamp(stamp int64) time.Time {
s := time.Unix(stamp, 0)
if s.Local().Year() > 9999 {
return time.Date(9999, 12, 13, 23, 59, 59, 0, time.Local)
// Firefox uses three timestamp units. Helpers emit UTC and return the zero
// time.Time for non-positive or out-of-JSON-range input.
//
// - firefoxMicros: PRTime (μs since Unix epoch) — moz_* tables.
// - firefoxMillis: Date.now() (ms) — logins.json, download endTime.
// - firefoxSeconds: seconds — moz_cookies.expiry only.
func firefoxMicros(us int64) time.Time {
if us <= 0 {
return time.Time{}
}
return s
return clampJSON(time.UnixMicro(us).UTC())
}
func firefoxMillis(ms int64) time.Time {
if ms <= 0 {
return time.Time{}
}
return clampJSON(time.UnixMilli(ms).UTC())
}
func firefoxSeconds(s int64) time.Time {
if s <= 0 {
return time.Time{}
}
return clampJSON(time.Unix(s, 0).UTC())
}
// clampJSON maps years outside time.Time.MarshalJSON's [1, 9999] window
// to the zero time, so JSON export can't crash on sentinel inputs.
func clampJSON(t time.Time) time.Time {
if t.Year() < 1 || t.Year() > 9999 {
return time.Time{}
}
return t
}
+77
View File
@@ -4,6 +4,7 @@ import (
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -315,3 +316,79 @@ func TestExtractCategory(t *testing.T) {
assert.Empty(t, data.SessionStorage)
})
}
// Anchor: 2024-01-15T10:30:00Z.
const anchorUnixSeconds = int64(1705314600)
func TestFirefoxMicros_AnchorDate(t *testing.T) {
got := firefoxMicros(anchorUnixSeconds * 1_000_000)
want := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
assert.Equal(t, want, got)
}
func TestFirefoxMicros_PrecisionPreserved(t *testing.T) {
got := firefoxMicros(anchorUnixSeconds*1_000_000 + 123456)
assert.Equal(t, 123456*int64(time.Microsecond), int64(got.Nanosecond()))
}
func TestFirefoxMillis_AnchorDate(t *testing.T) {
got := firefoxMillis(anchorUnixSeconds * 1_000)
want := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
assert.Equal(t, want, got)
}
func TestFirefoxMillis_PrecisionPreserved(t *testing.T) {
got := firefoxMillis(anchorUnixSeconds*1_000 + 789)
assert.Equal(t, 789*int64(time.Millisecond), int64(got.Nanosecond()))
}
func TestFirefoxSeconds_AnchorDate(t *testing.T) {
got := firefoxSeconds(anchorUnixSeconds)
want := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
assert.Equal(t, want, got)
}
func TestFirefoxHelpers_ZeroReturnsZeroTime(t *testing.T) {
assert.True(t, firefoxMicros(0).IsZero(), "micros")
assert.True(t, firefoxMillis(0).IsZero(), "millis")
assert.True(t, firefoxSeconds(0).IsZero(), "seconds")
}
func TestFirefoxHelpers_NegativeReturnsZeroTime(t *testing.T) {
assert.True(t, firefoxMicros(-1).IsZero(), "micros")
assert.True(t, firefoxMillis(-1).IsZero(), "millis")
assert.True(t, firefoxSeconds(-1).IsZero(), "seconds")
}
func TestFirefoxHelpers_AlwaysUTC(t *testing.T) {
// assert.Same: pointer equality reliably catches any helper that
// leaks time.Local, independent of the runner's configured TZ.
assert.Same(t, time.UTC, firefoxMicros(anchorUnixSeconds*1_000_000).Location())
assert.Same(t, time.UTC, firefoxMillis(anchorUnixSeconds*1_000).Location())
assert.Same(t, time.UTC, firefoxSeconds(anchorUnixSeconds).Location())
}
func TestFirefoxHelpers_SameMomentAcrossUnits(t *testing.T) {
us := firefoxMicros(anchorUnixSeconds * 1_000_000)
ms := firefoxMillis(anchorUnixSeconds * 1_000)
s := firefoxSeconds(anchorUnixSeconds)
assert.True(t, us.Equal(ms))
assert.True(t, ms.Equal(s))
}
func TestFirefoxHelpers_OutOfJSONRangeReturnsZero(t *testing.T) {
for _, tc := range []struct {
name string
got time.Time
}{
{"seconds", firefoxSeconds(1 << 50)},
{"millis", firefoxMillis(1 << 60)},
{"micros", firefoxMicros(1 << 62)},
} {
t.Run(tc.name, func(t *testing.T) {
b, err := tc.got.MarshalJSON()
require.NoError(t, err)
assert.JSONEq(t, `"0001-01-01T00:00:00Z"`, string(b))
})
}
}
+4 -2
View File
@@ -20,6 +20,8 @@ func extractCookies(path string) ([]types.CookieEntry, error) {
for _, page := range pages {
for _, c := range page.Cookies {
hasExpire := !c.Expires.IsZero()
// binarycookies returns time.Time in Local; normalize to UTC
// so exported JSON matches Chromium/Firefox cookie output.
cookies = append(cookies, types.CookieEntry{
Host: string(c.Domain),
Path: string(c.Path),
@@ -29,8 +31,8 @@ func extractCookies(path string) ([]types.CookieEntry, error) {
IsHTTPOnly: c.HTTPOnly,
HasExpire: hasExpire,
IsPersistent: hasExpire,
ExpireAt: c.Expires,
CreatedAt: c.Creation,
ExpireAt: c.Expires.UTC(),
CreatedAt: c.Creation.UTC(),
})
}
}
+4 -3
View File
@@ -95,9 +95,10 @@ func TestExtractHistories_NullTitle(t *testing.T) {
}
func TestCoredataTimestamp(t *testing.T) {
// 0 Core Data epoch = 2001-01-01 00:00:00 UTC = Unix 978307200
ts := coredataTimestamp(0)
assert.Equal(t, int64(978307200), ts.Unix())
// A zero Core Data value is treated as "no timestamp" and returns
// the zero time.Time rather than literal 2001-01-01 — matches the
// convention used by the Chromium and Firefox helpers.
assert.True(t, coredataTimestamp(0).IsZero())
// Known value: 700000000 Core Data = 1678307200 Unix
ts2 := coredataTimestamp(700000000)
+1 -1
View File
@@ -27,7 +27,7 @@ func extractPasswords(keychainPassword string) ([]types.LoginEntry, error) {
URL: url,
Username: p.Account,
Password: p.PlainPassword,
CreatedAt: p.Created,
CreatedAt: p.Created.UTC(),
})
}
+16 -2
View File
@@ -212,9 +212,23 @@ func resolveSourcePaths(sources map[types.Category][]sourcePath) map[types.Categ
return resolved
}
// Safari's History.db uses the Core Data epoch (2001-01-01) instead of Unix epoch.
// Offset from the Core Data epoch (2001-01-01 UTC) to the Unix epoch.
const coreDataEpochOffset = 978307200
// maxCoreDataSeconds is the largest CFAbsoluteTime that still lands inside
// time.Time.MarshalJSON's [1, 9999] year window. Also bounds the float →
// int64 conversion below; Go's spec makes out-of-range conversions return
// an implementation-dependent int64, which could silently corrupt results.
const maxCoreDataSeconds = 252423993600
// coredataTimestamp converts Core Data seconds (CFAbsoluteTime) to UTC.
// Returns zero for non-positive input or out-of-JSON-range values.
func coredataTimestamp(seconds float64) time.Time {
return time.Unix(int64(seconds)+coreDataEpochOffset, 0)
if seconds <= 0 || seconds > maxCoreDataSeconds {
return time.Time{}
}
whole := int64(seconds)
frac := seconds - float64(whole)
nanos := int64(frac * 1e9)
return time.Unix(whole+coreDataEpochOffset, nanos).UTC()
}
+30
View File
@@ -5,6 +5,7 @@ import (
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -334,3 +335,32 @@ func TestExtractCategory(t *testing.T) {
assert.Empty(t, data.CreditCards)
})
}
// Anchor: 2024-01-15T10:30:00Z, in seconds past the Core Data epoch (2001-01-01Z).
const anchorCoreDataSeconds = 1705314600 - 978307200
func TestCoredataTimestamp_AnchorDate(t *testing.T) {
got := coredataTimestamp(float64(anchorCoreDataSeconds))
want := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
assert.Equal(t, want, got)
}
func TestCoredataTimestamp_EpochZero(t *testing.T) {
assert.True(t, coredataTimestamp(0).IsZero())
}
func TestCoredataTimestamp_NegativeReturnsZeroTime(t *testing.T) {
assert.True(t, coredataTimestamp(-1).IsZero())
}
func TestCoredataTimestamp_FractionalSecondsPreserved(t *testing.T) {
got := coredataTimestamp(float64(anchorCoreDataSeconds) + 0.5)
assert.Equal(t, 500*int64(time.Millisecond), int64(got.Nanosecond()))
}
func TestCoredataTimestamp_AlwaysUTC(t *testing.T) {
// assert.Same: pointer equality reliably catches any regression that
// leaks time.Local, independent of the runner's configured TZ.
got := coredataTimestamp(float64(anchorCoreDataSeconds))
assert.Same(t, time.UTC, got.Location())
}