feat: add Firefox extract methods and complete data model fields (#527)

* feat: add Firefox extract methods and complete data model fields

Firefox extract methods:
- extractPasswords: JSON + ASN1PBE decryption via decryptPBE helper
- extractCookies: SQLite, plaintext (no encryption), journalOff
- extractHistories: SQLite, visit count ASC sort (matches old behavior)
- extractDownloads: SQLite, moz_annos JOIN with JSON content parsing
- extractBookmarks: SQLite, moz_bookmarks JOIN moz_places
- extractExtensions: JSON, filter by location=app-profile
- extractLocalStorage: SQLite webappsstore2, reversed originKey parsing

Complete data model fields (union of Chromium and Firefox):
- CookieEntry: add HasExpire, IsPersistent
- DownloadEntry: add MimeType
- CreditCardEntry: add NickName, Address
- ExtensionEntry: add HomepageURL, Enabled

Update Chromium extractors to populate new fields:
- extract_cookie.go: fill HasExpire, IsPersistent
- extract_download.go: SELECT and fill mime_type
- extract_creditcard.go: SELECT nickname, billing_address_id
- extract_extension.go: fill HomepageURL, Enabled (state==1)

Tests:
- Full test coverage for all 7 Firefox extract functions
- Password test uses known ASN1PBE test vectors from crypto package
- Table-driven tests for parseOriginKey
- Updated Chromium tests for new fields

* fix: add COALESCE for nullable bookmark title in Firefox query

Firefox moz_bookmarks.title can be NULL (PR #500 fixed this in old code).
Add COALESCE to handle NULL gracefully in SQL instead of relying on
driver-specific NULL→string conversion behavior.

* fix: enable journalOff for all Firefox SQLite extractors and populate cookie flags

- Set journalOff=true for extract_history, extract_download, extract_bookmark
  (Firefox databases require PRAGMA journal_mode=off to avoid lock errors)
- Populate HasExpire and IsPersistent for Firefox cookies (derived from expiry>0)
- Add test assertions for HasExpire/IsPersistent in both Chromium and Firefox
This commit is contained in:
Roger
2026-03-30 20:52:11 +08:00
committed by moonD4rk
parent 2c4e871e59
commit 1ec2781131
25 changed files with 937 additions and 38 deletions
+1 -1
View File
@@ -200,7 +200,7 @@ linters:
- path: "browser/chromium/(source|decrypt|extract_.*)\\.go"
linters:
- unused
- path: "browser/firefox/source\\.go"
- path: "browser/firefox/(source|extract_.*)\\.go"
linters:
- unused
+10 -8
View File
@@ -34,14 +34,16 @@ func extractCookies(masterKey []byte, path string) ([]types.CookieEntry, error)
value, _ := decryptValue(masterKey, encryptedValue)
value = stripCookieHash(value, host)
return types.CookieEntry{
Name: name,
Host: host,
Path: cookiePath,
Value: string(value),
IsSecure: isSecure != 0,
IsHTTPOnly: isHTTPOnly != 0,
ExpireAt: typeutil.TimeEpoch(expireAt),
CreatedAt: typeutil.TimeEpoch(createdAt),
Name: name,
Host: host,
Path: cookiePath,
Value: string(value),
IsSecure: isSecure != 0,
IsHTTPOnly: isHTTPOnly != 0,
HasExpire: hasExpire != 0,
IsPersistent: isPersistent != 0,
ExpireAt: typeutil.TimeEpoch(expireAt),
CreatedAt: typeutil.TimeEpoch(createdAt),
}, nil
})
if err != nil {
+2
View File
@@ -27,6 +27,8 @@ func TestExtractCookies(t *testing.T) {
assert.Equal(t, "/api", got[0].Path)
assert.True(t, got[0].IsSecure)
assert.False(t, got[0].IsHTTPOnly)
assert.True(t, got[0].HasExpire)
assert.True(t, got[0].IsPersistent)
assert.False(t, got[0].CreatedAt.IsZero())
assert.True(t, got[0].ExpireAt.After(got[0].CreatedAt))
assert.True(t, got[1].IsHTTPOnly)
+5 -3
View File
@@ -8,14 +8,14 @@ import (
)
const defaultCreditCardQuery = `SELECT name_on_card, expiration_month, expiration_year,
card_number_encrypted FROM credit_cards`
card_number_encrypted, COALESCE(nickname, ''), COALESCE(billing_address_id, '') FROM credit_cards`
func extractCreditCards(masterKey []byte, path string) ([]types.CreditCardEntry, error) {
return sqliteutil.QueryRows(path, false, defaultCreditCardQuery,
func(rows *sql.Rows) (types.CreditCardEntry, error) {
var name, month, year string
var name, month, year, nickName, address string
var encNumber []byte
if err := rows.Scan(&name, &month, &year, &encNumber); err != nil {
if err := rows.Scan(&name, &month, &year, &encNumber, &nickName, &address); err != nil {
return types.CreditCardEntry{}, err
}
number, _ := decryptValue(masterKey, encNumber)
@@ -24,6 +24,8 @@ func extractCreditCards(masterKey []byte, path string) ([]types.CreditCardEntry,
Number: string(number),
ExpMonth: month,
ExpYear: year,
NickName: nickName,
Address: address,
}, nil
})
}
+2 -2
View File
@@ -9,8 +9,8 @@ import (
func TestExtractCreditCards(t *testing.T) {
path := createTestDB(t, "Web Data", creditCardsSchema,
insertCreditCard("John Doe", 12, 2025, ""),
insertCreditCard("Jane Smith", 6, 2027, ""),
insertCreditCard("John Doe", 12, 2025, "", "Johnny", "addr-1"),
insertCreditCard("Jane Smith", 6, 2027, "", "", ""),
)
got, err := extractCreditCards(nil, path)
+5 -3
View File
@@ -9,19 +9,21 @@ import (
"github.com/moond4rk/hackbrowserdata/utils/typeutil"
)
const defaultDownloadQuery = `SELECT target_path, tab_url, total_bytes, start_time, end_time FROM downloads`
const defaultDownloadQuery = `SELECT target_path, tab_url, total_bytes, start_time, end_time,
mime_type FROM downloads`
func extractDownloads(path string) ([]types.DownloadEntry, error) {
downloads, err := sqliteutil.QueryRows(path, false, defaultDownloadQuery,
func(rows *sql.Rows) (types.DownloadEntry, error) {
var targetPath, url string
var targetPath, url, mimeType string
var totalBytes, startTime, endTime int64
if err := rows.Scan(&targetPath, &url, &totalBytes, &startTime, &endTime); err != nil {
if err := rows.Scan(&targetPath, &url, &totalBytes, &startTime, &endTime, &mimeType); err != nil {
return types.DownloadEntry{}, err
}
return types.DownloadEntry{
URL: url,
TargetPath: targetPath,
MimeType: mimeType,
TotalBytes: totalBytes,
StartTime: typeutil.TimeEpoch(startTime),
EndTime: typeutil.TimeEpoch(endTime),
+3 -2
View File
@@ -9,8 +9,8 @@ import (
func TestExtractDownloads(t *testing.T) {
path := createTestDB(t, "History", downloadsSchema,
insertDownload("/tmp/old.zip", "https://old.com/file.zip", 1024, 13340000000000000, 13340000100000000),
insertDownload("/tmp/new.pdf", "https://new.com/doc.pdf", 2048, 13360000000000000, 13360000200000000),
insertDownload("/tmp/old.zip", "https://old.com/file.zip", "application/zip", 1024, 13340000000000000, 13340000100000000),
insertDownload("/tmp/new.pdf", "https://new.com/doc.pdf", "application/pdf", 2048, 13360000000000000, 13360000200000000),
)
got, err := extractDownloads(path)
@@ -23,6 +23,7 @@ func TestExtractDownloads(t *testing.T) {
// Verify field mapping
assert.Equal(t, "/tmp/new.pdf", got[0].TargetPath)
assert.Equal(t, "application/pdf", got[0].MimeType)
assert.Equal(t, int64(2048), got[0].TotalBytes)
assert.False(t, got[0].StartTime.IsZero())
assert.False(t, got[0].EndTime.IsZero())
+2
View File
@@ -51,6 +51,8 @@ func extractExtensions(path string) ([]types.ExtensionEntry, error) {
ID: id.String(),
Description: manifest.Get("description").String(),
Version: manifest.Get("version").String(),
HomepageURL: manifest.Get("homepage_url").String(),
Enabled: ext.Get("state").Int() == 1,
})
return true
})
+7 -7
View File
@@ -173,22 +173,22 @@ func insertURL(url, title string, visitCount int, lastVisitTime int64) string {
)
}
func insertDownload(targetPath, tabURL string, totalBytes, startTime, endTime int64) string {
func insertDownload(targetPath, tabURL, mimeType string, totalBytes, startTime, endTime int64) string {
return fmt.Sprintf(
`INSERT INTO downloads (id, guid, current_path, target_path, start_time, received_bytes,
total_bytes, state, danger_type, interrupt_reason, hash, end_time, opened, last_access_time,
transient, referrer, site_url, embedder_download_data, tab_url, tab_referrer_url,
http_method, by_ext_id, by_ext_name, by_web_app_id, etag, last_modified, mime_type, original_mime_type)
VALUES (NULL, '', '', '%s', %d, %d, %d, 1, 0, 0, x'', %d, 0, 0, 0, '', '', '', '%s', '', 'GET', '', '', '', '', '', '', '')`,
targetPath, startTime, totalBytes, totalBytes, endTime, tabURL,
VALUES (NULL, '', '', '%s', %d, %d, %d, 1, 0, 0, x'', %d, 0, 0, 0, '', '', '', '%s', '', 'GET', '', '', '', '', '', '%s', '')`,
targetPath, startTime, totalBytes, totalBytes, endTime, tabURL, mimeType,
)
}
func insertCreditCard(name string, month, year int, encNumberHex string) string {
func insertCreditCard(name string, month, year int, encNumberHex, nickName, address string) string {
return fmt.Sprintf(
`INSERT INTO credit_cards (guid, name_on_card, expiration_month, expiration_year, card_number_encrypted)
VALUES ('%s-%d-%d', '%s', %d, %d, x'%s')`,
name, month, year, name, month, year, encNumberHex,
`INSERT INTO credit_cards (guid, name_on_card, expiration_month, expiration_year, card_number_encrypted, nickname, billing_address_id)
VALUES ('%s-%d-%d', '%s', %d, %d, x'%s', '%s', '%s')`,
name, month, year, name, month, year, encNumberHex, nickName, address,
)
}
+48
View File
@@ -0,0 +1,48 @@
package firefox
import (
"database/sql"
"sort"
"github.com/moond4rk/hackbrowserdata/types"
"github.com/moond4rk/hackbrowserdata/utils/sqliteutil"
"github.com/moond4rk/hackbrowserdata/utils/typeutil"
)
const firefoxBookmarkQuery = `SELECT id, url, type, dateAdded, COALESCE(title, '')
FROM (SELECT * FROM moz_bookmarks INNER JOIN moz_places ON moz_bookmarks.fk=moz_places.id)`
func extractBookmarks(path string) ([]types.BookmarkEntry, error) {
bookmarks, err := sqliteutil.QueryRows(path, true, firefoxBookmarkQuery,
func(rows *sql.Rows) (types.BookmarkEntry, error) {
var id, dateAdded int64
var url, title string
var bt int64
if err := rows.Scan(&id, &url, &bt, &dateAdded, &title); err != nil {
return types.BookmarkEntry{}, err
}
return types.BookmarkEntry{
Name: title,
URL: url,
Folder: bookmarkType(bt),
CreatedAt: typeutil.TimeStamp(dateAdded / 1000000),
}, nil
})
if err != nil {
return nil, err
}
sort.Slice(bookmarks, func(i, j int) bool {
return bookmarks[i].CreatedAt.After(bookmarks[j].CreatedAt)
})
return bookmarks, nil
}
func bookmarkType(bt int64) string {
switch bt {
case 1:
return "url"
default:
return "folder"
}
}
+31
View File
@@ -0,0 +1,31 @@
package firefox
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExtractBookmarks(t *testing.T) {
// Bookmarks require JOIN: moz_bookmarks.fk = moz_places.id
path := createTestDB(t, "places.sqlite", []string{mozPlacesSchema, mozBookmarksSchema},
insertMozPlace(1, "https://go.dev", "Go", 0, 0),
insertMozPlace(2, "https://github.com", "GitHub", 0, 0),
insertMozBookmark(1, 1, 1, "Go Website", 1700000000000000),
insertMozBookmark(2, 2, 1, "GitHub", 1710000000000000),
)
got, err := extractBookmarks(path)
require.NoError(t, err)
require.Len(t, got, 2)
// Verify sort order: dateAdded descending
assert.Equal(t, "GitHub", got[0].Name)
assert.Equal(t, "Go Website", got[1].Name)
// Verify field mapping
assert.Equal(t, "https://github.com", got[0].URL)
assert.Equal(t, "url", got[0].Folder) // type=1 → "url"
assert.False(t, got[0].CreatedAt.IsZero())
}
+49
View File
@@ -0,0 +1,49 @@
package firefox
import (
"database/sql"
"sort"
"github.com/moond4rk/hackbrowserdata/types"
"github.com/moond4rk/hackbrowserdata/utils/sqliteutil"
"github.com/moond4rk/hackbrowserdata/utils/typeutil"
)
const firefoxCookieQuery = `SELECT name, value, host, path,
creationTime, expiry, isSecure, isHttpOnly FROM moz_cookies`
func extractCookies(path string) ([]types.CookieEntry, error) {
cookies, err := sqliteutil.QueryRows(path, true, firefoxCookieQuery,
func(rows *sql.Rows) (types.CookieEntry, error) {
var (
name, value, host, cookiePath string
isSecure, isHTTPOnly int
createdAt, expiry int64
)
if err := rows.Scan(&name, &value, &host, &cookiePath,
&createdAt, &expiry, &isSecure, &isHTTPOnly); err != nil {
return types.CookieEntry{}, err
}
hasExpire := expiry > 0
return types.CookieEntry{
Name: name,
Host: host,
Path: cookiePath,
Value: value, // Firefox cookies are not encrypted
IsSecure: isSecure != 0,
IsHTTPOnly: isHTTPOnly != 0,
HasExpire: hasExpire,
IsPersistent: hasExpire,
ExpireAt: typeutil.TimeStamp(expiry),
CreatedAt: typeutil.TimeStamp(createdAt / 1000000),
}, nil
})
if err != nil {
return nil, err
}
sort.Slice(cookies, func(i, j int) bool {
return cookies[i].CreatedAt.After(cookies[j].CreatedAt)
})
return cookies, nil
}
+35
View File
@@ -0,0 +1,35 @@
package firefox
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExtractCookies(t *testing.T) {
path := createTestDB(t, "cookies.sqlite", []string{mozCookiesSchema},
insertMozCookie("session", "abc123", ".example.com", "/", 1700000000000000, 1800000000, 1, 1),
insertMozCookie("token", "xyz789", ".new.com", "/api", 1710000000000000, 1810000000, 1, 0),
)
got, err := extractCookies(path)
require.NoError(t, err)
require.Len(t, got, 2)
// Verify sort order: creation time descending
assert.Equal(t, ".new.com", got[0].Host)
assert.Equal(t, ".example.com", got[1].Host)
// Verify field mapping — Firefox cookies are plaintext
assert.Equal(t, "token", got[0].Name)
assert.Equal(t, "xyz789", got[0].Value)
assert.Equal(t, "/api", got[0].Path)
assert.True(t, got[0].IsSecure)
assert.False(t, got[0].IsHTTPOnly)
assert.True(t, got[0].HasExpire) // expiry > 0
assert.True(t, got[0].IsPersistent) // expiry > 0
assert.Equal(t, "abc123", got[1].Value)
assert.True(t, got[1].IsHTTPOnly)
}
+55
View File
@@ -0,0 +1,55 @@
package firefox
import (
"database/sql"
"sort"
"strings"
"github.com/tidwall/gjson"
"github.com/moond4rk/hackbrowserdata/types"
"github.com/moond4rk/hackbrowserdata/utils/sqliteutil"
"github.com/moond4rk/hackbrowserdata/utils/typeutil"
)
const firefoxDownloadQuery = `SELECT place_id, GROUP_CONCAT(content), url, dateAdded
FROM (SELECT * FROM moz_annos INNER JOIN moz_places ON moz_annos.place_id=moz_places.id)
t GROUP BY place_id`
func extractDownloads(path string) ([]types.DownloadEntry, error) {
downloads, err := sqliteutil.QueryRows(path, true, firefoxDownloadQuery,
func(rows *sql.Rows) (types.DownloadEntry, error) {
var placeID, dateAdded int64
var content, url string
if err := rows.Scan(&placeID, &content, &url, &dateAdded); err != nil {
return types.DownloadEntry{}, err
}
entry := types.DownloadEntry{
URL: url,
StartTime: typeutil.TimeStamp(dateAdded / 1000000),
}
// Firefox stores download metadata as: "target_path,{json}"
// Parse the JSON part to extract fileSize and endTime.
contentList := strings.SplitN(content, ",{", 2)
if len(contentList) == 2 {
entry.TargetPath = contentList[0]
json := "{" + contentList[1]
entry.TotalBytes = gjson.Get(json, "fileSize").Int()
entry.EndTime = typeutil.TimeStamp(gjson.Get(json, "endTime").Int() / 1000)
} else {
entry.TargetPath = content
}
return entry, nil
})
if err != nil {
return nil, err
}
sort.Slice(downloads, func(i, j int) bool {
return downloads[i].StartTime.After(downloads[j].StartTime)
})
return downloads, nil
}
+30
View File
@@ -0,0 +1,30 @@
package firefox
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExtractDownloads(t *testing.T) {
// Downloads require JOIN: moz_annos.place_id = moz_places.id
path := createTestDB(t, "places.sqlite", []string{mozPlacesSchema, mozAnnosSchema},
insertMozPlace(1, "https://example.com/old.zip", "Old File", 0, 0),
insertMozPlace(2, "https://example.com/new.pdf", "New File", 0, 0),
insertMozAnno(1, "/tmp/old.zip", 1700000000000000),
insertMozAnno(2, "/tmp/new.pdf", 1710000000000000),
)
got, err := extractDownloads(path)
require.NoError(t, err)
require.Len(t, got, 2)
// Verify sort order: StartTime descending
assert.Equal(t, "https://example.com/new.pdf", got[0].URL)
assert.Equal(t, "https://example.com/old.zip", got[1].URL)
// Verify field mapping
assert.Equal(t, "/tmp/new.pdf", got[0].TargetPath)
assert.False(t, got[0].StartTime.IsZero())
}
+36
View File
@@ -0,0 +1,36 @@
package firefox
import (
"os"
"github.com/tidwall/gjson"
"github.com/moond4rk/hackbrowserdata/types"
)
func extractExtensions(path string) ([]types.ExtensionEntry, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var extensions []types.ExtensionEntry
for _, v := range gjson.GetBytes(data, "addons").Array() {
// Only include user-installed extensions
// https://searchfox.org/mozilla-central/source/toolkit/mozapps/extensions/internal/XPIDatabase.jsm#157
if v.Get("location").String() != "app-profile" {
continue
}
extensions = append(extensions, types.ExtensionEntry{
Name: v.Get("defaultLocale.name").String(),
ID: v.Get("id").String(),
Description: v.Get("defaultLocale.description").String(),
Version: v.Get("version").String(),
HomepageURL: v.Get("defaultLocale.homepageURL").String(),
Enabled: v.Get("active").Bool(),
})
}
return extensions, nil
}
+62
View File
@@ -0,0 +1,62 @@
package firefox
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExtractExtensions(t *testing.T) {
path := createTestJSON(t, "extensions.json", `{
"addons": [
{
"id": "ublock@gorhill.org",
"location": "app-profile",
"version": "1.52.0",
"active": true,
"defaultLocale": {
"name": "uBlock Origin",
"description": "An efficient blocker"
}
},
{
"id": "system@mozilla.org",
"location": "app-system-defaults",
"version": "1.0",
"defaultLocale": {"name": "System Addon"}
},
{
"id": "bitwarden@bitwarden.com",
"location": "app-profile",
"version": "2024.1.0",
"active": true,
"defaultLocale": {
"name": "Bitwarden",
"description": "Password manager"
}
}
]
}`)
got, err := extractExtensions(path)
require.NoError(t, err)
require.Len(t, got, 2) // system addon filtered out
ids := map[string]bool{}
for _, ext := range got {
ids[ext.ID] = true
assert.NotEmpty(t, ext.Name)
assert.NotEmpty(t, ext.Version)
}
assert.True(t, ids["ublock@gorhill.org"])
assert.True(t, ids["bitwarden@bitwarden.com"])
assert.False(t, ids["system@mozilla.org"])
}
func TestExtractExtensions_EmptyAddons(t *testing.T) {
path := createTestJSON(t, "extensions.json", `{"addons": []}`)
got, err := extractExtensions(path)
require.NoError(t, err)
assert.Empty(t, got)
}
+39
View File
@@ -0,0 +1,39 @@
package firefox
import (
"database/sql"
"sort"
"github.com/moond4rk/hackbrowserdata/types"
"github.com/moond4rk/hackbrowserdata/utils/sqliteutil"
"github.com/moond4rk/hackbrowserdata/utils/typeutil"
)
const firefoxHistoryQuery = `SELECT url, COALESCE(last_visit_date, 0),
COALESCE(title, ''), visit_count FROM moz_places`
func extractHistories(path string) ([]types.HistoryEntry, error) {
histories, err := sqliteutil.QueryRows(path, true, firefoxHistoryQuery,
func(rows *sql.Rows) (types.HistoryEntry, error) {
var url, title string
var visitCount int
var lastVisit int64
if err := rows.Scan(&url, &lastVisit, &title, &visitCount); err != nil {
return types.HistoryEntry{}, err
}
return types.HistoryEntry{
URL: url,
Title: title,
VisitCount: visitCount,
LastVisit: typeutil.TimeStamp(lastVisit / 1000000),
}, nil
})
if err != nil {
return nil, err
}
sort.Slice(histories, func(i, j int) bool {
return histories[i].VisitCount < histories[j].VisitCount
})
return histories, nil
}
+44
View File
@@ -0,0 +1,44 @@
package firefox
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExtractHistories(t *testing.T) {
path := createTestDB(t, "places.sqlite", []string{mozPlacesSchema},
insertMozPlace(1, "https://github.com", "GitHub", 100, 1700000000000000),
insertMozPlace(2, "https://go.dev", "Go", 50, 1710000000000000),
insertMozPlace(3, "https://example.com", "Example", 200, 1690000000000000),
)
got, err := extractHistories(path)
require.NoError(t, err)
require.Len(t, got, 3)
// Verify sort order: visit count ascending (Firefox convention)
assert.Equal(t, 50, got[0].VisitCount)
assert.Equal(t, 100, got[1].VisitCount)
assert.Equal(t, 200, got[2].VisitCount)
// Verify field mapping (first = least visited)
assert.Equal(t, "https://go.dev", got[0].URL)
assert.Equal(t, "Go", got[0].Title)
assert.False(t, got[0].LastVisit.IsZero())
}
func TestExtractHistories_NullFields(t *testing.T) {
path := createTestDB(t, "places.sqlite", []string{mozPlacesSchema},
// last_visit_date=NULL, title=NULL — COALESCE should handle
`INSERT INTO moz_places (id, url, visit_count, rev_host, guid, url_hash)
VALUES (1, 'https://null.test', 1, '', 'g1', 0)`,
)
got, err := extractHistories(path)
require.NoError(t, err)
require.Len(t, got, 1)
assert.Equal(t, "https://null.test", got[0].URL)
assert.Equal(t, "", got[0].Title)
}
+70
View File
@@ -0,0 +1,70 @@
package firefox
import (
"encoding/base64"
"fmt"
"os"
"sort"
"github.com/tidwall/gjson"
"github.com/moond4rk/hackbrowserdata/crypto"
"github.com/moond4rk/hackbrowserdata/log"
"github.com/moond4rk/hackbrowserdata/types"
"github.com/moond4rk/hackbrowserdata/utils/typeutil"
)
// decryptPBE combines base64 decode + ASN1 PBE parse + decrypt into one call.
func decryptPBE(encoded string, masterKey []byte) ([]byte, error) {
raw, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, fmt.Errorf("base64 decode: %w", err)
}
pbe, err := crypto.NewASN1PBE(raw)
if err != nil {
return nil, fmt.Errorf("parse asn1 pbe: %w", err)
}
plaintext, err := pbe.Decrypt(masterKey)
if err != nil {
return nil, fmt.Errorf("decrypt: %w", err)
}
return plaintext, nil
}
func extractPasswords(masterKey []byte, path string) ([]types.LoginEntry, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var logins []types.LoginEntry
for _, v := range gjson.GetBytes(data, "logins").Array() {
user, err := decryptPBE(v.Get("encryptedUsername").String(), masterKey)
if err != nil {
log.Debugf("decrypt username: %v", err)
continue
}
pwd, err := decryptPBE(v.Get("encryptedPassword").String(), masterKey)
if err != nil {
log.Debugf("decrypt password: %v", err)
continue
}
url := v.Get("formSubmitURL").String()
if url == "" {
url = v.Get("hostname").String()
}
logins = append(logins, types.LoginEntry{
URL: url,
Username: string(user),
Password: string(pwd),
CreatedAt: typeutil.TimeStamp(v.Get("timeCreated").Int() / 1000),
})
}
sort.Slice(logins, func(i, j int) bool {
return logins[i].CreatedAt.After(logins[j].CreatedAt)
})
return logins, nil
}
+106
View File
@@ -0,0 +1,106 @@
package firefox
import (
"bytes"
"encoding/base64"
"encoding/hex"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// These values are from crypto/asn1pbe_test.go loginPBETestCases.
// loginPBE hex decrypts to "Hello, World!" with globalSalt = "moond4rk" * 3.
const loginPBEHex = "303b0410f8000000000000000000000000000001301506092a864886f70d010503040830313233343536370410fe968b6565149114ea688defd6683e45"
var testGlobalSalt = bytes.Repeat([]byte("moond4rk"), 3) // 24 bytes
func loginPBEBase64(t *testing.T) string {
t.Helper()
raw, err := hex.DecodeString(loginPBEHex)
require.NoError(t, err)
return base64.StdEncoding.EncodeToString(raw)
}
func TestExtractPasswords(t *testing.T) {
encB64 := loginPBEBase64(t)
// Construct a logins.json with known encrypted username/password
json := fmt.Sprintf(`{
"logins": [
{
"hostname": "https://example.com",
"formSubmitURL": "https://example.com/login",
"encryptedUsername": "%s",
"encryptedPassword": "%s",
"timeCreated": 1700000000000
}
]
}`, encB64, encB64)
path := createTestJSON(t, "logins.json", json)
got, err := extractPasswords(testGlobalSalt, path)
require.NoError(t, err)
require.Len(t, got, 1)
// Both username and password decrypt to "Hello, World!"
assert.Equal(t, "Hello, World!", got[0].Username)
assert.Equal(t, "Hello, World!", got[0].Password)
assert.Equal(t, "https://example.com/login", got[0].URL)
assert.False(t, got[0].CreatedAt.IsZero())
}
func TestExtractPasswords_FormSubmitURLFallback(t *testing.T) {
encB64 := loginPBEBase64(t)
// When formSubmitURL is empty, should fall back to hostname
json := fmt.Sprintf(`{
"logins": [
{
"hostname": "https://fallback.com",
"formSubmitURL": "",
"encryptedUsername": "%s",
"encryptedPassword": "%s",
"timeCreated": 1700000000000
}
]
}`, encB64, encB64)
path := createTestJSON(t, "logins.json", json)
got, err := extractPasswords(testGlobalSalt, path)
require.NoError(t, err)
require.Len(t, got, 1)
assert.Equal(t, "https://fallback.com", got[0].URL)
}
func TestExtractPasswords_InvalidBase64Skipped(t *testing.T) {
// Invalid base64 in encryptedUsername — entry should be skipped
json := `{
"logins": [
{
"hostname": "https://bad.com",
"encryptedUsername": "not-valid-base64!!!",
"encryptedPassword": "also-bad",
"timeCreated": 1700000000000
}
]
}`
path := createTestJSON(t, "logins.json", json)
got, err := extractPasswords(testGlobalSalt, path)
require.NoError(t, err)
assert.Empty(t, got) // skipped, not error
}
func TestExtractPasswords_EmptyLogins(t *testing.T) {
path := createTestJSON(t, "logins.json", `{"logins": []}`)
got, err := extractPasswords(testGlobalSalt, path)
require.NoError(t, err)
assert.Empty(t, got)
}
+44
View File
@@ -0,0 +1,44 @@
package firefox
import (
"database/sql"
"fmt"
"strings"
"github.com/moond4rk/hackbrowserdata/types"
"github.com/moond4rk/hackbrowserdata/utils/sqliteutil"
"github.com/moond4rk/hackbrowserdata/utils/typeutil"
)
const firefoxLocalStorageQuery = `SELECT originKey, key, value FROM webappsstore2`
func extractLocalStorage(path string) ([]types.StorageEntry, error) {
return sqliteutil.QueryRows(path, true, firefoxLocalStorageQuery,
func(rows *sql.Rows) (types.StorageEntry, error) {
var originKey, key, value string
if err := rows.Scan(&originKey, &key, &value); err != nil {
return types.StorageEntry{}, err
}
return types.StorageEntry{
URL: parseOriginKey(originKey),
Key: key,
Value: value,
}, nil
})
}
// parseOriginKey converts Firefox's reversed origin format to a URL.
// Example: "moc.buhtig.:https:443" → "https://github.com:443"
func parseOriginKey(originKey string) string {
parts := strings.SplitN(originKey, ":", 3)
if len(parts) < 2 {
return originKey
}
host := string(typeutil.Reverse([]byte(parts[0])))
host = strings.TrimPrefix(host, ".")
scheme := parts[1]
if len(parts) == 3 {
return fmt.Sprintf("%s://%s:%s", scheme, host, parts[2])
}
return fmt.Sprintf("%s://%s", scheme, host)
}
+65
View File
@@ -0,0 +1,65 @@
package firefox
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExtractLocalStorage(t *testing.T) {
path := createTestDB(t, "webappsstore.sqlite", []string{webappsstore2Schema},
insertWebappsstore("moc.buhtig.:https:443", "theme", "dark"),
insertWebappsstore("moc.buhtig.:https:443", "lang", "en"),
insertWebappsstore("moc.elpmaxe.:http:8080", "token", "abc123"),
)
got, err := extractLocalStorage(path)
require.NoError(t, err)
require.Len(t, got, 3)
// Verify field mapping by collecting into lookup
byKey := map[string]string{}
for _, entry := range got {
byKey[entry.URL+"/"+entry.Key] = entry.Value
}
assert.Equal(t, "dark", byKey["https://github.com:443/theme"])
assert.Equal(t, "en", byKey["https://github.com:443/lang"])
assert.Equal(t, "abc123", byKey["http://example.com:8080/token"])
}
func TestParseOriginKey(t *testing.T) {
tests := []struct {
name string
originKey string
want string
}{
{
name: "https with port",
originKey: "moc.buhtig.:https:443",
want: "https://github.com:443",
},
{
name: "http with non-standard port",
originKey: "moc.elpmaxe.:http:8080",
want: "http://example.com:8080",
},
{
name: "no port",
originKey: "moc.elpmaxe.:https",
want: "https://example.com",
},
{
name: "invalid format",
originKey: "something",
want: "something",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := parseOriginKey(tt.originKey)
assert.Equal(t, tt.want, got)
})
}
}
+167
View File
@@ -0,0 +1,167 @@
package firefox
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
_ "modernc.org/sqlite"
)
// ---------------------------------------------------------------------------
// Real Firefox table schemas — extracted via `sqlite3 <db> ".schema <table>"`.
// ---------------------------------------------------------------------------
const mozCookiesSchema = `CREATE TABLE moz_cookies (
id INTEGER PRIMARY KEY,
originAttributes TEXT NOT NULL DEFAULT '',
name TEXT,
value TEXT,
host TEXT,
path TEXT,
expiry INTEGER,
lastAccessed INTEGER,
creationTime INTEGER,
isSecure INTEGER,
isHttpOnly INTEGER,
inBrowserElement INTEGER DEFAULT 0,
sameSite INTEGER DEFAULT 0,
rawSameSite INTEGER DEFAULT 0,
schemeMap INTEGER DEFAULT 0,
isPartitionedAttributeSet INTEGER DEFAULT 0,
CONSTRAINT moz_uniqueid UNIQUE (name, host, path, originAttributes)
)`
const mozPlacesSchema = `CREATE TABLE moz_places (
id INTEGER PRIMARY KEY,
url LONGVARCHAR,
title LONGVARCHAR,
rev_host LONGVARCHAR,
visit_count INTEGER DEFAULT 0,
hidden INTEGER DEFAULT 0 NOT NULL,
typed INTEGER DEFAULT 0 NOT NULL,
frecency INTEGER DEFAULT -1 NOT NULL,
last_visit_date INTEGER,
guid TEXT,
foreign_count INTEGER DEFAULT 0 NOT NULL,
url_hash INTEGER DEFAULT 0 NOT NULL,
description TEXT,
preview_image_url TEXT,
site_name TEXT,
origin_id INTEGER,
recalc_frecency INTEGER NOT NULL DEFAULT 0,
alt_frecency INTEGER,
recalc_alt_frecency INTEGER NOT NULL DEFAULT 0
)`
const mozBookmarksSchema = `CREATE TABLE moz_bookmarks (
id INTEGER PRIMARY KEY,
type INTEGER,
fk INTEGER DEFAULT NULL,
parent INTEGER,
position INTEGER,
title LONGVARCHAR,
keyword_id INTEGER,
folder_type TEXT,
dateAdded INTEGER,
lastModified INTEGER,
guid TEXT,
syncStatus INTEGER NOT NULL DEFAULT 0,
syncChangeCounter INTEGER NOT NULL DEFAULT 1
)`
const mozAnnosSchema = `CREATE TABLE moz_annos (
id INTEGER PRIMARY KEY,
place_id INTEGER NOT NULL,
anno_attribute_id INTEGER,
content LONGVARCHAR,
flags INTEGER DEFAULT 0,
expiration INTEGER DEFAULT 0,
type INTEGER DEFAULT 0,
dateAdded INTEGER DEFAULT 0,
lastModified INTEGER DEFAULT 0
)`
const webappsstore2Schema = `CREATE TABLE webappsstore2 (
originAttributes TEXT,
originKey TEXT,
scope TEXT,
key TEXT,
value TEXT
)`
// ---------------------------------------------------------------------------
// INSERT helpers
// ---------------------------------------------------------------------------
func insertMozCookie(name, value, host, path string, creationTime, expiry int64, isSecure, isHTTPOnly int) string {
return fmt.Sprintf(
`INSERT INTO moz_cookies (name, value, host, path, creationTime, expiry, isSecure, isHttpOnly, lastAccessed)
VALUES ('%s', '%s', '%s', '%s', %d, %d, %d, %d, %d)`,
name, value, host, path, creationTime, expiry, isSecure, isHTTPOnly, creationTime,
)
}
func insertMozPlace(id int, url, title string, visitCount int, lastVisitDate int64) string {
return fmt.Sprintf(
`INSERT INTO moz_places (id, url, title, visit_count, last_visit_date, rev_host, guid, url_hash)
VALUES (%d, '%s', '%s', %d, %d, '', 'guid-%d', 0)`,
id, url, title, visitCount, lastVisitDate, id,
)
}
func insertMozBookmark(id, fk, bookmarkType int, title string, dateAdded int64) string {
return fmt.Sprintf(
`INSERT INTO moz_bookmarks (id, type, fk, parent, position, title, dateAdded, lastModified, guid)
VALUES (%d, %d, %d, 0, 0, '%s', %d, %d, 'bm-guid-%d')`,
id, bookmarkType, fk, title, dateAdded, dateAdded, id,
)
}
func insertMozAnno(placeID int, content string, dateAdded int64) string {
return fmt.Sprintf(
`INSERT INTO moz_annos (place_id, anno_attribute_id, content, dateAdded, lastModified)
VALUES (%d, 1, '%s', %d, %d)`,
placeID, content, dateAdded, dateAdded,
)
}
func insertWebappsstore(originKey, key, value string) string {
return fmt.Sprintf(
`INSERT INTO webappsstore2 (originAttributes, originKey, scope, key, value)
VALUES ('', '%s', '', '%s', '%s')`,
originKey, key, value,
)
}
// ---------------------------------------------------------------------------
// Test fixture builders
// ---------------------------------------------------------------------------
func createTestDB(t *testing.T, name string, schemas []string, inserts ...string) string {
t.Helper()
path := filepath.Join(t.TempDir(), name)
db, err := sql.Open("sqlite", path)
require.NoError(t, err)
defer db.Close()
for _, schema := range schemas {
_, err = db.Exec(schema)
require.NoError(t, err)
}
for _, stmt := range inserts {
_, err = db.Exec(stmt)
require.NoError(t, err)
}
return path
}
func createTestJSON(t *testing.T, name, content string) string {
t.Helper()
path := filepath.Join(t.TempDir(), name)
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
return path
}
+19 -12
View File
@@ -12,14 +12,16 @@ type LoginEntry struct {
// CookieEntry represents a single browser cookie.
type CookieEntry struct {
Host string `json:"host" csv:"host"`
Path string `json:"path" csv:"path"`
Name string `json:"name" csv:"name"`
Value string `json:"value" csv:"value"`
IsSecure bool `json:"is_secure" csv:"is_secure"`
IsHTTPOnly bool `json:"is_http_only" csv:"is_http_only"`
ExpireAt time.Time `json:"expire_at" csv:"expire_at"`
CreatedAt time.Time `json:"created_at" csv:"created_at"`
Host string `json:"host" csv:"host"`
Path string `json:"path" csv:"path"`
Name string `json:"name" csv:"name"`
Value string `json:"value" csv:"value"`
IsSecure bool `json:"is_secure" csv:"is_secure"`
IsHTTPOnly bool `json:"is_http_only" csv:"is_http_only"`
HasExpire bool `json:"has_expire" csv:"has_expire"`
IsPersistent bool `json:"is_persistent" csv:"is_persistent"`
ExpireAt time.Time `json:"expire_at" csv:"expire_at"`
CreatedAt time.Time `json:"created_at" csv:"created_at"`
}
// BookmarkEntry represents a single browser bookmark.
@@ -42,6 +44,7 @@ type HistoryEntry struct {
type DownloadEntry struct {
URL string `json:"url" csv:"url"`
TargetPath string `json:"target_path" csv:"target_path"`
MimeType string `json:"mime_type" csv:"mime_type"`
TotalBytes int64 `json:"total_bytes" csv:"total_bytes"`
StartTime time.Time `json:"start_time" csv:"start_time"`
EndTime time.Time `json:"end_time" csv:"end_time"`
@@ -53,6 +56,8 @@ type CreditCardEntry struct {
Number string `json:"number" csv:"number"`
ExpMonth string `json:"exp_month" csv:"exp_month"`
ExpYear string `json:"exp_year" csv:"exp_year"`
NickName string `json:"nick_name" csv:"nick_name"`
Address string `json:"address" csv:"address"`
}
// StorageEntry represents a single key-value pair from local or session storage.
@@ -64,8 +69,10 @@ type StorageEntry struct {
// ExtensionEntry represents a single browser extension.
type ExtensionEntry struct {
Name string `json:"name" csv:"name"`
ID string `json:"id" csv:"id"`
Description string `json:"description" csv:"description"`
Version string `json:"version" csv:"version"`
Name string `json:"name" csv:"name"`
ID string `json:"id" csv:"id"`
Description string `json:"description" csv:"description"`
Version string `json:"version" csv:"version"`
HomepageURL string `json:"homepage_url" csv:"homepage_url"`
Enabled bool `json:"enabled" csv:"enabled"`
}