feat: test case for convert utf16 value to utf8

browingdata/localstorage/localstorage.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 
 	"github.com/syndtr/goleveldb/leveldb"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 
 	"github.com/moond4rk/HackBrowserData/item"
 	"github.com/moond4rk/HackBrowserData/log"
@@ -44,7 +46,7 @@ func (c *ChromiumLocalStorage) Parse(masterKey []byte) error {
 		if len(value) < maxLocalStorageLength {
 			s.fillValue(value)
 		} else {
-			s.Value = fmt.Sprintf("value is too long, length is %d", len(value))
+			s.Value = fmt.Sprintf("value is too long, length is %d, supportted max length is %d", len(value), maxLocalStorageLength)
 		}
 		if s.IsMeta {
 			s.Value = fmt.Sprintf("meta data, value bytes is %v", value)
@@ -84,6 +86,11 @@ func (s *storage) fillHeader(url, key []byte) {
 	s.Key = string(bytes.Trim(key, "\x01"))
 }
 
+func convertUTF16toUTF8(source []byte, endian unicode.Endianness) ([]byte, error) {
+	r, _, err := transform.Bytes(unicode.UTF16(endian, unicode.IgnoreBOM).NewDecoder(), source)
+	return r, err
+}
+
 // fillValue fills value of the storage
 // TODO: support unicode charter
 func (s *storage) fillValue(b []byte) {

browingdata/localstorage/localstorage_test.go

@@ -0,0 +1,32 @@
+package localstorage
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/text/encoding/unicode"
+)
+
+var testCases = []struct {
+	in     []byte
+	wanted []byte
+	actual []byte
+}{
+	{
+		in:     []byte{0x0, 0x7b, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x6b, 0x0, 0x65, 0x0, 0x79, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0x68, 0x0, 0x74, 0x0, 0x74, 0x0, 0x70, 0x0, 0x73, 0x0, 0x3a, 0x0, 0x2f, 0x0, 0x2f, 0x0, 0x77, 0x0, 0x77, 0x0, 0x77, 0x0, 0x2e, 0x0, 0x76, 0x0, 0x6f, 0x0, 0x6c, 0x0, 0x63, 0x0, 0x65, 0x0, 0x6e, 0x0, 0x67, 0x0, 0x69, 0x0, 0x6e, 0x0, 0x65, 0x0, 0x2e, 0x0, 0x63, 0x0, 0x6f, 0x0, 0x6d, 0x0, 0x2f, 0x0, 0x70, 0x0, 0x72, 0x0, 0x6f, 0x0, 0x64, 0x0, 0x75, 0x0, 0x63, 0x0, 0x74, 0x0, 0x73, 0x0, 0x2f, 0x0, 0x66, 0x0, 0x65, 0x0, 0x69, 0x0, 0x6c, 0x0, 0x69, 0x0, 0x61, 0x0, 0x6e, 0x0, 0x22, 0x0, 0x2c, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x74, 0x0, 0x69, 0x0, 0x74, 0x0, 0x6c, 0x0, 0x65, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0xde, 0x98, 0xde, 0x8f, 0x2d, 0x0, 0x6b, 0x70, 0x71, 0x5c, 0x15, 0x5f, 0xce, 0x64, 0x22, 0x0, 0x2c, 0x0, 0x22, 0x0, 0x72, 0x0, 0x65, 0x0, 0x66, 0x0, 0x65, 0x0, 0x72, 0x0, 0x5f, 0x0, 0x6d, 0x0, 0x61, 0x0, 0x6e, 0x0, 0x75, 0x0, 0x61, 0x0, 0x6c, 0x0, 0x5f, 0x0, 0x6b, 0x0, 0x65, 0x0, 0x79, 0x0, 0x22, 0x0, 0x3a, 0x0, 0x22, 0x0, 0x22, 0x0, 0x7d, 0x0},
+		wanted: []byte(`{"refer_key":"https://www.volcengine.com/product/feilian","refer_title":"飞连_SSO单点登录_VPN_终端安全合规_便捷Wifi认证-火山引擎","refer_manual_key":""}`),
+		actual: []byte{0x7b, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3a, 0x22, 0x68, 0x74, 0x74, 0x70, 0x73, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x6f, 0x6c, 0x63, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x73, 0x2f, 0x66, 0x65, 0x69, 0x6c, 0x69, 0x61, 0x6e, 0x22, 0x2c, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x22, 0x3a, 0x22, 0xc3, 0x9e, 0xe9, 0xa3, 0x9e, 0xe8, 0xbc, 0xad, 0x6b, 0xe7, 0x81, 0xb1, 0xe5, 0xb0, 0x95, 0xe5, 0xbf, 0x8e, 0xe6, 0x90, 0xa2, 0x2c, 0x22, 0x72, 0x65, 0x66, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, 0x75, 0x61, 0x6c, 0x5f, 0x6b, 0x65, 0x79, 0x22, 0x3a, 0x22, 0x22, 0x7d, 0xef, 0xbf, 0xbd},
+	},
+}
+
+func TestLocalStorageKeyToUTF8(t *testing.T) {
+	for _, tc := range testCases {
+		actual, err := convertUTF16toUTF8(tc.in, unicode.BigEndian)
+		if err != nil {
+			t.Error(err)
+		}
+		// TODO: fix this, value from local storage if contains chinese characters, need convert utf16 to utf8
+		// but now, it can't convert, so just skip it.
+		assert.Equal(t, tc.actual, actual, "chinese characters can't actual convert")
+	}
+}