From e954e847c705121acf57bfe3ca0cee507a4e68a5 Mon Sep 17 00:00:00 2001 From: zarzet <42882290+zarzet@users.noreply.github.com> Date: Sat, 31 Oct 2026 22:32:20 +0700 Subject: [PATCH] fix(player): read saved sessions longer than 512 KB in pieces Android's sqflite reads each result row through a cursor window of about 2 MB, so a saved queue whose media_json passes it (roughly 4,000 to 6,000 songs) could not be read back: cold start, Android Auto start and Resume did not restore it, the Home mixes hint failed, and Library cover pruning was skipped while such a session existed. Readers now select the text only when it fits 512 KB, as one read as before. Longer text is read in one transaction as 512 KB pieces of substr(CAST(text AS BLOB)), joined, and decoded (UTF-8 or the database's UTF-16) together with its JSON on another isolate. This covers the active session, the Resume archive and the cover reference scan. The stored data is unchanged, so every build still reads and writes it. --- lib/services/app_state_database.dart | 18 ++- lib/services/library_cover_snapshot.dart | 40 +++-- lib/services/playback_resume_store.dart | 14 +- lib/services/sqlite_long_text.dart | 137 ++++++++++++++++ test/playback_session_storage_test.dart | 194 +++++++++++++++++++++++ 5 files changed, 383 insertions(+), 20 deletions(-) create mode 100644 lib/services/sqlite_long_text.dart diff --git a/lib/services/app_state_database.dart b/lib/services/app_state_database.dart index 6fef2b9e6..06f66ab32 100644 --- a/lib/services/app_state_database.dart +++ b/lib/services/app_state_database.dart @@ -6,6 +6,7 @@ import 'package:sqflite/sqflite.dart'; import 'package:mornye/services/download_queue_codec.dart'; import 'package:mornye/services/playback_resume_store.dart'; import 'package:mornye/services/sqlite_helpers.dart' as sqlite; +import 'package:mornye/services/sqlite_long_text.dart'; import 'package:mornye/utils/logger.dart'; final _log = AppLogger('AppStateDb'); @@ -405,14 +406,17 @@ class AppStateDatabase { } Future?> getPlaybackSession() async { - final db = await database; - final rows = await db.query(_playbackSessionTable, limit: 1); - if (rows.isEmpty) return null; - final row = rows.first; - final raw = row['media_json'] as String?; - if (raw == null || raw.isEmpty) return null; + final stored = await readJsonTextRow( + await database, + table: _playbackSessionTable, + jsonColumn: 'media_json', + columns: const ['current_index', 'position_ms', 'shuffle', 'repeat_mode'], + ); + if (stored == null) return null; + final row = stored.row; + if ((row['media_json_bytes'] as int? ?? 0) == 0) return null; try { - final decoded = jsonDecode(raw); + final decoded = stored.json(); if (decoded is! List) return null; return { 'version': 2, diff --git a/lib/services/library_cover_snapshot.dart b/lib/services/library_cover_snapshot.dart index e537dc062..5a6182c3e 100644 --- a/lib/services/library_cover_snapshot.dart +++ b/lib/services/library_cover_snapshot.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:sqflite/sqflite.dart'; +import 'package:mornye/services/sqlite_long_text.dart'; import 'package:mornye/utils/logger.dart'; var _snapshotSequence = 0; @@ -18,6 +19,22 @@ typedef LibraryCoverReferenceColumn = ({ final _storedCoverName = RegExp(r'''library_covers/([^"'\s\\?#]+)'''); +/// The Library cover file names a stored text column value names. +List _storedCoverNames(String stored) { + final value = stored.replaceAll(r'\/', '/'); + final names = []; + for (final match in _storedCoverName.allMatches(value)) { + var name = match.group(1)!; + try { + name = Uri.decodeComponent(name); + } catch (_) { + // A literal percent sign names the file as stored. + } + names.add(name); + } + return names; +} + /// Files under [directory] that [columns] name, as paths in [directory]: a /// stored `file://` URI or a path from an older app container still matches. /// A failed read throws, so incomplete references never permit a deletion. @@ -30,20 +47,25 @@ Future> readStoredLibraryCoverPaths( var after = 0; while (true) { final page = await database.rawQuery( - 'SELECT rowid AS stored_rowid, $column AS stored_value FROM $table ' + 'SELECT rowid AS stored_rowid, ' + '${selectShortText(column, 'stored_value')} FROM $table ' "WHERE rowid > ? AND instr($column, 'library_covers') > 0 " 'ORDER BY rowid LIMIT 256', [after], ); for (final row in page) { - final value = (row['stored_value'] as String).replaceAll(r'\/', '/'); - for (final match in _storedCoverName.allMatches(value)) { - var name = match.group(1)!; - try { - name = Uri.decodeComponent(name); - } catch (_) { - // A literal percent sign names the file as stored. - } + final value = row['stored_value'] as String?; + final names = value != null + ? _storedCoverNames(value) + : (await readLongTextRow( + database, + table: table, + rowid: row['stored_rowid'] as int, + textColumn: column, + decode: _storedCoverNames, + ))?.$2 ?? + const []; + for (final name in names) { paths.add('${directory.path}/$name'); } } diff --git a/lib/services/playback_resume_store.dart b/lib/services/playback_resume_store.dart index bc71f58f9..e231279fe 100644 --- a/lib/services/playback_resume_store.dart +++ b/lib/services/playback_resume_store.dart @@ -1,6 +1,7 @@ import 'dart:convert'; import 'package:sqflite/sqflite.dart'; +import 'package:mornye/services/sqlite_long_text.dart'; /// A dismissed player's last session, separate from the active player state. class PlaybackResumeStore { @@ -55,12 +56,17 @@ class PlaybackResumeStore { } Future?> session() async { - final rows = await database.query(_table, limit: 1); - if (rows.isEmpty) return null; - final row = rows.first; + final stored = await readJsonTextRow( + database, + table: _table, + jsonColumn: 'media_json', + columns: const ['current_index', 'position_ms', 'shuffle', 'repeat_mode'], + ); + if (stored == null) return null; + final row = stored.row; return { 'version': 2, - 'media': jsonDecode(row['media_json'] as String), + 'media': stored.json(), 'index': row['current_index'], 'positionMs': row['position_ms'], 'shuffle': row['shuffle'] == 1, diff --git a/lib/services/sqlite_long_text.dart b/lib/services/sqlite_long_text.dart new file mode 100644 index 000000000..b6d24d71a --- /dev/null +++ b/lib/services/sqlite_long_text.dart @@ -0,0 +1,137 @@ +import 'dart:convert'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:sqflite/sqflite.dart'; + +/// Android reads query results row by row through a cursor window of about +/// 2 MB (less on some devices) and fails on a row that does not fit. Text +/// that can grow past this many bytes is read in pieces of this size. +const longTextPieceBytes = 524288; + +/// Selects [column] as [alias] only when it fits one piece (else null), and +/// its size in bytes as `_bytes`, so no row returned is too long. +String selectShortText(String column, String alias) => + 'CASE WHEN length(CAST($column AS BLOB)) <= $longTextPieceBytes ' + 'THEN $column END AS $alias, ' + 'length(CAST($column AS BLOB)) AS ${alias}_bytes'; + +/// The first row of [table] with [columns] and `_bytes`, and +/// [json], which gives [jsonColumn] decoded (null for NULL) or throws its +/// decode error. Text that fits one piece comes in the same single read and +/// decodes when [json] is called; longer text is read in pieces and decoded +/// on another isolate. +Future<({Map row, Object? Function() json})?> readJsonTextRow( + DatabaseExecutor db, { + required String table, + required String jsonColumn, + required List columns, +}) async { + final rows = await db.rawQuery( + 'SELECT ${[...columns, 'rowid AS long_text_rowid', selectShortText(jsonColumn, jsonColumn)].join(', ')} ' + 'FROM $table LIMIT 1', + ); + if (rows.isEmpty) return null; + final row = rows.single; + final text = row[jsonColumn] as String?; + if (text != null || (row['${jsonColumn}_bytes'] as int? ?? 0) == 0) { + return (row: row, json: () => text == null ? null : jsonDecode(text)); + } + final long = await readLongTextRow( + db, + table: table, + rowid: row['long_text_rowid'] as int, + textColumn: jsonColumn, + columns: [ + ...columns, + 'length(CAST($jsonColumn AS BLOB)) AS ${jsonColumn}_bytes', + ], + decode: _tryJsonDecode, + ); + if (long == null) return null; + final (longRow, (value, error)) = long; + return ( + row: longRow, + json: () { + if (error != null) throw error; + return value; + }, + ); +} + +(Object?, FormatException?) _tryJsonDecode(String text) { + try { + return (jsonDecode(text), null); + } on FormatException catch (error) { + return (null, error); + } +} + +/// Reads [columns] and the long [textColumn] of the row of [table] with +/// [rowid] in one transaction, the text in pieces, and returns the columns +/// with [decode] of the text, decoded on another isolate. Null when the row +/// is gone. +Future<(Map, T)?> readLongTextRow( + DatabaseExecutor db, { + required String table, + required int rowid, + required String textColumn, + List columns = const [], + required T Function(String text) decode, +}) async { + Future<(Map, Uint8List, String)?> read( + DatabaseExecutor txn, + ) async { + final rows = await txn.rawQuery( + 'SELECT ${[...columns, 'length(CAST($textColumn AS BLOB)) AS long_text_bytes'].join(', ')} ' + 'FROM $table WHERE rowid = ?', + [rowid], + ); + if (rows.isEmpty) return null; + final length = rows.single['long_text_bytes'] as int? ?? 0; + final text = BytesBuilder(copy: false); + for (var start = 1; start <= length; start += longTextPieceBytes) { + final piece = await txn.rawQuery( + 'SELECT substr(CAST($textColumn AS BLOB), ?, $longTextPieceBytes) ' + 'AS piece FROM $table WHERE rowid = ?', + [start, rowid], + ); + text.add(piece.single['piece'] as Uint8List); + } + final encoding = await txn.rawQuery('PRAGMA encoding'); + return ( + rows.single, + text.takeBytes(), + encoding.single.values.single as String, + ); + } + + final stored = db is Database ? await db.transaction(read) : await read(db); + if (stored == null) return null; + final (row, bytes, encoding) = stored; + return (row, await _decodeElsewhere(bytes, encoding, decode)); +} + +Future _decodeElsewhere( + Uint8List bytes, + String encoding, + T Function(String text) decode, +) async { + T run() => decode(_decodeText(bytes, encoding)); + try { + return await Isolate.run(run); + } on IsolateSpawnException { + return run(); + } +} + +/// [bytes] of text read as a BLOB, which SQLite gives in the database +/// encoding. +String _decodeText(Uint8List bytes, String encoding) { + if (encoding == 'UTF-8') return utf8.decode(bytes, allowMalformed: true); + final data = ByteData.sublistView(bytes); + final endian = encoding == 'UTF-16be' ? Endian.big : Endian.little; + return String.fromCharCodes([ + for (var i = 0; i + 1 < bytes.length; i += 2) data.getUint16(i, endian), + ]); +} diff --git a/test/playback_session_storage_test.dart b/test/playback_session_storage_test.dart index 42eacbea2..11bfcf875 100644 --- a/test/playback_session_storage_test.dart +++ b/test/playback_session_storage_test.dart @@ -8,9 +8,12 @@ import 'package:sqflite/sqflite.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart' show databaseFactoryFfi, sqfliteFfiInit; import 'package:mornye/services/app_state_database.dart'; +import 'package:mornye/services/library_cover_snapshot.dart'; import 'package:mornye/services/playable_media.dart'; +import 'package:mornye/services/playback_resume_store.dart'; import 'package:mornye/services/playback_session_restore.dart'; import 'package:mornye/services/playback_session_writer.dart'; +import 'package:mornye/services/sqlite_long_text.dart'; PlayableMedia _song(int i) => PlayableMedia( id: 'song-$i', @@ -200,4 +203,195 @@ void main() { expect(row['type'], 'text'); expect(row['media_json'], _legacyJson(songs)); }); + + test('a session over 2 MB reads back whole in pieces', () async { + final songs = [ + for (var i = 0; i < 10000; i++) + i % 1000 == 999 + ? PlayableMedia( + id: 'covered-$i', + source: '/Music/$i.flac', + title: 'Covered $i', + artist: 'Artist', + artUri: 'file:///data/library_covers/cover_$i.jpg', + ) + : _song(i), + ]; + final errors = []; + await PlaybackSessionWriter(database: store, onError: errors.add).persist( + queue: () => (media: List.of(songs), originalIndices: null), + index: 9999, + position: const Duration(seconds: 61), + shuffle: false, + repeatMode: 'all', + ); + expect(errors, isEmpty); + final legacy = _legacyJson(songs); + expect(utf8.encode(legacy).length, greaterThan(2 * 1024 * 1024)); + final covers = { + for (var i = 999; i < songs.length; i += 1000) + '${root.path}/cover_$i.jpg', + }; + + final watched = _Watched(db); + expect( + await readStoredLibraryCoverPaths(root, [ + (database: watched, table: 'playback_session', column: 'media_json'), + ]), + covers, + ); + final session = await store.getPlaybackSession(); + expect(session?['media'], jsonDecode(legacy)); + expect(session?['index'], 9999); + expect(session?['positionMs'], 61000); + + await store.archivePlaybackSession(songs.last.toJson()); + final resumed = await PlaybackResumeStore(watched).session(); + expect(resumed?['media'], jsonDecode(legacy)); + expect(resumed?['index'], 9999); + expect(resumed?['repeat'], 'all'); + expect( + await readStoredLibraryCoverPaths(root, [ + (database: watched, table: 'playback_resume', column: 'media_json'), + ]), + covers, + ); + expect(watched.largest, lessThanOrEqualTo(longTextPieceBytes)); + expect(watched.pieces, greaterThan(8)); + }); + + test('pieces join to the same text at every boundary', () async { + for (final encoding in ['UTF-8', 'UTF-16le', 'UTF-16be']) { + final other = await databaseFactoryFfi.openDatabase( + inMemoryDatabasePath, + options: OpenDatabaseOptions(singleInstance: false), + ); + await other.execute("PRAGMA encoding = '$encoding'"); + await other.execute('CREATE TABLE t (id INTEGER PRIMARY KEY, body TEXT)'); + final watched = _Watched(other); + // Byte sizes around one and two pieces, with two-, three- and + // four-byte characters split by a piece boundary. + final bodies = [ + '', + 'a' * longTextPieceBytes, + 'a' * (longTextPieceBytes + 1), + 'a' * (2 * longTextPieceBytes), + '${'a' * (longTextPieceBytes - 1)}é${'b' * 10}', + '${'a' * (longTextPieceBytes - 2)}坂${'b' * 10}', + '${'a' * (longTextPieceBytes - 3)}🎧${'b' * longTextPieceBytes}', + ]; + for (final body in bodies) { + final id = await other.insert('t', {'body': body}); + final row = (await watched.rawQuery( + 'SELECT ${selectShortText('body', 'body')} FROM t WHERE id = ?', + [id], + )).single; + final bytes = + (await other.rawQuery( + 'SELECT length(CAST(body AS BLOB)) AS n FROM t WHERE id = ?', + [id], + )).single['n']! + as int; + expect(row['body_bytes'], bytes, reason: encoding); + expect( + row['body'], + bytes <= longTextPieceBytes ? body : isNull, + reason: '$encoding ${body.length}', + ); + final read = await readLongTextRow( + watched, + table: 't', + rowid: id, + textColumn: 'body', + decode: _sameText, + ); + expect(read?.$2, body, reason: '$encoding ${body.length}'); + } + expect(watched.largest, lessThanOrEqualTo(longTextPieceBytes)); + await other.close(); + } + }); + + test('a long session that does not decode throws when read', () async { + final text = '[${'"x",' * longTextPieceBytes}'; + await db.insert('playback_session', { + 'id': 1, + 'media_json': text, + 'current_index': 0, + 'updated_at': '2026-10-31T00:00:00.000', + }); + final stored = await readJsonTextRow( + db, + table: 'playback_session', + jsonColumn: 'media_json', + columns: const ['current_index'], + ); + expect(stored?.row['media_json_bytes'], utf8.encode(text).length); + expect(stored?.json, throwsFormatException); + expect(await store.getPlaybackSession(), isNull); + }); +} + +String _sameText(String text) => text; + +/// Records the largest value any read returns and how many long-text +/// pieces it read, through the database and its transactions. +class _Watched implements Database { + _Watched(this._db); + + final Database _db; + int largest = 0; + int pieces = 0; + + List> watch( + String sql, + List> rows, + ) { + if (sql.contains('substr(')) pieces++; + for (final row in rows) { + for (final value in row.values) { + final size = switch (value) { + final String text => utf8.encode(text).length, + final Uint8List bytes => bytes.length, + _ => 0, + }; + if (size > largest) largest = size; + } + } + return rows; + } + + @override + Future>> rawQuery( + String sql, [ + List? arguments, + ]) async => watch(sql, await _db.rawQuery(sql, arguments)); + + @override + Future transaction( + Future Function(Transaction txn) action, { + bool? exclusive, + }) => _db.transaction( + (txn) => action(_WatchedTransaction(txn, this)), + exclusive: exclusive, + ); + + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); +} + +class _WatchedTransaction implements Transaction { + _WatchedTransaction(this._txn, this._owner); + + final Transaction _txn; + final _Watched _owner; + + @override + Future>> rawQuery( + String sql, [ + List? arguments, + ]) async => _owner.watch(sql, await _txn.rawQuery(sql, arguments)); + + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); }