diff --git a/lib/services/app_state_database.dart b/lib/services/app_state_database.dart index 6fef2b9e6..06f66ab32 100644 --- a/lib/services/app_state_database.dart +++ b/lib/services/app_state_database.dart @@ -6,6 +6,7 @@ import 'package:sqflite/sqflite.dart'; import 'package:mornye/services/download_queue_codec.dart'; import 'package:mornye/services/playback_resume_store.dart'; import 'package:mornye/services/sqlite_helpers.dart' as sqlite; +import 'package:mornye/services/sqlite_long_text.dart'; import 'package:mornye/utils/logger.dart'; final _log = AppLogger('AppStateDb'); @@ -405,14 +406,17 @@ class AppStateDatabase { } Future?> getPlaybackSession() async { - final db = await database; - final rows = await db.query(_playbackSessionTable, limit: 1); - if (rows.isEmpty) return null; - final row = rows.first; - final raw = row['media_json'] as String?; - if (raw == null || raw.isEmpty) return null; + final stored = await readJsonTextRow( + await database, + table: _playbackSessionTable, + jsonColumn: 'media_json', + columns: const ['current_index', 'position_ms', 'shuffle', 'repeat_mode'], + ); + if (stored == null) return null; + final row = stored.row; + if ((row['media_json_bytes'] as int? ?? 0) == 0) return null; try { - final decoded = jsonDecode(raw); + final decoded = stored.json(); if (decoded is! List) return null; return { 'version': 2, diff --git a/lib/services/library_cover_snapshot.dart b/lib/services/library_cover_snapshot.dart index e537dc062..5a6182c3e 100644 --- a/lib/services/library_cover_snapshot.dart +++ b/lib/services/library_cover_snapshot.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:sqflite/sqflite.dart'; +import 'package:mornye/services/sqlite_long_text.dart'; import 'package:mornye/utils/logger.dart'; var _snapshotSequence = 0; @@ -18,6 +19,22 @@ typedef LibraryCoverReferenceColumn = ({ final _storedCoverName = RegExp(r'''library_covers/([^"'\s\\?#]+)'''); +/// The Library cover file names a stored text column value names. +List _storedCoverNames(String stored) { + final value = stored.replaceAll(r'\/', '/'); + final names = []; + for (final match in _storedCoverName.allMatches(value)) { + var name = match.group(1)!; + try { + name = Uri.decodeComponent(name); + } catch (_) { + // A literal percent sign names the file as stored. + } + names.add(name); + } + return names; +} + /// Files under [directory] that [columns] name, as paths in [directory]: a /// stored `file://` URI or a path from an older app container still matches. /// A failed read throws, so incomplete references never permit a deletion. @@ -30,20 +47,25 @@ Future> readStoredLibraryCoverPaths( var after = 0; while (true) { final page = await database.rawQuery( - 'SELECT rowid AS stored_rowid, $column AS stored_value FROM $table ' + 'SELECT rowid AS stored_rowid, ' + '${selectShortText(column, 'stored_value')} FROM $table ' "WHERE rowid > ? AND instr($column, 'library_covers') > 0 " 'ORDER BY rowid LIMIT 256', [after], ); for (final row in page) { - final value = (row['stored_value'] as String).replaceAll(r'\/', '/'); - for (final match in _storedCoverName.allMatches(value)) { - var name = match.group(1)!; - try { - name = Uri.decodeComponent(name); - } catch (_) { - // A literal percent sign names the file as stored. - } + final value = row['stored_value'] as String?; + final names = value != null + ? _storedCoverNames(value) + : (await readLongTextRow( + database, + table: table, + rowid: row['stored_rowid'] as int, + textColumn: column, + decode: _storedCoverNames, + ))?.$2 ?? + const []; + for (final name in names) { paths.add('${directory.path}/$name'); } } diff --git a/lib/services/playback_resume_store.dart b/lib/services/playback_resume_store.dart index bc71f58f9..e231279fe 100644 --- a/lib/services/playback_resume_store.dart +++ b/lib/services/playback_resume_store.dart @@ -1,6 +1,7 @@ import 'dart:convert'; import 'package:sqflite/sqflite.dart'; +import 'package:mornye/services/sqlite_long_text.dart'; /// A dismissed player's last session, separate from the active player state. class PlaybackResumeStore { @@ -55,12 +56,17 @@ class PlaybackResumeStore { } Future?> session() async { - final rows = await database.query(_table, limit: 1); - if (rows.isEmpty) return null; - final row = rows.first; + final stored = await readJsonTextRow( + database, + table: _table, + jsonColumn: 'media_json', + columns: const ['current_index', 'position_ms', 'shuffle', 'repeat_mode'], + ); + if (stored == null) return null; + final row = stored.row; return { 'version': 2, - 'media': jsonDecode(row['media_json'] as String), + 'media': stored.json(), 'index': row['current_index'], 'positionMs': row['position_ms'], 'shuffle': row['shuffle'] == 1, diff --git a/lib/services/sqlite_long_text.dart b/lib/services/sqlite_long_text.dart new file mode 100644 index 000000000..b6d24d71a --- /dev/null +++ b/lib/services/sqlite_long_text.dart @@ -0,0 +1,137 @@ +import 'dart:convert'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:sqflite/sqflite.dart'; + +/// Android reads query results row by row through a cursor window of about +/// 2 MB (less on some devices) and fails on a row that does not fit. Text +/// that can grow past this many bytes is read in pieces of this size. +const longTextPieceBytes = 524288; + +/// Selects [column] as [alias] only when it fits one piece (else null), and +/// its size in bytes as `_bytes`, so no row returned is too long. +String selectShortText(String column, String alias) => + 'CASE WHEN length(CAST($column AS BLOB)) <= $longTextPieceBytes ' + 'THEN $column END AS $alias, ' + 'length(CAST($column AS BLOB)) AS ${alias}_bytes'; + +/// The first row of [table] with [columns] and `_bytes`, and +/// [json], which gives [jsonColumn] decoded (null for NULL) or throws its +/// decode error. Text that fits one piece comes in the same single read and +/// decodes when [json] is called; longer text is read in pieces and decoded +/// on another isolate. +Future<({Map row, Object? Function() json})?> readJsonTextRow( + DatabaseExecutor db, { + required String table, + required String jsonColumn, + required List columns, +}) async { + final rows = await db.rawQuery( + 'SELECT ${[...columns, 'rowid AS long_text_rowid', selectShortText(jsonColumn, jsonColumn)].join(', ')} ' + 'FROM $table LIMIT 1', + ); + if (rows.isEmpty) return null; + final row = rows.single; + final text = row[jsonColumn] as String?; + if (text != null || (row['${jsonColumn}_bytes'] as int? ?? 0) == 0) { + return (row: row, json: () => text == null ? null : jsonDecode(text)); + } + final long = await readLongTextRow( + db, + table: table, + rowid: row['long_text_rowid'] as int, + textColumn: jsonColumn, + columns: [ + ...columns, + 'length(CAST($jsonColumn AS BLOB)) AS ${jsonColumn}_bytes', + ], + decode: _tryJsonDecode, + ); + if (long == null) return null; + final (longRow, (value, error)) = long; + return ( + row: longRow, + json: () { + if (error != null) throw error; + return value; + }, + ); +} + +(Object?, FormatException?) _tryJsonDecode(String text) { + try { + return (jsonDecode(text), null); + } on FormatException catch (error) { + return (null, error); + } +} + +/// Reads [columns] and the long [textColumn] of the row of [table] with +/// [rowid] in one transaction, the text in pieces, and returns the columns +/// with [decode] of the text, decoded on another isolate. Null when the row +/// is gone. +Future<(Map, T)?> readLongTextRow( + DatabaseExecutor db, { + required String table, + required int rowid, + required String textColumn, + List columns = const [], + required T Function(String text) decode, +}) async { + Future<(Map, Uint8List, String)?> read( + DatabaseExecutor txn, + ) async { + final rows = await txn.rawQuery( + 'SELECT ${[...columns, 'length(CAST($textColumn AS BLOB)) AS long_text_bytes'].join(', ')} ' + 'FROM $table WHERE rowid = ?', + [rowid], + ); + if (rows.isEmpty) return null; + final length = rows.single['long_text_bytes'] as int? ?? 0; + final text = BytesBuilder(copy: false); + for (var start = 1; start <= length; start += longTextPieceBytes) { + final piece = await txn.rawQuery( + 'SELECT substr(CAST($textColumn AS BLOB), ?, $longTextPieceBytes) ' + 'AS piece FROM $table WHERE rowid = ?', + [start, rowid], + ); + text.add(piece.single['piece'] as Uint8List); + } + final encoding = await txn.rawQuery('PRAGMA encoding'); + return ( + rows.single, + text.takeBytes(), + encoding.single.values.single as String, + ); + } + + final stored = db is Database ? await db.transaction(read) : await read(db); + if (stored == null) return null; + final (row, bytes, encoding) = stored; + return (row, await _decodeElsewhere(bytes, encoding, decode)); +} + +Future _decodeElsewhere( + Uint8List bytes, + String encoding, + T Function(String text) decode, +) async { + T run() => decode(_decodeText(bytes, encoding)); + try { + return await Isolate.run(run); + } on IsolateSpawnException { + return run(); + } +} + +/// [bytes] of text read as a BLOB, which SQLite gives in the database +/// encoding. +String _decodeText(Uint8List bytes, String encoding) { + if (encoding == 'UTF-8') return utf8.decode(bytes, allowMalformed: true); + final data = ByteData.sublistView(bytes); + final endian = encoding == 'UTF-16be' ? Endian.big : Endian.little; + return String.fromCharCodes([ + for (var i = 0; i + 1 < bytes.length; i += 2) data.getUint16(i, endian), + ]); +} diff --git a/test/playback_session_storage_test.dart b/test/playback_session_storage_test.dart index 42eacbea2..11bfcf875 100644 --- a/test/playback_session_storage_test.dart +++ b/test/playback_session_storage_test.dart @@ -8,9 +8,12 @@ import 'package:sqflite/sqflite.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart' show databaseFactoryFfi, sqfliteFfiInit; import 'package:mornye/services/app_state_database.dart'; +import 'package:mornye/services/library_cover_snapshot.dart'; import 'package:mornye/services/playable_media.dart'; +import 'package:mornye/services/playback_resume_store.dart'; import 'package:mornye/services/playback_session_restore.dart'; import 'package:mornye/services/playback_session_writer.dart'; +import 'package:mornye/services/sqlite_long_text.dart'; PlayableMedia _song(int i) => PlayableMedia( id: 'song-$i', @@ -200,4 +203,195 @@ void main() { expect(row['type'], 'text'); expect(row['media_json'], _legacyJson(songs)); }); + + test('a session over 2 MB reads back whole in pieces', () async { + final songs = [ + for (var i = 0; i < 10000; i++) + i % 1000 == 999 + ? PlayableMedia( + id: 'covered-$i', + source: '/Music/$i.flac', + title: 'Covered $i', + artist: 'Artist', + artUri: 'file:///data/library_covers/cover_$i.jpg', + ) + : _song(i), + ]; + final errors = []; + await PlaybackSessionWriter(database: store, onError: errors.add).persist( + queue: () => (media: List.of(songs), originalIndices: null), + index: 9999, + position: const Duration(seconds: 61), + shuffle: false, + repeatMode: 'all', + ); + expect(errors, isEmpty); + final legacy = _legacyJson(songs); + expect(utf8.encode(legacy).length, greaterThan(2 * 1024 * 1024)); + final covers = { + for (var i = 999; i < songs.length; i += 1000) + '${root.path}/cover_$i.jpg', + }; + + final watched = _Watched(db); + expect( + await readStoredLibraryCoverPaths(root, [ + (database: watched, table: 'playback_session', column: 'media_json'), + ]), + covers, + ); + final session = await store.getPlaybackSession(); + expect(session?['media'], jsonDecode(legacy)); + expect(session?['index'], 9999); + expect(session?['positionMs'], 61000); + + await store.archivePlaybackSession(songs.last.toJson()); + final resumed = await PlaybackResumeStore(watched).session(); + expect(resumed?['media'], jsonDecode(legacy)); + expect(resumed?['index'], 9999); + expect(resumed?['repeat'], 'all'); + expect( + await readStoredLibraryCoverPaths(root, [ + (database: watched, table: 'playback_resume', column: 'media_json'), + ]), + covers, + ); + expect(watched.largest, lessThanOrEqualTo(longTextPieceBytes)); + expect(watched.pieces, greaterThan(8)); + }); + + test('pieces join to the same text at every boundary', () async { + for (final encoding in ['UTF-8', 'UTF-16le', 'UTF-16be']) { + final other = await databaseFactoryFfi.openDatabase( + inMemoryDatabasePath, + options: OpenDatabaseOptions(singleInstance: false), + ); + await other.execute("PRAGMA encoding = '$encoding'"); + await other.execute('CREATE TABLE t (id INTEGER PRIMARY KEY, body TEXT)'); + final watched = _Watched(other); + // Byte sizes around one and two pieces, with two-, three- and + // four-byte characters split by a piece boundary. + final bodies = [ + '', + 'a' * longTextPieceBytes, + 'a' * (longTextPieceBytes + 1), + 'a' * (2 * longTextPieceBytes), + '${'a' * (longTextPieceBytes - 1)}é${'b' * 10}', + '${'a' * (longTextPieceBytes - 2)}坂${'b' * 10}', + '${'a' * (longTextPieceBytes - 3)}🎧${'b' * longTextPieceBytes}', + ]; + for (final body in bodies) { + final id = await other.insert('t', {'body': body}); + final row = (await watched.rawQuery( + 'SELECT ${selectShortText('body', 'body')} FROM t WHERE id = ?', + [id], + )).single; + final bytes = + (await other.rawQuery( + 'SELECT length(CAST(body AS BLOB)) AS n FROM t WHERE id = ?', + [id], + )).single['n']! + as int; + expect(row['body_bytes'], bytes, reason: encoding); + expect( + row['body'], + bytes <= longTextPieceBytes ? body : isNull, + reason: '$encoding ${body.length}', + ); + final read = await readLongTextRow( + watched, + table: 't', + rowid: id, + textColumn: 'body', + decode: _sameText, + ); + expect(read?.$2, body, reason: '$encoding ${body.length}'); + } + expect(watched.largest, lessThanOrEqualTo(longTextPieceBytes)); + await other.close(); + } + }); + + test('a long session that does not decode throws when read', () async { + final text = '[${'"x",' * longTextPieceBytes}'; + await db.insert('playback_session', { + 'id': 1, + 'media_json': text, + 'current_index': 0, + 'updated_at': '2026-10-31T00:00:00.000', + }); + final stored = await readJsonTextRow( + db, + table: 'playback_session', + jsonColumn: 'media_json', + columns: const ['current_index'], + ); + expect(stored?.row['media_json_bytes'], utf8.encode(text).length); + expect(stored?.json, throwsFormatException); + expect(await store.getPlaybackSession(), isNull); + }); +} + +String _sameText(String text) => text; + +/// Records the largest value any read returns and how many long-text +/// pieces it read, through the database and its transactions. +class _Watched implements Database { + _Watched(this._db); + + final Database _db; + int largest = 0; + int pieces = 0; + + List> watch( + String sql, + List> rows, + ) { + if (sql.contains('substr(')) pieces++; + for (final row in rows) { + for (final value in row.values) { + final size = switch (value) { + final String text => utf8.encode(text).length, + final Uint8List bytes => bytes.length, + _ => 0, + }; + if (size > largest) largest = size; + } + } + return rows; + } + + @override + Future>> rawQuery( + String sql, [ + List? arguments, + ]) async => watch(sql, await _db.rawQuery(sql, arguments)); + + @override + Future transaction( + Future Function(Transaction txn) action, { + bool? exclusive, + }) => _db.transaction( + (txn) => action(_WatchedTransaction(txn, this)), + exclusive: exclusive, + ); + + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); +} + +class _WatchedTransaction implements Transaction { + _WatchedTransaction(this._txn, this._owner); + + final Transaction _txn; + final _Watched _owner; + + @override + Future>> rawQuery( + String sql, [ + List? arguments, + ]) async => _owner.watch(sql, await _txn.rawQuery(sql, arguments)); + + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); }