From 0331d5ecac5d0e188ea3dc280519a63202cc5bc8 Mon Sep 17 00:00:00 2001 From: zarzet <42882290+zarzet@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:14:05 +0700 Subject: [PATCH] perf(library): skip snapshot builds for pages and failed builds A track page or a search that was the first shadow read after a write waited for the whole snapshot build: 124 to 440 ms on the host's Android fixture, against 1.6 ms for a 60-song page with the per-row check. Home's recently added shelf, the Library tab, Android Auto and Link read pages on their own, so those reads got slower. They now use the snapshot only when it is already current and otherwise keep the per-row check at its old cost, without building. A page with an album completeness filter assesses every album, so it still builds. A failed build is now tried again only after the stamp moves, so a persistent failure no longer costs a pairs query on every read. Its warning, like a new one for a failing snapshot query, is logged once per connection. --- lib/services/library_database.dart | 16 ++- lib/services/sqlite_helpers.dart | 76 +++++++++---- test/library_shadow_snapshot_test.dart | 147 ++++++++++++++++++++++++- 3 files changed, 217 insertions(+), 22 deletions(-) diff --git a/lib/services/library_database.dart b/lib/services/library_database.dart index d5d265f42..a32d2ea8d 100644 --- a/lib/services/library_database.dart +++ b/lib/services/library_database.dart @@ -4,6 +4,7 @@ import 'package:path/path.dart'; import 'package:mornye/utils/logger.dart'; import 'package:mornye/utils/audio_format_utils.dart'; import 'package:mornye/utils/file_access.dart'; +import 'package:mornye/services/album_completeness.dart'; import 'package:mornye/services/history_database.dart'; import 'package:mornye/services/library_queue_store.dart'; import 'package:mornye/services/library_file_sizes.dart'; @@ -139,6 +140,8 @@ class LibraryDatabase { }) async { final db = await database; await _ensureHistoryAttached(db); + // The row check runs only on the matching rows, so a search after a + // write keeps it rather than wait for a snapshot build. return _shadowRead( db, includeLocal, @@ -155,15 +158,18 @@ class LibraryDatabase { limit: limit, offset: offset, ), + rebuild: false, ); } /// [includeLocal] reads scanned rows, which the shadow snapshot serves. + /// [rebuild] false keeps the row check while the snapshot is stale. Future _shadowRead( Database db, bool includeLocal, - Future Function(bool snapshot) read, - ) => includeLocal ? _shadow.read(db, read) : read(false); + Future Function(bool snapshot) read, { + bool rebuild = true, + }) => includeLocal ? _shadow.read(db, read, rebuild: rebuild) : read(false); Future _ensureHistoryAttached(Database db) async { if (_historyAttached) return; @@ -411,6 +417,7 @@ class LibraryDatabase { Future Function(LibraryQueueStore store) read, { required bool includeLocal, QueueLibraryDbQuery? sizes, + bool rebuild = true, }) async { final db = await database; await _ensureHistoryAttached(db); @@ -431,15 +438,20 @@ class LibraryDatabase { shadowSnapshot: snapshot, ), ), + rebuild: rebuild, ); } + /// After a write a page keeps the row check, at its old cost, rather than + /// wait for a snapshot build. An album completeness filter assesses every + /// album, so it builds one. Future getQueueTrackPageResult( QueueLibraryDbQuery request, ) => _queueRead( (store) => store.trackPage(request), includeLocal: request.includeLocal, sizes: request, + rebuild: isAlbumCompletenessFilter(request.metadata), ); Future getQueueCounts(QueueLibraryDbQuery request) => diff --git a/lib/services/sqlite_helpers.dart b/lib/services/sqlite_helpers.dart index 99937f448..c071b6b33 100644 --- a/lib/services/sqlite_helpers.dart +++ b/lib/services/sqlite_helpers.dart @@ -554,45 +554,84 @@ class LibraryShadowSnapshot { Database? _db; String? _stamp; Future? _building; + // A failed build is tried again only once the data has changed. + ({Database db, String stamp})? _failed; + Database? _buildWarned; + Database? _queryWarned; int _rebuilds = 0; + /// Builds started, failed ones included. @visibleForTesting int get rebuilds => _rebuilds; /// The stamp of a current [libraryShadowTable], or null to keep the - /// per-row check, as when the platform returns no stamp. - Future refresh(Database db) async { + /// per-row check, as when the platform returns no stamp. With [rebuild] + /// false a stale table is not built and also gives null. + Future refresh(Database db, {bool rebuild = true}) async { // A rebuild in flight may have read the files before the caller's write. final building = _building; - if (building != null) await building; + if (building != null) { + if (!rebuild) return null; + await building; + } final stamp = await readWriteStamp(db, _schemas); if (stamp == null) return null; if (identical(db, _db) && stamp == _stamp) return stamp; - return _building ??= _rebuild(db).whenComplete(() => _building = null); + final failed = _failed; + if (!rebuild || + (failed != null && identical(failed.db, db) && failed.stamp == stamp)) { + return null; + } + return _building ??= _rebuild( + db, + stamp, + ).whenComplete(() => _building = null); } /// Runs [query] on [libraryShadowTable] when the table stays current /// until [query] has finished. Otherwise, as after a write between the /// two, its rows and the table could be from different states, so - /// [query] runs again with the per-row check. + /// [query] runs again with the per-row check. [rebuild] false suits reads + /// the row check keeps cheap, such as a page that stops after a few rows: + /// they use the table only when it is already current, never waiting for + /// a build. Future read( Database db, - Future Function(bool snapshot) query, - ) async { - final stamp = await refresh(db); + Future Function(bool snapshot) query, { + bool rebuild = true, + }) async { + final stamp = await refresh(db, rebuild: rebuild); if (stamp == null) return query(false); try { final result = await query(true); if (await readWriteStamp(db, _schemas) == stamp) return result; - } on DatabaseException { + } on DatabaseException catch (error) { // A failed rebuild can leave no table; the row check needs none. + if (!identical(_queryWarned, db)) { + _queryWarned = db; + _log.w( + 'Download shadow snapshot query failed; using the row check: $error', + ); + } } return query(false); } - Future _rebuild(Database db) async { + String? _buildFailed(Database db, String stamp, Object error) { + _failed = (db: db, stamp: stamp); + if (!identical(_buildWarned, db)) { + _buildWarned = db; + _log.w( + 'Download shadow snapshot unavailable; using the row check: $error', + ); + } + return null; + } + + Future _rebuild(Database db, String stamp) async { _db = null; _stamp = null; + _rebuilds++; try { final sizes = (await db.rawQuery( 'SELECT (SELECT COUNT(*) FROM library) AS library_rows, ' @@ -627,17 +666,16 @@ class LibraryShadowSnapshot { ) ..rawQuery('SELECT total_changes()')) .apply(); - _rebuilds++; - final stamp = _writeStamp(results, const [0, 1, 9]); - if (stamp == null) return null; + final built = _writeStamp(results, const [0, 1, 9]); + if (built == null) { + return _buildFailed(db, stamp, 'no write stamp after the build'); + } _db = db; - _stamp = stamp; - return stamp; + _stamp = built; + _failed = null; + return built; } catch (error) { - _log.w( - 'Download shadow snapshot unavailable; using the row check: $error', - ); - return null; + return _buildFailed(db, stamp, error); } } diff --git a/test/library_shadow_snapshot_test.dart b/test/library_shadow_snapshot_test.dart index 4667b1781..78471bdd5 100644 --- a/test/library_shadow_snapshot_test.dart +++ b/test/library_shadow_snapshot_test.dart @@ -13,6 +13,7 @@ import 'package:mornye/services/library_queue_store.dart'; import 'package:mornye/services/library_row_mapper.dart'; import 'package:mornye/services/library_search.dart'; import 'package:mornye/services/sqlite_helpers.dart' as sqlite; +import 'package:mornye/utils/logger.dart'; const _sources = ['legacy', 'music', 'offline', 'disabled']; @@ -397,6 +398,18 @@ Future _expectCurrent(String step) async { expect(await _shadowPairs(db), await _currentPairs(db), reason: step); } +/// Keeps warnings in [LogBuffer] for the rest of the test, from an empty one. +void _captureWarnings() { + final enabled = LogBuffer.loggingEnabled; + LogBuffer.loggingEnabled = true; + LogBuffer().clear(); + addTearDown(() => LogBuffer.loggingEnabled = enabled); +} + +int _warnings(String text) => LogBuffer().entries + .where((entry) => entry.level == 'WARN' && entry.message.contains(text)) + .length; + Future _scannedSongs() async => (await LibraryDatabase.instance.getQueueCounts( const QueueLibraryDbQuery(source: 'local'), @@ -431,10 +444,14 @@ void main() { () async { await _seed(extraDownloads: extraDownloads); final db = await LibraryDatabase.instance.database; - // Size sorts read the size index this call creates. + // Size sorts read the size index this page creates; the counts + // build the snapshot. await LibraryDatabase.instance.getQueueTrackPageResult( const QueueLibraryDbQuery(sortMode: 'size-desc'), ); + await LibraryDatabase.instance.getQueueCounts( + const QueueLibraryDbQuery(), + ); final sizes = (await db.rawQuery( 'SELECT (SELECT COUNT(*) FROM library) AS library, ' '(SELECT COUNT(*) FROM history_db.history) AS history', @@ -676,6 +693,134 @@ void main() { } }); + test( + 'a page or a search after a write does not build the snapshot', + () async { + await _seed(); + final library = LibraryDatabase.instance; + await library.getQueueCounts(const QueueLibraryDbQuery()); + final db = await library.database; + final built = await _shadowPairs(db); + // The download shadows l-4-1, which the built snapshot does not hold. + await HistoryDatabase.instance.upsert( + _download('h-new', _saf(4, 1), album: 4, number: 1), + ); + const page = QueueLibraryDbQuery(source: 'local', limit: 1000); + expect( + (await library.getQueueTrackPageResult(page)).rows, + (await _store(db).trackPage(page)).rows, + ); + expect( + _hits( + await library.searchLibrary( + query: 'song', + kind: LibrarySearchKind.songs, + includeLocal: true, + limit: 1000, + ), + ), + _hits( + await LibrarySearchStore( + db, + historyFts: HistoryDatabase.instance.searchFtsAvailable, + localFts: library.searchFtsAvailable, + ).search( + query: 'song', + kind: LibrarySearchKind.songs, + includeLocal: true, + limit: 1000, + ), + ), + ); + expect(await _shadowPairs(db), built); + expect(await _currentPairs(db), isNot(built)); + + // A completeness filter assesses every album, so it builds one. + const incomplete = QueueLibraryDbQuery( + metadata: incompleteAlbumFilter, + limit: 1000, + ); + expect( + (await library.getQueueTrackPageResult(incomplete)).rows, + (await _store(db).trackPage(incomplete)).rows, + ); + expect(await _shadowPairs(db), await _currentPairs(db)); + }, + ); + + test('a read that does not build uses only a current snapshot', () async { + await _seed(); + final library = LibraryDatabase.instance; + await library.getQueueCounts(const QueueLibraryDbQuery()); + final db = await library.database; + final snapshot = sqlite.LibraryShadowSnapshot(); + Future pageRead() => + snapshot.read(db, (snap) async => snap, rebuild: false); + expect(await pageRead(), isFalse); + expect(snapshot.rebuilds, 0); + expect(await snapshot.refresh(db), isNotNull); + expect(await pageRead(), isTrue); + await HistoryDatabase.instance.upsert( + _download('h-new', _saf(4, 1), album: 4, number: 1), + ); + expect(await pageRead(), isFalse); + expect(snapshot.rebuilds, 1); + }); + + test('a failed build waits for the next write and warns once', () async { + _captureWarnings(); + final db = await _connection(inMemoryDatabasePath); + addTearDown(db.close); + await db.execute('ATTACH DATABASE ? AS history_db', [':memory:']); + // Without History tables every build fails. + await db.execute('CREATE TABLE library(id TEXT PRIMARY KEY)'); + await db.execute( + 'CREATE TABLE library_path_keys(item_id TEXT NOT NULL, ' + 'path_key TEXT NOT NULL)', + ); + final snapshot = sqlite.LibraryShadowSnapshot(); + for (var i = 0; i < 3; i++) { + expect(await snapshot.read(db, (snap) async => snap), isFalse); + } + expect(snapshot.rebuilds, 1); + await db.insert('library', {'id': 'a'}); + expect(await snapshot.refresh(db), isNull); + expect(await snapshot.refresh(db), isNull); + expect(snapshot.rebuilds, 2); + expect(_warnings('Download shadow snapshot unavailable'), 1); + + await db.execute('CREATE TABLE history_db.history(id TEXT)'); + await db.execute( + 'CREATE TABLE history_db.history_path_keys(item_id TEXT NOT NULL, ' + 'path_key TEXT NOT NULL)', + ); + // DDL is no data change; the next write is. + expect(await snapshot.refresh(db), isNull); + await db.insert('library', {'id': 'b'}); + expect(await snapshot.read(db, (snap) async => snap), isTrue); + expect(snapshot.rebuilds, 3); + }); + + test('a failing snapshot query keeps the row check and warns once', () async { + await _seed(); + final library = LibraryDatabase.instance; + await library.getQueueCounts(const QueueLibraryDbQuery()); + final db = await library.database; + final snapshot = sqlite.LibraryShadowSnapshot(); + expect(await snapshot.refresh(db), isNotNull); + _captureWarnings(); + for (var i = 0; i < 2; i++) { + final modes = []; + await snapshot.read(db, (snap) async { + modes.add(snap); + if (snap) await db.rawQuery('SELECT * FROM temp.no_such_snapshot'); + return snap; + }); + expect(modes, [true, false]); + } + expect(_warnings('Download shadow snapshot query failed'), 1); + }); + test('without a stamp the reads keep the row check', () async { final memory = await databaseFactoryFfi.openDatabase(inMemoryDatabasePath); addTearDown(memory.close);