From 4d0fe68f17253cdd037b1fcf6ec84d5ca280681a Mon Sep 17 00:00:00 2001 From: zarzet <42882290+zarzet@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:19:14 +0700 Subject: [PATCH] perf(collections): hydrate native snapshots on bounded workers --- .../library_collections_workers_test.dart | 290 +++++++++ .../library_collections_provider.dart | 183 ++---- .../library_collections_database.dart | 361 +++++++++-- .../library_collections_hydration.dart | 609 ++++++++++++++++++ rust_backend/crates/mobile/src/data_jobs.rs | 1 + rust_backend/crates/mobile/src/lib.rs | 1 + .../crates/mobile/src/native_collections.rs | 175 +++++ .../mobile/src/native_collections/tests.rs | 220 +++++++ test/library_collections_benchmark_test.dart | 34 + ...ibrary_collections_database_read_test.dart | 367 +++++++++++ test/library_collections_hydration_test.dart | 352 ++++++++++ test/library_collections_mutation_test.dart | 169 ++++- .../library_collections_benchmark.dart | 334 ++++++++++ 13 files changed, 2915 insertions(+), 181 deletions(-) create mode 100644 integration_test/library_collections_workers_test.dart create mode 100644 lib/services/library_collections_hydration.dart create mode 100644 rust_backend/crates/mobile/src/native_collections.rs create mode 100644 rust_backend/crates/mobile/src/native_collections/tests.rs create mode 100644 test/library_collections_benchmark_test.dart create mode 100644 test/library_collections_database_read_test.dart create mode 100644 test/library_collections_hydration_test.dart create mode 100644 test/support/library_collections_benchmark.dart diff --git a/integration_test/library_collections_workers_test.dart b/integration_test/library_collections_workers_test.dart new file mode 100644 index 00000000..c97d740c --- /dev/null +++ b/integration_test/library_collections_workers_test.dart @@ -0,0 +1,290 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:crypto/crypto.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; +import 'package:sqflite/sqflite.dart'; +import 'package:spotiflac_android/models/library_collections.dart'; +import 'package:spotiflac_android/services/library_collections_database.dart'; +import 'package:spotiflac_android/services/library_collections_hydration.dart'; +import 'package:spotiflac_android/services/platform_bridge.dart'; + +import '../test/support/library_collections_benchmark.dart'; + +Future _database(String path) => openDatabase( + path, + version: 2, + onConfigure: (db) async { + await db.execute('PRAGMA foreign_keys=ON'); + await db.rawQuery('PRAGMA journal_mode=WAL'); + await db.rawQuery('PRAGMA busy_timeout=5000'); + }, + onCreate: (db, _) async { + final batch = db.batch(); + for (final sql in [ + 'CREATE TABLE wishlist_tracks(track_key TEXT PRIMARY KEY,track_json TEXT NOT NULL,added_at TEXT NOT NULL)', + 'CREATE TABLE loved_tracks(track_key TEXT PRIMARY KEY,track_json TEXT NOT NULL,added_at TEXT NOT NULL)', + 'CREATE TABLE favorite_artists(artist_key TEXT PRIMARY KEY,artist_json TEXT NOT NULL,added_at TEXT NOT NULL)', + 'CREATE TABLE playlists(id TEXT PRIMARY KEY,name TEXT NOT NULL,cover_image_path TEXT,created_at TEXT NOT NULL,updated_at TEXT NOT NULL)', + 'CREATE TABLE playlist_tracks(playlist_id TEXT NOT NULL,track_key TEXT NOT NULL,track_json TEXT NOT NULL,added_at TEXT NOT NULL,PRIMARY KEY(playlist_id,track_key),FOREIGN KEY(playlist_id) REFERENCES playlists(id) ON DELETE CASCADE)', + 'CREATE INDEX idx_wishlist_tracks_added_at ON wishlist_tracks(added_at DESC)', + 'CREATE INDEX idx_loved_tracks_added_at ON loved_tracks(added_at DESC)', + 'CREATE INDEX idx_favorite_artists_added_at ON favorite_artists(added_at DESC)', + 'CREATE INDEX idx_playlists_created_at ON playlists(created_at DESC)', + 'CREATE INDEX idx_playlist_tracks_playlist_id ON playlist_tracks(playlist_id)', + ]) { + batch.execute(sql); + } + await batch.commit(noResult: true); + }, +); + +Future> _databaseFixture(int count) => Isolate.run(() { + final tracks = [ + for (var index = 0; index < count; index++) + { + 'key': 'example:$index', + 'addedAt': '2026-10-0${index % 3 + 1}T00:00:00Z', + 'track': { + 'id': '$index', + 'name': '音楽 🎵 Track $index', + 'artistName': 'Artist ${index % 97}', + 'albumName': 'Album ${index ~/ 12}', + 'duration': 180, + 'coverUrl': 'https://example.test/cover-$index.jpg', + 'source': 'example', + }, + }, + ]; + return { + 'wishlist': tracks, + 'loved': tracks.take(32).toList(), + 'favoriteArtists': [ + { + 'key': 'example:artist', + 'artistId': 'artist', + 'providerId': 'example', + 'name': 'İstanbul', + 'addedAt': '2026-10-01T00:00:00Z', + }, + ], + 'playlists': [ + for (var index = 0; index < 4; index++) + { + 'id': 'p-$index', + 'name': 'Playlist $index', + 'createdAt': '2026-10-01T00:00:00Z', + 'updatedAt': '2026-10-01T00:00:00Z', + if (index == 1) 'coverImagePath': '/covers/custom.jpg', + 'tracks': tracks.skip(index * (count ~/ 4)).take(count ~/ 4).toList(), + }, + ], + }; +}); + +Future _legacyRead(Database db) async { + final snapshot = LibraryCollectionsSnapshot( + wishlistRows: await db.rawQuery( + 'SELECT * FROM wishlist_tracks ORDER BY added_at DESC,rowid DESC', + ), + lovedRows: await db.rawQuery( + 'SELECT * FROM loved_tracks ORDER BY added_at DESC,rowid DESC', + ), + playlistRows: await db.rawQuery( + """SELECT p.*, CASE WHEN p.cover_image_path IS NULL OR p.cover_image_path='' THEN + (SELECT pt.track_json FROM playlist_tracks pt WHERE pt.playlist_id=p.id ORDER BY pt.added_at ASC,pt.rowid ASC LIMIT 1) + END AS preview_track_json FROM playlists p ORDER BY p.created_at DESC,p.rowid DESC""", + ), + playlistTrackRows: await db.rawQuery( + 'SELECT playlist_id,track_key FROM playlist_tracks ORDER BY playlist_id ASC,rowid ASC', + ), + favoriteArtistRows: await db.rawQuery( + 'SELECT * FROM favorite_artists ORDER BY added_at DESC,rowid DESC', + ), + ); + return decodeLibraryCollectionsSnapshot(snapshot); +} + +Future _digest(LibraryCollectionsState state) => Isolate.run( + () => sha256 + .convert( + utf8.encode( + jsonEncode({ + 'state': state.toJson(), + 'membership': [ + for (final playlist in state.playlists) + { + 'id': playlist.id, + 'keys': playlist.trackKeys.toList()..sort(), + }, + ], + }), + ), + ) + .toString(), +); + +void main() { + final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + final report = { + 'platform': Platform.operatingSystem, + 'note': 'Five alternating paired samples; not physical-device FPS/RAM.', + }; + void publish() { + binding.reportData ??= {}; + binding.reportData!['collection_workers'] = report; + } + + testWidgets( + 'real sqflite native snapshot and playlist read preserve rows and cleanup', + (tester) async { + final directory = await Directory.systemTemp.createTemp( + 'collection-channel-read-', + ); + final db = await _database('${directory.path}/library_collections.db'); + try { + await LibraryCollectionsDatabase.replaceDatabaseFromBackup( + db, + await _databaseFixture(8000), + ); + await db.execute('CREATE TABLE projection_write_probe(value INTEGER)'); + await db.execute('INSERT INTO projection_write_probe VALUES(0)'); + var writes = 0; + var verifyIsolation = false; + Future> nativeRead( + Map request, + ) async { + expect(request['collections_path'], isNot(db.path)); + if (verifyIsolation) { + final databases = await db.rawQuery('PRAGMA database_list'); + expect( + databases.any( + (row) => (row['name'] as String).startsWith('collection_read_'), + ), + false, + reason: 'sqflite must detach the private file before native IO', + ); + // Verify outside the paired timing samples: live writes cannot + // alter the detached projection and remain on the same engine. + await db.execute('UPDATE projection_write_probe SET value=value+1'); + writes++; + } + return PlatformBridge.runNativeDataJob(request); + } + + final expectedDigest = await _digest(await _legacyRead(db)); + final baseline = >[]; + final native = >[]; + for (var sample = 0; sample < 5; sample++) { + for (final useNative + in sample.isEven ? [false, true] : [true, false]) { + final result = + await measureCollectionOperation( + () async { + if (!useNative) return _legacyRead(db); + final snapshot = + await LibraryCollectionsDatabase.readDatabaseSnapshot( + db, + nativeJobRunner: nativeRead, + ); + expect(snapshot.nativeRowsPath, isNotNull); + final staged = snapshot.nativeRowsDirectory!; + final state = await hydrateLibraryCollectionsSnapshot( + snapshot, + ); + expect(await Directory(staged).exists(), isFalse); + return state; + }, + ); + expect(await _digest(result.value), expectedDigest); + (useNative ? native : baseline).add(result.timing); + } + } + verifyIsolation = true; + final rows = + await LibraryCollectionsDatabase.readDatabasePlaylistTracks( + db, + 'p-0', + nativeJobRunner: nativeRead, + ); + expect(rows.nativeRowsPath, isNotNull); + final expectedTracks = await db.rawQuery( + 'SELECT * FROM playlist_tracks WHERE playlist_id=? ORDER BY added_at ASC,rowid ASC', + ['p-0'], + ); + final expected = await hydratePlaylistTracks( + expectedTracks, + knownKeys: expectedTracks + .map((row) => row['track_key'] as String) + .toSet(), + ); + final staged = rows.nativeRowsDirectory!; + final tracks = await hydratePlaylistTracksSnapshot( + rows, + knownKeys: expected.trackKeys, + ); + expect( + jsonEncode(tracks.tracks.map((entry) => entry.toJson()).toList()), + jsonEncode(expected.tracks.map((entry) => entry.toJson()).toList()), + ); + expect(tracks.membershipUnchanged, isTrue); + expect(await Directory(staged).exists(), isFalse); + expect(await db.rawQuery('SELECT value FROM projection_write_probe'), [ + {'value': writes}, + ]); + expect(await db.rawQuery('PRAGMA integrity_check'), [ + {'integrity_check': 'ok'}, + ]); + report['native_database_read'] = { + 'tracks': 8000, + 'membership_rows': 8000, + 'baseline': baseline, + 'native': native, + 'digest': expectedDigest, + 'note': + 'Full original sqflite queries+caller hydration versus same-engine private SQL projection+native file+worker, five alternating pairs; native never opens the live DB.', + }; + publish(); + } finally { + await db.close(); + await directory.delete(recursive: true); + } + }, + ); + + testWidgets( + 'collection snapshot worker preserves parity and caller heartbeat', + (tester) async { + final result = await benchmarkCollectionHydration(); + final samples = result['worker'] as List>; + expect(samples.every((sample) => sample['heartbeat_count']! > 0), isTrue); + report['collection_hydration'] = result; + publish(); + }, + ); + + testWidgets('large playlist search preserves original ranked pages', ( + tester, + ) async { + report['playlist_search'] = await benchmarkPlaylistSearch(); + publish(); + }); + + testWidgets('metadata-only playlist copies retain all membership indices', ( + tester, + ) async { + report['playlist_metadata_copy'] = await benchmarkPlaylistMetadataCopy(); + publish(); + }); + + testWidgets( + 'collection export worker preserves model JSON and caller heartbeat', + (tester) async { + report['collection_export'] = await benchmarkCollectionExport(); + report['collection_json_export'] = await benchmarkCollectionJsonExport(); + publish(); + }, + ); +} diff --git a/lib/providers/library_collections_provider.dart b/lib/providers/library_collections_provider.dart index f38f11f3..7841a8c0 100644 --- a/lib/providers/library_collections_provider.dart +++ b/lib/providers/library_collections_provider.dart @@ -8,6 +8,7 @@ import 'package:spotiflac_android/models/library_collections.dart'; import 'package:spotiflac_android/models/track.dart'; import 'package:spotiflac_android/services/ffmpeg_service.dart'; import 'package:spotiflac_android/services/library_collections_database.dart'; +import 'package:spotiflac_android/services/library_collections_hydration.dart'; import 'package:spotiflac_android/utils/logger.dart'; export 'package:spotiflac_android/models/library_collections.dart'; @@ -58,84 +59,16 @@ class LibraryCollectionsNotifier extends Notifier { await _db.migrateFromSharedPreferences(); if (!ref.mounted) return; final snapshot = await _db.loadSnapshot(); - if (!ref.mounted) return; - - final wishlist = []; - for (final row in snapshot.wishlistRows) { - final parsed = _parseTrackEntryRow(row); - if (parsed != null) { - wishlist.add(parsed); - } - } - - final loved = []; - for (final row in snapshot.lovedRows) { - final parsed = _parseTrackEntryRow(row); - if (parsed != null) { - loved.add(parsed); - } - } - - final favoriteArtists = []; - for (final row in snapshot.favoriteArtistRows) { - final parsed = _parseArtistEntryRow(row); - if (parsed != null) { - favoriteArtists.add(parsed); - } - } - - final trackKeysByPlaylist = >{}; - for (final row in snapshot.playlistTrackRows) { - final playlistId = row['playlist_id'] as String?; - if (playlistId == null || playlistId.isEmpty) continue; - final trackKey = row['track_key'] as String?; - if (trackKey == null || trackKey.isEmpty) continue; - trackKeysByPlaylist.putIfAbsent(playlistId, () => {}).add(trackKey); - } - - final playlists = []; - for (final row in snapshot.playlistRows) { - final id = row['id'] as String?; - if (id == null || id.isEmpty) continue; - - final createdAtRaw = row['created_at'] as String?; - final updatedAtRaw = row['updated_at'] as String?; - final createdAt = - DateTime.tryParse(createdAtRaw ?? '') ?? DateTime.now(); - final updatedAt = DateTime.tryParse(updatedAtRaw ?? '') ?? createdAt; - String? previewCover; - final previewTrackJson = row['preview_track_json'] as String?; - if (previewTrackJson != null && previewTrackJson.isNotEmpty) { - try { - final decoded = jsonDecode(previewTrackJson); - if (decoded is Map) { - previewCover = decoded['coverUrl']?.toString(); - } - } catch (_) {} - } - - playlists.add( - UserPlaylistCollection( - id: id, - name: row['name'] as String? ?? '', - coverImagePath: row['cover_image_path'] as String?, - createdAt: createdAt, - updatedAt: updatedAt, - tracks: const [], - previewCover: previewCover, - tracksLoaded: false, - trackKeys: trackKeysByPlaylist[id], - ), + try { + if (!ref.mounted) return; + final loaded = await hydrateLibraryCollectionsSnapshot(snapshot); + if (ref.mounted) state = loaded; + } finally { + await cleanupCollectionRows( + nativeRowsPath: snapshot.nativeRowsPath, + nativeRowsDirectory: snapshot.nativeRowsDirectory, ); } - - state = LibraryCollectionsState( - wishlist: wishlist, - loved: loved, - playlists: playlists, - favoriteArtists: favoriteArtists, - isLoaded: true, - ); } catch (error, stack) { _log.e('Failed to load library collections', error, stack); if (ref.mounted) state = state.copyWith(isLoaded: true); @@ -157,16 +90,39 @@ class LibraryCollectionsNotifier extends Notifier { final pending = _playlistLoadFutures[playlistId]; if (pending != null) return pending; final load = () async { - final rows = await _db.loadPlaylistTracks(playlistId); - if (!ref.mounted) return; - final tracks = rows - .map(_parseTrackEntryRow) - .whereType() - .toList(growable: false); - _replacePlaylistById( - playlistId, - (current) => current.copyWith(tracks: tracks, tracksLoaded: true), - ); + while (ref.mounted) { + final loading = state.playlistById(playlistId); + if (loading == null || loading.tracksLoaded) return; + final snapshot = await _db.loadPlaylistTracksSnapshot(playlistId); + late HydratedPlaylistTracks hydrated; + try { + if (!ref.mounted) return; + hydrated = await hydratePlaylistTracksSnapshot( + snapshot, + knownKeys: loading.trackKeys, + ); + } finally { + await cleanupCollectionRows( + nativeRowsPath: snapshot.nativeRowsPath, + nativeRowsDirectory: snapshot.nativeRowsDirectory, + ); + } + if (!ref.mounted) return; + final current = state.playlistById(playlistId); + if (current == null || current.tracksLoaded) return; + // Metadata-only edits share the index. A restore/reload can replace it + // while the worker is running; retry rather than publish stale tracks. + if (!current.sharesMembershipIndexWith(loading)) continue; + _replacePlaylistById( + playlistId, + (current) => current.withHydratedTracks( + tracks: hydrated.tracks, + trackKeys: hydrated.trackKeys, + membershipUnchanged: hydrated.membershipUnchanged, + ), + ); + return; + } }(); _playlistLoadFutures[playlistId] = load; try { @@ -182,53 +138,6 @@ class LibraryCollectionsNotifier extends Notifier { await Future.wait(playlistIds.toSet().map(ensurePlaylistLoaded)); } - CollectionTrackEntry? _parseTrackEntryRow(Map row) { - final key = row['track_key'] as String?; - final trackJson = row['track_json'] as String?; - if (key == null || key.isEmpty || trackJson == null || trackJson.isEmpty) { - return null; - } - - try { - final decoded = jsonDecode(trackJson); - if (decoded is! Map) return null; - final track = Track.fromJson(Map.from(decoded)); - final addedAtRaw = row['added_at'] as String?; - return CollectionTrackEntry( - key: key, - track: track, - addedAt: DateTime.tryParse(addedAtRaw ?? '') ?? DateTime.now(), - ); - } catch (_) { - return null; - } - } - - CollectionArtistEntry? _parseArtistEntryRow(Map row) { - final key = row['artist_key'] as String?; - final artistJson = row['artist_json'] as String?; - if (key == null || - key.isEmpty || - artistJson == null || - artistJson.isEmpty) { - return null; - } - - try { - final decoded = jsonDecode(artistJson); - if (decoded is! Map) return null; - final map = Map.from(decoded); - final addedAtRaw = row['added_at'] as String?; - return CollectionArtistEntry.fromJson({ - ...map, - 'key': key, - 'addedAt': map['addedAt'] ?? addedAtRaw, - }); - } catch (_) { - return null; - } - } - bool _replacePlaylistById( String playlistId, UserPlaylistCollection Function(UserPlaylistCollection playlist) update, @@ -731,7 +640,13 @@ class LibraryCollectionsNotifier extends Notifier { Future> exportCollections() async { await _ensureLoaded(); await ensurePlaylistsLoaded(state.playlists.map((playlist) => playlist.id)); - return state.toJson(); + return exportLibraryCollectionsState(state); + } + + Future exportCollectionsJson() async { + await _ensureLoaded(); + await ensurePlaylistsLoaded(state.playlists.map((playlist) => playlist.id)); + return exportLibraryCollectionsJson(state); } /// Exports custom playlist cover images as base64, keyed by playlist id. diff --git a/lib/services/library_collections_database.dart b/lib/services/library_collections_database.dart index 2b170ab7..057ce0dd 100644 --- a/lib/services/library_collections_database.dart +++ b/lib/services/library_collections_database.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:io'; @@ -31,6 +32,9 @@ class LibraryCollectionsSnapshot { final List> playlistRows; final List> playlistTrackRows; final List> favoriteArtistRows; + final String? nativeRowsPath; + final String? nativeRowsDirectory; + final int? nativeRowCount; const LibraryCollectionsSnapshot({ required this.wishlistRows, @@ -38,9 +42,28 @@ class LibraryCollectionsSnapshot { required this.playlistRows, required this.playlistTrackRows, required this.favoriteArtistRows, + this.nativeRowsPath, + this.nativeRowsDirectory, + this.nativeRowCount, }); } +class LibraryPlaylistTracksSnapshot { + const LibraryPlaylistTracksSnapshot({ + this.rows = const [], + this.nativeRowsPath, + this.nativeRowsDirectory, + this.nativeRowCount, + }); + final List> rows; + final String? nativeRowsPath; + final String? nativeRowsDirectory; + final int? nativeRowCount; +} + +typedef NativeCollectionsJobRunner = + Future> Function(Map request); + class LibraryCollectionsDatabase { static int _restoreAttachmentSequence = 0; static final LibraryCollectionsDatabase instance = @@ -239,60 +262,320 @@ class LibraryCollectionsDatabase { } } - Future loadSnapshot() async { - final db = await database; - final wishlistRows = await db.query( - _tableWishlist, - orderBy: 'added_at DESC, rowid DESC', - ); - final lovedRows = await db.query( - _tableLoved, - orderBy: 'added_at DESC, rowid DESC', - ); - final playlistRows = await db.rawQuery(''' - SELECT - p.*, + Future loadSnapshot() async => + readDatabaseSnapshot(await database); + + static Future readDatabaseSnapshot( + Database db, { + NativeCollectionsJobRunner? nativeJobRunner, + Directory? temporaryDirectory, + bool forceLegacy = false, + }) async { + if (!forceLegacy && + (nativeJobRunner != null || PlatformBridge.supportsCoreBackend)) { + final probe = await db.rawQuery(''' + SELECT 1 FROM $_tableWishlist UNION ALL SELECT 1 FROM $_tableLoved + UNION ALL SELECT 1 FROM $_tablePlaylists + UNION ALL SELECT 1 FROM $_tablePlaylistTracks + UNION ALL SELECT 1 FROM $_tableFavoriteArtists LIMIT 64 + '''); + if (probe.length == 64) { + return _readNativeRows( + db, + nativeJobRunner: nativeJobRunner, + temporaryDirectory: temporaryDirectory, + ); + } + } + return db.transaction((txn) async { + final wishlistRows = await _readDateOrderedRows( + txn, + _tableWishlist, + 'added_at', + ); + final lovedRows = await _readDateOrderedRows( + txn, + _tableLoved, + 'added_at', + ); + final playlistRows = await _readDateOrderedRows( + txn, + _tablePlaylists, + 'created_at', + projection: + '''p.*, CASE WHEN p.cover_image_path IS NULL OR p.cover_image_path = '' THEN ( SELECT pt.track_json FROM $_tablePlaylistTracks pt WHERE pt.playlist_id = p.id ORDER BY pt.added_at ASC, pt.rowid ASC LIMIT 1 - ) END AS preview_track_json - FROM $_tablePlaylists p - ORDER BY p.created_at DESC, p.rowid DESC - '''); - final playlistTrackRows = await db.query( - _tablePlaylistTracks, - columns: ['playlist_id', 'track_key'], - orderBy: 'playlist_id ASC, rowid ASC', - ); - final favoriteArtistRows = await db.query( - _tableFavoriteArtists, - orderBy: 'added_at DESC, rowid DESC', - ); - - return LibraryCollectionsSnapshot( - wishlistRows: wishlistRows, - lovedRows: lovedRows, - playlistRows: playlistRows, - playlistTrackRows: playlistTrackRows, - favoriteArtistRows: favoriteArtistRows, - ); + ) END AS preview_track_json''', + alias: 'p', + ); + final playlistTrackRows = >[]; + Object? lastId; + int? lastRowId; + while (true) { + final page = await txn.rawQuery(''' + SELECT rowid AS __collection_rowid, playlist_id, track_key + FROM $_tablePlaylistTracks + ${lastRowId == null ? '' : 'WHERE playlist_id>? OR (playlist_id=? AND rowid>?)'} + ORDER BY playlist_id ASC,rowid ASC LIMIT 256 + ''', lastRowId == null ? const [] : [lastId, lastId, lastRowId]); + if (page.isEmpty) break; + lastId = page.last['playlist_id']; + lastRowId = page.last['__collection_rowid'] as int; + playlistTrackRows.addAll(page.map(_withoutRowCursor)); + if (page.length < 256) break; + } + final favoriteArtistRows = await _readDateOrderedRows( + txn, + _tableFavoriteArtists, + 'added_at', + ); + return LibraryCollectionsSnapshot( + wishlistRows: wishlistRows, + lovedRows: lovedRows, + playlistRows: playlistRows, + playlistTrackRows: playlistTrackRows, + favoriteArtistRows: favoriteArtistRows, + ); + }); } Future>> loadPlaylistTracks( String playlistId, ) async { final db = await database; - return db.query( - _tablePlaylistTracks, - where: 'playlist_id = ?', - whereArgs: [playlistId], - orderBy: 'added_at ASC, rowid ASC', + return db.transaction( + (txn) => _readDateOrderedRows( + txn, + _tablePlaylistTracks, + 'added_at', + ascending: true, + where: 'playlist_id=?', + whereArgs: [playlistId], + ), ); } + Future loadPlaylistTracksSnapshot( + String playlistId, + ) async => readDatabasePlaylistTracks(await database, playlistId); + + static Future readDatabasePlaylistTracks( + Database db, + String playlistId, { + NativeCollectionsJobRunner? nativeJobRunner, + Directory? temporaryDirectory, + bool forceLegacy = false, + }) async { + if (!forceLegacy && + (nativeJobRunner != null || PlatformBridge.supportsCoreBackend)) { + final probe = await db.rawQuery( + 'SELECT 1 FROM $_tablePlaylistTracks WHERE playlist_id=? LIMIT 64', + [playlistId], + ); + if (probe.length == 64) { + final snapshot = await _readNativeRows( + db, + playlistId: playlistId, + nativeJobRunner: nativeJobRunner, + temporaryDirectory: temporaryDirectory, + ); + return LibraryPlaylistTracksSnapshot( + nativeRowsPath: snapshot.nativeRowsPath, + nativeRowsDirectory: snapshot.nativeRowsDirectory, + nativeRowCount: snapshot.nativeRowCount, + ); + } + } + final rows = await db.transaction( + (txn) => _readDateOrderedRows( + txn, + _tablePlaylistTracks, + 'added_at', + ascending: true, + where: 'playlist_id=?', + whereArgs: [playlistId], + ), + ); + return LibraryPlaylistTracksSnapshot(rows: rows); + } + + static Map _withoutRowCursor(Map row) => + Map.from(row)..remove('__collection_rowid'); + + static Future>> _readDateOrderedRows( + DatabaseExecutor db, + String table, + String dateColumn, { + bool ascending = false, + String projection = '*', + String? alias, + String? where, + List whereArgs = const [], + }) async { + final rows = >[]; + final qualifier = alias == null ? '' : '$alias.'; + final comparison = ascending ? '>' : '<'; + final direction = ascending ? 'ASC' : 'DESC'; + Object? lastDate; + int? lastRowId; + while (true) { + final predicates = [ + if (where != null) '($where)', + if (lastRowId != null) + '($qualifier$dateColumn$comparison? OR ($qualifier$dateColumn=? AND ${qualifier}rowid$comparison?))', + ]; + final page = await db.rawQuery( + ''' + SELECT ${qualifier}rowid AS __collection_rowid, $projection + FROM $table ${alias ?? ''} + ${predicates.isEmpty ? '' : 'WHERE ${predicates.join(' AND ')}'} + ORDER BY $qualifier$dateColumn $direction,${qualifier}rowid $direction LIMIT 256 + ''', + [ + ...whereArgs, + if (lastRowId != null) ...[lastDate, lastDate, lastRowId], + ], + ); + if (page.isEmpty) break; + lastDate = page.last[dateColumn]; + lastRowId = page.last['__collection_rowid'] as int; + rows.addAll(page.map(_withoutRowCursor)); + if (page.length < 256) break; + } + return rows; + } + + static Future _readNativeRows( + Database db, { + String? playlistId, + NativeCollectionsJobRunner? nativeJobRunner, + Directory? temporaryDirectory, + }) async { + final temporary = + await (temporaryDirectory ?? await getTemporaryDirectory()).createTemp( + 'collection-read-', + ); + final path = '${temporary.path}/rows.ndjson'; + try { + final snapshotPath = '${temporary.path}/$_dbFileName'; + await _projectCollectionReadDatabase( + db, + snapshotPath, + playlistId: playlistId, + ); + final request = { + 'operation': 'collections_snapshot', + 'collections_path': snapshotPath, + 'output_path': path, + 'playlist_id': ?playlistId, + }; + final result = await (nativeJobRunner ?? PlatformBridge.runNativeDataJob)( + request, + ); + final counts = result['counts']; + if (result['published'] != true || + result['rows_path'] != path || + counts is! Map) { + throw const FormatException('Invalid native collection snapshot'); + } + var total = 0; + for (final count in counts.values) { + if (count is! int || count < 0) { + throw const FormatException('Invalid native collection row count'); + } + total += count; + } + return LibraryCollectionsSnapshot( + wishlistRows: const [], + lovedRows: const [], + playlistRows: const [], + playlistTrackRows: const [], + favoriteArtistRows: const [], + nativeRowsPath: path, + nativeRowsDirectory: temporary.path, + nativeRowCount: total, + ); + } on NativeSqliteSnapshotDetachException { + _log.w( + 'Retaining collection read staging after a failed database detach', + ); + rethrow; + } catch (_) { + await temporary.delete(recursive: true); + rethrow; + } + } + + static final _collectionProjectionQueue = Expando>(); + static var _collectionProjectionSequence = 0; + + /// Both the live database and this projection are owned by the same SQLite + /// engine during copying. DETACH closes its private file before Rust opens + /// it; independent SQLite copies must never share a live app database. + static Future _projectCollectionReadDatabase( + Database db, + String outputPath, { + String? playlistId, + }) async { + final previous = _collectionProjectionQueue[db]; + final finished = Completer(); + _collectionProjectionQueue[db] = finished.future; + if (previous != null) await previous; + final alias = 'collection_read_${_collectionProjectionSequence++}'; + var attached = false; + try { + await db.execute('ATTACH DATABASE ? AS $alias', [outputPath]); + attached = true; + await db.transaction((txn) async { + for (final table in [ + _tableWishlist, + _tableLoved, + _tableFavoriteArtists, + _tablePlaylists, + _tablePlaylistTracks, + ]) { + final where = playlistId == null + ? '' + : table == _tablePlaylists + ? 'WHERE id=?' + : table == _tablePlaylistTracks + ? 'WHERE playlist_id=?' + : 'WHERE 0'; + // CTAS copies every original column and value. Inserting in original + // rowid order preserves all date-tie and membership ordering without + // transferring any collection rows through the platform channel. + await txn.execute( + 'CREATE TABLE $alias.$table AS ' + 'SELECT * FROM main.$table $where ORDER BY rowid ASC', + where.contains('?') ? [playlistId] : const [], + ); + } + await txn.execute('PRAGMA $alias.user_version=$_dbVersion'); + await txn.execute( + 'CREATE INDEX $alias.collection_read_members ' + 'ON $_tablePlaylistTracks(playlist_id)', + ); + await txn.execute( + 'CREATE INDEX $alias.collection_read_preview ' + 'ON $_tablePlaylistTracks(playlist_id,added_at)', + ); + }); + } finally { + try { + if (attached) { + await detachNativeSqliteSnapshot(db, alias, outputPath); + } + } finally { + finished.complete(); + } + } + } + Future> loadPlaylistPickerSummaries( List requestedTrackKeys, ) async => readPlaylistPickerSummaries(await database, requestedTrackKeys); diff --git a/lib/services/library_collections_hydration.dart b/lib/services/library_collections_hydration.dart new file mode 100644 index 00000000..0f9d69d8 --- /dev/null +++ b/lib/services/library_collections_hydration.dart @@ -0,0 +1,609 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:spotiflac_android/models/library_collections.dart'; +import 'package:spotiflac_android/models/track.dart'; +import 'package:spotiflac_android/services/library_collections_database.dart'; + +const _workerRowThreshold = 64; +const _workerJsonThreshold = 64 * 1024; + +/// Small collections avoid isolate startup. Large snapshots leave JSON parsing, +/// model construction and all membership indices on the worker together. +Future hydrateLibraryCollectionsSnapshot( + LibraryCollectionsSnapshot snapshot, +) async { + if (snapshot.nativeRowsPath != null) { + try { + return await _hydrateRowsInWorker( + rows: const {}, + nativeRowsPath: snapshot.nativeRowsPath, + nativeRowCount: snapshot.nativeRowCount, + ); + } finally { + await cleanupCollectionRows( + nativeRowsPath: snapshot.nativeRowsPath, + nativeRowsDirectory: snapshot.nativeRowsDirectory, + ); + } + } + final rows = + snapshot.wishlistRows.length + + snapshot.lovedRows.length + + snapshot.favoriteArtistRows.length + + snapshot.playlistRows.length + + snapshot.playlistTrackRows.length; + if (rows < _workerRowThreshold && + !_hasLargeJson([ + ...snapshot.wishlistRows, + ...snapshot.lovedRows, + ...snapshot.favoriteArtistRows, + ...snapshot.playlistRows, + ])) { + return decodeLibraryCollectionsSnapshot(snapshot); + } + return _hydrateRowsInWorker( + rows: { + _CollectionRows.wishlist: snapshot.wishlistRows, + _CollectionRows.loved: snapshot.lovedRows, + _CollectionRows.artists: snapshot.favoriteArtistRows, + _CollectionRows.membership: snapshot.playlistTrackRows, + _CollectionRows.playlists: snapshot.playlistRows, + }, + ); +} + +LibraryCollectionsState decodeLibraryCollectionsSnapshot( + LibraryCollectionsSnapshot snapshot, +) { + final accumulator = _CollectionHydrationAccumulator() + ..add(_CollectionRows.wishlist, snapshot.wishlistRows) + ..add(_CollectionRows.loved, snapshot.lovedRows) + ..add(_CollectionRows.artists, snapshot.favoriteArtistRows) + ..add(_CollectionRows.membership, snapshot.playlistTrackRows) + ..add(_CollectionRows.playlists, snapshot.playlistRows); + return accumulator.finish(); +} + +List _decodePlaylistRows( + List> rows, + Map> trackKeysByPlaylist, +) { + final playlists = []; + for (final row in rows) { + final id = row['id'] as String?; + if (id == null || id.isEmpty) continue; + final createdAtRaw = row['created_at'] as String?; + final updatedAtRaw = row['updated_at'] as String?; + final createdAt = DateTime.tryParse(createdAtRaw ?? '') ?? DateTime.now(); + final updatedAt = DateTime.tryParse(updatedAtRaw ?? '') ?? createdAt; + String? previewCover; + final previewTrackJson = row['preview_track_json'] as String?; + if (previewTrackJson != null && previewTrackJson.isNotEmpty) { + try { + final decoded = jsonDecode(previewTrackJson); + if (decoded is Map) previewCover = decoded['coverUrl']?.toString(); + } catch (_) {} + } + playlists.add( + UserPlaylistCollection( + id: id, + name: row['name'] as String? ?? '', + coverImagePath: row['cover_image_path'] as String?, + createdAt: createdAt, + updatedAt: updatedAt, + tracks: const [], + previewCover: previewCover, + tracksLoaded: false, + trackKeys: trackKeysByPlaylist[id], + ), + ); + } + + return playlists; +} + +class HydratedPlaylistTracks { + const HydratedPlaylistTracks({ + required this.tracks, + required this.trackKeys, + required this.membershipUnchanged, + }); + + final List tracks; + final Set trackKeys; + final bool membershipUnchanged; +} + +typedef _PlaylistHydrationRequest = ({ + List> rows, + Set knownKeys, +}); + +Future hydratePlaylistTracks( + List> rows, { + required Set knownKeys, +}) async { + final request = (rows: rows, knownKeys: knownKeys); + if (rows.length < _workerRowThreshold && !_hasLargeJson(rows)) { + return _decodePlaylistTracks(request); + } + return _hydrateRowsInWorker( + rows: {_CollectionRows.wishlist: rows}, + knownKeys: knownKeys, + ); +} + +Future hydratePlaylistTracksSnapshot( + LibraryPlaylistTracksSnapshot snapshot, { + required Set knownKeys, +}) async { + if (snapshot.nativeRowsPath == null) { + return hydratePlaylistTracks(snapshot.rows, knownKeys: knownKeys); + } + try { + return await _hydrateRowsInWorker( + rows: const {}, + knownKeys: knownKeys, + nativeRowsPath: snapshot.nativeRowsPath, + nativeRowCount: snapshot.nativeRowCount, + ); + } finally { + await cleanupCollectionRows( + nativeRowsPath: snapshot.nativeRowsPath, + nativeRowsDirectory: snapshot.nativeRowsDirectory, + ); + } +} + +Future cleanupCollectionRows({ + String? nativeRowsPath, + String? nativeRowsDirectory, +}) async { + try { + if (nativeRowsDirectory != null) { + await Directory(nativeRowsDirectory).delete(recursive: true); + } else if (nativeRowsPath != null) { + await File(nativeRowsPath).delete(); + } + } on FileSystemException { + // The consuming worker may already have removed its row file. + } +} + +HydratedPlaylistTracks _decodePlaylistTracks( + _PlaylistHydrationRequest request, +) { + final tracks = _decodeTrackRows(request.rows).toList(growable: false); + final keys = tracks.map((entry) => entry.key).toSet(); + return HydratedPlaylistTracks( + tracks: tracks, + trackKeys: keys, + membershipUnchanged: + keys.length == request.knownKeys.length && + keys.every(request.knownKeys.contains), + ); +} + +List _decodeTrackRows(List> rows) => + [for (final row in rows) ?_parseTrackRow(row)]; + +CollectionTrackEntry? _parseTrackRow(Map row) { + final key = row['track_key'] as String?; + final trackJson = row['track_json'] as String?; + if (key == null || key.isEmpty || trackJson == null || trackJson.isEmpty) { + return null; + } + try { + final decoded = jsonDecode(trackJson); + if (decoded is! Map) return null; + return CollectionTrackEntry( + key: key, + track: Track.fromJson(Map.from(decoded)), + addedAt: + DateTime.tryParse(row['added_at'] as String? ?? '') ?? DateTime.now(), + ); + } catch (_) { + return null; + } +} + +CollectionArtistEntry? _parseArtistRow(Map row) { + final key = row['artist_key'] as String?; + final artistJson = row['artist_json'] as String?; + if (key == null || key.isEmpty || artistJson == null || artistJson.isEmpty) { + return null; + } + try { + final decoded = jsonDecode(artistJson); + if (decoded is! Map) return null; + final map = Map.from(decoded); + return CollectionArtistEntry.fromJson({ + ...map, + 'key': key, + 'addedAt': map['addedAt'] ?? row['added_at'], + }); + } catch (_) { + return null; + } +} + +bool _hasLargeJson(List> rows) { + var length = 0; + for (final row in rows) { + for (final column in ['track_json', 'artist_json', 'preview_track_json']) { + final value = row[column]; + if (value is String) length += value.length; + } + if (length >= _workerJsonThreshold) return true; + } + return false; +} + +enum _CollectionRows { wishlist, loved, artists, membership, playlists } + +class _CollectionHydrationAccumulator { + final wishlist = []; + final loved = []; + final artists = []; + final playlistRows = >[]; + final trackKeysByPlaylist = >{}; + + void add(_CollectionRows kind, List> rows) { + switch (kind) { + case _CollectionRows.wishlist: + wishlist.addAll(_decodeTrackRows(rows)); + case _CollectionRows.loved: + loved.addAll(_decodeTrackRows(rows)); + case _CollectionRows.artists: + for (final row in rows) { + final artist = _parseArtistRow(row); + if (artist != null) artists.add(artist); + } + case _CollectionRows.playlists: + playlistRows.addAll(rows); + case _CollectionRows.membership: + for (final row in rows) { + final id = row['playlist_id'] as String?; + final key = row['track_key'] as String?; + if (id == null || id.isEmpty || key == null || key.isEmpty) continue; + trackKeysByPlaylist.putIfAbsent(id, () => {}).add(key); + } + } + } + + LibraryCollectionsState finish() => LibraryCollectionsState( + wishlist: wishlist, + loved: loved, + favoriteArtists: artists, + playlists: _decodePlaylistRows(playlistRows, trackKeysByPlaylist), + isLoaded: true, + ); +} + +enum _CollectionWorkerMode { snapshot, playlist, export, exportJson } + +typedef _CollectionWorkerRequest = ({ + SendPort result, + _CollectionWorkerMode mode, +}); +typedef _CollectionRowBatch = ({ + _CollectionRows kind, + List> rows, +}); + +typedef _NativeCollectionRows = ({String path, int? count}); + +/// A single giant compute argument copies every row map before the isolate +/// starts, still blocking the caller. Stream bounded batches instead. The +/// worker consumes JSON as it arrives; Isolate.exit transfers the final model +/// graph without another full copy on the UI isolate. +Future _hydrateRowsInWorker({ + required Map<_CollectionRows, List>> rows, + Set? knownKeys, + String? nativeRowsPath, + int? nativeRowCount, +}) => _runCollectionWorker( + mode: knownKeys == null + ? _CollectionWorkerMode.snapshot + : _CollectionWorkerMode.playlist, + feed: (sendBatch) async { + for (final entry in rows.entries) { + for (var offset = 0; offset < entry.value.length; offset += 256) { + final end = (offset + 256).clamp(0, entry.value.length); + await sendBatch(( + kind: entry.key, + rows: entry.value.sublist(offset, end), + )); + } + } + if (knownKeys != null) { + var batch = []; + for (final key in knownKeys) { + batch.add(key); + if (batch.length < 256) continue; + await sendBatch(batch); + batch = []; + } + if (batch.isNotEmpty) await sendBatch(batch); + } + if (nativeRowsPath != null) { + await sendBatch((path: nativeRowsPath, count: nativeRowCount)); + } + }, +); + +typedef _SendCollectionBatch = Future Function(Object batch); + +Future _runCollectionWorker({ + required _CollectionWorkerMode mode, + required Future Function(_SendCollectionBatch sendBatch) feed, +}) async { + final replies = ReceivePort(); + final ready = Completer(); + final result = Completer(); + Completer? acknowledgement; + // An error can arrive while batches are being sent, before the final await. + result.future.ignore(); + final subscription = replies.listen((Object? message) { + if (message is SendPort) { + ready.complete(message); + } else if (message == true) { + acknowledgement?.complete(); + } else if (message is List && message.length == 1) { + result.complete(message.single as T); + } else { + final error = message is List && message.length == 2 + ? RemoteError(message[0].toString(), message[1].toString()) + : StateError('Collections hydration worker exited without a result'); + if (!ready.isCompleted) ready.completeError(error); + if (!result.isCompleted) result.completeError(error); + if (acknowledgement case final pending? when !pending.isCompleted) { + pending.completeError(error); + } + } + }); + Isolate? isolate; + try { + isolate = await Isolate.spawn( + _collectionHydrationWorker, + (result: replies.sendPort, mode: mode), + onError: replies.sendPort, + onExit: replies.sendPort, + debugName: 'library collections hydration', + ); + final commands = await ready.future; + Future sendBatch(Object batch) async { + final received = Completer(); + acknowledgement = received; + commands.send(batch); + await received.future; + } + + await feed(sendBatch); + commands.send(null); + return await result.future; + } finally { + isolate?.kill(priority: Isolate.immediate); + await subscription.cancel(); + replies.close(); + } +} + +void _collectionHydrationWorker(_CollectionWorkerRequest request) async { + final commands = ReceivePort(); + final accumulator = _CollectionHydrationAccumulator(); + final exporter = _CollectionExportAccumulator(); + final knownKeys = {}; + request.result.send(commands.sendPort); + await for (final Object? message in commands) { + if (message case _CollectionRowBatch batch) { + accumulator.add(batch.kind, batch.rows); + request.result.send(true); + } else if (message is List) { + knownKeys.addAll(message); + request.result.send(true); + } else if (message case _NativeCollectionRows input) { + await _readNativeCollectionRows(input, accumulator); + request.result.send(true); + } else if (message case _CollectionExportBatch batch) { + exporter.add(batch); + request.result.send(true); + } else if (message case _PlaylistExportHeader header) { + exporter.startPlaylist(header); + request.result.send(true); + } else if (message == null) { + commands.close(); + if (request.mode == _CollectionWorkerMode.export || + request.mode == _CollectionWorkerMode.exportJson) { + final exported = exporter.finish(); + Isolate.exit(request.result, [ + request.mode == _CollectionWorkerMode.exportJson + ? jsonEncode(exported) + : exported, + ]); + } + if (request.mode == _CollectionWorkerMode.snapshot) { + Isolate.exit(request.result, [accumulator.finish()]); + } + final tracks = accumulator.wishlist.toList(growable: false); + final keys = tracks.map((entry) => entry.key).toSet(); + Isolate.exit(request.result, [ + HydratedPlaylistTracks( + tracks: tracks, + trackKeys: keys, + membershipUnchanged: + keys.length == knownKeys.length && keys.every(knownKeys.contains), + ), + ]); + } + } +} + +Future _readNativeCollectionRows( + _NativeCollectionRows input, + _CollectionHydrationAccumulator accumulator, +) async { + var count = 0; + _CollectionRows? previousKind; + var batch = >[]; + try { + await for (final line in File( + input.path, + ).openRead().transform(utf8.decoder).transform(const LineSplitter())) { + final decoded = jsonDecode(line) as Map; + final kind = switch (decoded['kind']) { + 'wishlist' || 'playlist_track' => _CollectionRows.wishlist, + 'loved' => _CollectionRows.loved, + 'artist' => _CollectionRows.artists, + 'membership' => _CollectionRows.membership, + 'playlist' => _CollectionRows.playlists, + _ => throw const FormatException( + 'Unknown collection snapshot row kind', + ), + }; + if (previousKind != null && + (kind != previousKind || batch.length == 256)) { + accumulator.add(previousKind, batch); + batch = >[]; + } + previousKind = kind; + batch.add(Map.from(decoded['row'] as Map)); + count++; + } + if (previousKind != null && batch.isNotEmpty) { + accumulator.add(previousKind, batch); + } + if (input.count != null && count != input.count) { + throw const FormatException('Incomplete native collection snapshot'); + } + } finally { + await File(input.path).delete(); + } +} + +enum _CollectionExportRows { wishlist, loved, artist, playlistTrack } + +typedef _CollectionExportBatch = ({ + _CollectionExportRows kind, + List entries, +}); +typedef _PlaylistExportHeader = ({ + String id, + String name, + String? cover, + DateTime createdAt, + DateTime updatedAt, +}); + +/// Converts the stable published model snapshot on a worker without cloning +/// the whole graph in a single send. Playlist headers exclude membership sets +/// and their tracks are submitted separately in bounded ordered batches. +Future> exportLibraryCollectionsState( + LibraryCollectionsState state, +) async { + var count = + state.wishlistCount + + state.lovedCount + + state.favoriteArtistCount + + state.playlistCount; + for (final playlist in state.playlists) { + if (count >= _workerRowThreshold) break; + count += playlist.tracks.length; + } + if (count < _workerRowThreshold) return state.toJson(); + return _exportLibraryCollectionsInWorker(state, _CollectionWorkerMode.export); +} + +/// Backup writers can embed this trusted JSON fragment directly, avoiding a +/// second transfer of the large exported map from the UI to another worker. +Future exportLibraryCollectionsJson(LibraryCollectionsState state) => + _exportLibraryCollectionsInWorker(state, _CollectionWorkerMode.exportJson); + +Future _exportLibraryCollectionsInWorker( + LibraryCollectionsState state, + _CollectionWorkerMode mode, +) => _runCollectionWorker( + mode: mode, + feed: (sendBatch) async { + Future entries( + _CollectionExportRows kind, + List values, + ) async { + for (var offset = 0; offset < values.length; offset += 256) { + await sendBatch(( + kind: kind, + entries: values.sublist( + offset, + (offset + 256).clamp(0, values.length), + ), + )); + } + } + + await entries(_CollectionExportRows.wishlist, state.wishlist); + await entries(_CollectionExportRows.loved, state.loved); + for (final playlist in state.playlists) { + await sendBatch(( + id: playlist.id, + name: playlist.name, + cover: playlist.coverImagePath, + createdAt: playlist.createdAt, + updatedAt: playlist.updatedAt, + )); + await entries(_CollectionExportRows.playlistTrack, playlist.tracks); + } + await entries(_CollectionExportRows.artist, state.favoriteArtists); + }, +); + +class _CollectionExportAccumulator { + final wishlist = >[]; + final loved = >[]; + final artists = >[]; + final playlists = >[]; + List>? playlistTracks; + + void startPlaylist(_PlaylistExportHeader header) { + playlistTracks = >[]; + playlists.add({ + 'id': header.id, + 'name': header.name, + if (header.cover != null) 'coverImagePath': header.cover, + 'createdAt': header.createdAt.toIso8601String(), + 'updatedAt': header.updatedAt.toIso8601String(), + 'tracks': playlistTracks, + }); + } + + void add(_CollectionExportBatch batch) { + if (batch.kind == _CollectionExportRows.artist) { + artists.addAll( + batch.entries.map((entry) => (entry as CollectionArtistEntry).toJson()), + ); + return; + } + final target = switch (batch.kind) { + _CollectionExportRows.wishlist => wishlist, + _CollectionExportRows.loved => loved, + _CollectionExportRows.playlistTrack => + playlistTracks ?? + (throw const FormatException('Playlist tracks require a header')), + _CollectionExportRows.artist => throw StateError( + 'Artist target is handled above', + ), + }; + target.addAll( + batch.entries.map((entry) => (entry as CollectionTrackEntry).toJson()), + ); + } + + Map finish() => { + 'wishlist': wishlist, + 'loved': loved, + 'playlists': playlists, + 'favoriteArtists': artists, + }; +} diff --git a/rust_backend/crates/mobile/src/data_jobs.rs b/rust_backend/crates/mobile/src/data_jobs.rs index 430abc90..f57f5840 100644 --- a/rust_backend/crates/mobile/src/data_jobs.rs +++ b/rust_backend/crates/mobile/src/data_jobs.rs @@ -125,6 +125,7 @@ fn execute( request, check, ), + "collections_snapshot" => crate::native_collections::execute(request, check), operation if operation.starts_with("library_") => { crate::native_library::execute(request, check) } diff --git a/rust_backend/crates/mobile/src/lib.rs b/rust_backend/crates/mobile/src/lib.rs index 4dc2befe..c8537d28 100644 --- a/rust_backend/crates/mobile/src/lib.rs +++ b/rust_backend/crates/mobile/src/lib.rs @@ -13,6 +13,7 @@ mod manager; mod metadata; mod native_backup; mod native_cache; +mod native_collections; mod native_id3; mod native_library; mod native_listing; diff --git a/rust_backend/crates/mobile/src/native_collections.rs b/rust_backend/crates/mobile/src/native_collections.rs new file mode 100644 index 00000000..a6f498be --- /dev/null +++ b/rust_backend/crates/mobile/src/native_collections.rs @@ -0,0 +1,175 @@ +//! Collection rows stay on the IO worker until a bounded Dart hydration worker +//! consumes their NDJSON file. The bridge returns only a path and row counts. +use rusqlite::{Connection, OpenFlags, Transaction, types::ValueRef}; +use serde_json::{Map, Value, json}; +use std::{ + io::{BufWriter, Write}, + path::Path, + time::Duration, +}; + +type Check<'a> = &'a dyn Fn() -> Result<(), String>; + +const SNAPSHOT_QUERIES: &[(&str, &str)] = &[ + ( + "wishlist", + "SELECT * FROM wishlist_tracks ORDER BY added_at DESC,rowid DESC", + ), + ( + "loved", + "SELECT * FROM loved_tracks ORDER BY added_at DESC,rowid DESC", + ), + ( + "playlist", + "SELECT p.*, CASE WHEN p.cover_image_path IS NULL OR p.cover_image_path='' THEN ( + SELECT pt.track_json FROM playlist_tracks pt WHERE pt.playlist_id=p.id + ORDER BY pt.added_at ASC,pt.rowid ASC LIMIT 1 + ) END AS preview_track_json FROM playlists p ORDER BY p.created_at DESC,p.rowid DESC", + ), + ( + "membership", + "SELECT playlist_id,track_key FROM playlist_tracks ORDER BY playlist_id ASC,rowid ASC", + ), + ( + "artist", + "SELECT * FROM favorite_artists ORDER BY added_at DESC,rowid DESC", + ), +]; + +fn required<'a>(request: &'a Value, field: &str) -> Result<&'a str, String> { + request[field] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| format!("Collection snapshot requires {field}")) +} + +fn write_rows( + transaction: &Transaction<'_>, + writer: &mut impl Write, + kind: &str, + query: &str, + playlist_id: Option<&str>, + check: Check<'_>, +) -> Result { + check()?; + let mut statement = transaction.prepare(query).map_err(|e| e.to_string())?; + let columns = statement + .column_names() + .iter() + .map(|name| (*name).to_owned()) + .collect::>(); + let mut rows = if let Some(id) = playlist_id { + statement.query([id]) + } else { + statement.query([]) + } + .map_err(|e| e.to_string())?; + let mut count = 0; + while let Some(row) = rows.next().map_err(|e| e.to_string())? { + check()?; + let mut values = Map::new(); + for (index, name) in columns.iter().enumerate() { + let value = match row.get_ref(index).map_err(|e| e.to_string())? { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) => Value::from(value), + ValueRef::Real(value) => Value::from(value), + ValueRef::Text(value) => { + Value::from(std::str::from_utf8(value).map_err(|e| e.to_string())?) + } + ValueRef::Blob(value) => { + Value::Array(value.iter().copied().map(Value::from).collect()) + } + }; + values.insert(name.clone(), value); + } + serde_json::to_writer(&mut *writer, &json!({"kind":kind,"row":values})) + .map_err(|e| format!("write collection snapshot: {e}"))?; + writer.write_all(b"\n").map_err(|e| e.to_string())?; + count += 1; + } + Ok(count) +} + +pub(crate) fn execute(request: &Value, check: Check<'_>) -> Result { + check()?; + if required(request, "operation")? != "collections_snapshot" { + return Err("Unknown native collection operation".into()); + } + let database_path = Path::new(required(request, "collections_path")?); + if !database_path.is_absolute() + || database_path.file_name().and_then(|name| name.to_str()) + != Some("library_collections.db") + { + return Err("Collection snapshot requires existing library_collections.db".into()); + } + let output_path = Path::new(required(request, "output_path")?); + if !output_path.is_absolute() { + return Err("Collection snapshot output path must be absolute".into()); + } + let mut db = Connection::open_with_flags(database_path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(|e| format!("open collection snapshot: {e}"))?; + db.busy_timeout(Duration::from_secs(5)) + .map_err(|e| e.to_string())?; + let transaction = db.transaction().map_err(|e| e.to_string())?; + let version: i64 = transaction + .pragma_query_value(None, "user_version", |row| row.get(0)) + .map_err(|e| e.to_string())?; + if version != 2 { + return Err(format!( + "Collection schema v{version} is incompatible with native v2" + )); + } + let mut staged = tempfile::NamedTempFile::new_in( + output_path + .parent() + .ok_or("Collection snapshot requires output parent")?, + ) + .map_err(|e| e.to_string())?; + let mut counts = Map::new(); + { + let mut writer = BufWriter::with_capacity(64 * 1024, staged.as_file_mut()); + if let Some(id) = request.get("playlist_id") { + let id = id + .as_str() + .ok_or("Collection playlist_id must be a string")?; + counts.insert( + "playlist_track".into(), + Value::from(write_rows( + &transaction, + &mut writer, + "playlist_track", + "SELECT * FROM playlist_tracks WHERE playlist_id=? + ORDER BY added_at ASC,rowid ASC", + Some(id), + check, + )?), + ); + } else { + for (kind, sql) in SNAPSHOT_QUERIES { + counts.insert( + (*kind).into(), + Value::from(write_rows( + &transaction, + &mut writer, + kind, + sql, + None, + check, + )?), + ); + } + } + writer.flush().map_err(|e| e.to_string())?; + } + transaction.commit().map_err(|e| e.to_string())?; + check()?; + // Never replace an existing result. RAII removes partial files on SQL, IO, + // schema or cancellation errors before this atomic publication. + staged + .persist_noclobber(output_path) + .map_err(|e| e.to_string())?; + Ok(json!({"rows_path":output_path,"counts":counts,"published":true,"committed":true})) +} + +#[cfg(test)] +mod tests; diff --git a/rust_backend/crates/mobile/src/native_collections/tests.rs b/rust_backend/crates/mobile/src/native_collections/tests.rs new file mode 100644 index 00000000..48b7b082 --- /dev/null +++ b/rust_backend/crates/mobile/src/native_collections/tests.rs @@ -0,0 +1,220 @@ +use super::*; +use std::{cell::Cell, fs}; + +struct Fixture { + directory: tempfile::TempDir, + db: Connection, + request: Value, +} + +impl Fixture { + fn new() -> Self { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("library_collections.db"); + let db = Connection::open(&path).unwrap(); + db.execute_batch(r#"PRAGMA user_version=2; PRAGMA journal_mode=WAL; + CREATE TABLE wishlist_tracks(track_key TEXT PRIMARY KEY,track_json TEXT NOT NULL,added_at TEXT NOT NULL); + CREATE TABLE loved_tracks(track_key TEXT PRIMARY KEY,track_json TEXT NOT NULL,added_at TEXT NOT NULL); + CREATE TABLE favorite_artists(artist_key TEXT PRIMARY KEY,artist_json TEXT NOT NULL,added_at TEXT NOT NULL); + CREATE TABLE playlists(id TEXT PRIMARY KEY,name TEXT NOT NULL,cover_image_path TEXT,created_at TEXT NOT NULL,updated_at TEXT NOT NULL); + CREATE TABLE playlist_tracks(playlist_id TEXT NOT NULL,track_key TEXT NOT NULL,track_json TEXT NOT NULL,added_at TEXT NOT NULL,PRIMARY KEY(playlist_id,track_key)); + INSERT INTO wishlist_tracks VALUES('first','{"id":"a"}','date'); + INSERT INTO wishlist_tracks VALUES('second','corrupt json','date'); + INSERT INTO loved_tracks VALUES('loved','[]','date'); + INSERT INTO favorite_artists VALUES('artist','{}','date'); + INSERT INTO playlists VALUES('p','音楽 🎵',NULL,'date','date'); + INSERT INTO playlists VALUES('custom','İstanbul','/cover.jpg','date','date'); + INSERT INTO playlists VALUES('empty','Empty','', 'date','date'); + INSERT INTO playlist_tracks VALUES('p','second','{"coverUrl":"first.jpg"}','date'); + INSERT INTO playlist_tracks VALUES('p','first','{"coverUrl":"second.jpg"}','date'); + INSERT INTO playlist_tracks VALUES('custom','custom','{}','date');"#).unwrap(); + let request = json!({"operation":"collections_snapshot","collections_path":path, + "output_path":directory.path().join("snapshot.ndjson")}); + Self { + directory, + db, + request, + } + } + + fn rows(&self) -> Vec { + fs::read_to_string(self.request["output_path"].as_str().unwrap()) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() + } + + fn staged_files(&self) -> Vec { + fs::read_dir(self.directory.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_str().unwrap().to_owned()) + .filter(|name| !name.starts_with("library_collections.db")) + .collect() + } +} + +#[test] +fn snapshot_preserves_nulls_previews_unicode_and_all_five_query_orders() { + let fixture = Fixture::new(); + let result = execute(&fixture.request, &|| Ok(())).unwrap(); + assert_eq!( + result["counts"], + json!({"wishlist":2,"loved":1,"playlist":3, + "membership":3,"artist":1}) + ); + assert_eq!(result["published"], true); + assert_eq!(result["committed"], true); + let rows = fixture.rows(); + assert_eq!(rows[0]["row"]["track_key"], "second"); + assert_eq!(rows[1]["row"]["track_key"], "first"); + let playlists = rows + .iter() + .filter(|row| row["kind"] == "playlist") + .collect::>(); + assert_eq!( + playlists + .iter() + .map(|row| row["row"]["id"].as_str().unwrap()) + .collect::>(), + ["empty", "custom", "p"] + ); + assert_eq!(playlists[0]["row"]["preview_track_json"], Value::Null); + assert_eq!(playlists[1]["row"]["preview_track_json"], Value::Null); + assert_eq!( + playlists[2]["row"]["preview_track_json"], + "{\"coverUrl\":\"first.jpg\"}" + ); + assert_eq!(playlists[2]["row"]["name"], "音楽 🎵"); + let members = rows + .iter() + .filter(|row| row["kind"] == "membership") + .collect::>(); + assert_eq!( + members + .iter() + .map(|row| row["row"]["track_key"].as_str().unwrap()) + .collect::>(), + ["custom", "second", "first"] + ); + assert_eq!(members[0]["row"].as_object().unwrap().len(), 2); +} + +#[test] +fn playlist_projection_is_bound_and_retains_ascending_date_rowid_order() { + let mut fixture = Fixture::new(); + fixture.request["playlist_id"] = Value::from("p"); + let result = execute(&fixture.request, &|| Ok(())).unwrap(); + assert_eq!(result["counts"], json!({"playlist_track":2})); + assert_eq!( + fixture + .rows() + .iter() + .map(|row| row["row"]["track_key"].as_str().unwrap()) + .collect::>(), + ["second", "first"] + ); + let mut missing = Fixture::new(); + missing.request["playlist_id"] = Value::from("p' OR 1=1 --"); + assert_eq!( + execute(&missing.request, &|| Ok(())).unwrap()["counts"], + json!({"playlist_track":0}) + ); +} + +#[test] +fn cancellation_every_checkpoint_removes_staging_and_preserves_database() { + let fixture = Fixture::new(); + let calls = Cell::new(0); + execute(&fixture.request, &|| { + calls.set(calls.get() + 1); + Ok(()) + }) + .unwrap(); + for cancel_at in 1..=calls.get() { + let fixture = Fixture::new(); + let current = Cell::new(0); + let result = execute(&fixture.request, &|| { + current.set(current.get() + 1); + if current.get() == cancel_at { + Err("cancelled".into()) + } else { + Ok(()) + } + }); + assert!(result.is_err(), "checkpoint {cancel_at}"); + assert!(fixture.staged_files().is_empty(), "checkpoint {cancel_at}"); + assert_eq!( + fixture + .db + .query_row("SELECT COUNT(*) FROM wishlist_tracks", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); + } +} + +#[test] +fn schema_missing_database_and_existing_output_never_publish_or_overwrite() { + for kind in 0..3 { + let mut fixture = Fixture::new(); + match kind { + 0 => fixture.db.execute_batch("PRAGMA user_version=3").unwrap(), + 1 => { + fixture.request["collections_path"] = Value::from( + fixture + .directory + .path() + .join("missing/library_collections.db") + .to_str() + .unwrap(), + ) + } + _ => fs::write(fixture.request["output_path"].as_str().unwrap(), "kept").unwrap(), + } + assert!(execute(&fixture.request, &|| Ok(())).is_err()); + if kind == 2 { + assert_eq!( + fs::read_to_string(fixture.request["output_path"].as_str().unwrap()).unwrap(), + "kept" + ); + assert_eq!(fixture.staged_files(), ["snapshot.ndjson"]); + } else { + assert!(fixture.staged_files().is_empty()); + } + } +} + +#[test] +fn all_queries_share_one_stable_wal_read_snapshot() { + let fixture = Fixture::new(); + let calls = Cell::new(0); + execute(&fixture.request, &|| { + calls.set(calls.get() + 1); + if calls.get() == 3 { + fixture + .db + .execute("INSERT INTO loved_tracks VALUES('new','{}','date')", []) + .unwrap(); + } + Ok(()) + }) + .unwrap(); + assert_eq!( + fixture + .rows() + .iter() + .filter(|row| row["kind"] == "loved") + .count(), + 1 + ); + assert_eq!( + fixture + .db + .query_row("SELECT COUNT(*) FROM loved_tracks", [], |row| row + .get::<_, i64>(0)) + .unwrap(), + 2 + ); +} diff --git a/test/library_collections_benchmark_test.dart b/test/library_collections_benchmark_test.dart new file mode 100644 index 00000000..5cde4567 --- /dev/null +++ b/test/library_collections_benchmark_test.dart @@ -0,0 +1,34 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'support/library_collections_benchmark.dart'; + +void main() { + test( + 'paired large collection workers benchmark', + () async { + final report = { + 'platform': Platform.operatingSystem, + 'mode': 'flutter test host', + 'collection_hydration': await benchmarkCollectionHydration(), + 'playlist_search': await benchmarkPlaylistSearch(), + 'playlist_metadata_copy': await benchmarkPlaylistMetadataCopy(), + 'collection_export': await benchmarkCollectionExport(), + 'collection_json_export': await benchmarkCollectionJsonExport(), + }; + const output = String.fromEnvironment('COLLECTIONS_BENCHMARK_OUTPUT'); + if (output.isNotEmpty) { + final file = File(output); + await file.parent.create(recursive: true); + await file.writeAsString( + const JsonEncoder.withIndent(' ').convert(report), + ); + } + expect(report['collection_hydration'], isA>()); + }, + skip: !const bool.fromEnvironment('RUN_COLLECTIONS_BENCHMARK'), + timeout: const Timeout(Duration(minutes: 3)), + ); +} diff --git a/test/library_collections_database_read_test.dart b/test/library_collections_database_read_test.dart new file mode 100644 index 00000000..777f80f4 --- /dev/null +++ b/test/library_collections_database_read_test.dart @@ -0,0 +1,367 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:sqflite/sqflite.dart'; +import 'package:spotiflac_android/services/library_collections_database.dart'; +import 'package:spotiflac_android/services/library_collections_hydration.dart'; + +import 'support/sqlite_process_database.dart'; + +// Host parity executes the production keyset SQL against real SQLite. Device +// integration separately covers the sqflite channel and Rust read route. +class _ReadDatabase implements Database, Transaction { + _ReadDatabase(this.database); + @override + final SqliteProcessDatabase database; + @override + String get path => database.path; + final pageSizes = []; + bool failProjection = false; + + @override + Future transaction( + Future Function(Transaction txn) action, { + bool? exclusive, + }) => database.transaction((_) => action(this), exclusive: exclusive); + + @override + Future execute(String sql, [List? arguments]) { + if (failProjection && sql.startsWith('CREATE TABLE collection_read_')) { + throw StateError('projection failed'); + } + return database.execute(sql, arguments); + } + + @override + Future>> rawQuery( + String sql, [ + List? arguments, + ]) async { + final rows = await database.rawQuery(sql, arguments); + pageSizes.add(rows.length); + return rows; + } + + @override + Future close() => database.close(); + + @override + dynamic noSuchMethod(Invocation invocation) => super.noSuchMethod(invocation); +} + +Future<_ReadDatabase> _fixture(String path, int count) async { + final result = await Process.run('python3', [ + '-c', + r""" +import json,sqlite3,sys +db=sqlite3.connect(sys.argv[1]); count=int(sys.argv[2]) +db.executescript('''PRAGMA user_version=2; +CREATE TABLE wishlist_tracks(track_key TEXT PRIMARY KEY,track_json TEXT NOT NULL,added_at TEXT NOT NULL); +CREATE TABLE loved_tracks(track_key TEXT PRIMARY KEY,track_json TEXT NOT NULL,added_at TEXT NOT NULL); +CREATE TABLE favorite_artists(artist_key TEXT PRIMARY KEY,artist_json TEXT NOT NULL,added_at TEXT NOT NULL); +CREATE TABLE playlists(id TEXT PRIMARY KEY,name TEXT NOT NULL,cover_image_path TEXT,created_at TEXT NOT NULL,updated_at TEXT NOT NULL); +CREATE TABLE playlist_tracks(playlist_id TEXT NOT NULL,track_key TEXT NOT NULL,track_json TEXT NOT NULL,added_at TEXT NOT NULL,PRIMARY KEY(playlist_id,track_key));''') +for p in range(4): + db.execute('INSERT INTO playlists VALUES(?,?,?,?,?)',(str(p),'音楽 '+str(p),'/custom.jpg' if p==1 else None,'2026-10-01','2026-10-01')) +for i in range(count): + track=json.dumps(dict(id=str(i),name='Track '+str(i),artistName='Artist',albumName='Album',duration=180,coverUrl='cover-'+str(i)),ensure_ascii=False) + date='2026-10-0'+str(i%3+1) + db.execute('INSERT INTO wishlist_tracks VALUES(?,?,?)',('example:'+str(i),track,date)) + db.execute('INSERT INTO playlist_tracks VALUES(?,?,?,?)',(str(i%4),'example:'+str(i),track,date)) +db.execute('INSERT INTO loved_tracks VALUES(?,?,?)',('corrupt','{','2026-10-01')) +db.execute('INSERT INTO favorite_artists VALUES(?,?,?)',('example:artist',json.dumps(dict(artistId='artist',providerId='example',name='İstanbul')),'2026-10-01')) +db.commit() +""", + path, + '$count', + ]); + if (result.exitCode != 0) throw StateError(result.stderr.toString()); + final db = _ReadDatabase(await SqliteProcessDatabase.open(path: path)); + addTearDown(db.close); + return db; +} + +Future _legacy( + _ReadDatabase db, +) async => LibraryCollectionsSnapshot( + wishlistRows: await db.rawQuery( + 'SELECT * FROM wishlist_tracks ORDER BY added_at DESC,rowid DESC', + ), + lovedRows: await db.rawQuery( + 'SELECT * FROM loved_tracks ORDER BY added_at DESC,rowid DESC', + ), + favoriteArtistRows: await db.rawQuery( + 'SELECT * FROM favorite_artists ORDER BY added_at DESC,rowid DESC', + ), + playlistRows: await db.rawQuery( + """SELECT p.*,CASE WHEN p.cover_image_path IS NULL OR p.cover_image_path='' THEN + (SELECT pt.track_json FROM playlist_tracks pt WHERE pt.playlist_id=p.id ORDER BY pt.added_at ASC,pt.rowid ASC LIMIT 1) + END AS preview_track_json FROM playlists p ORDER BY p.created_at DESC,p.rowid DESC""", + ), + playlistTrackRows: await db.rawQuery( + 'SELECT playlist_id,track_key FROM playlist_tracks ORDER BY playlist_id ASC,rowid ASC', + ), +); + +void main() { + late Directory root; + setUp(() async { + root = await Directory.systemTemp.createTemp('collection-db-read-test-'); + }); + tearDown(() => root.delete(recursive: true)); + + test( + 'keyset pages preserve all original query rows and tied-date ordering', + () async { + final db = await _fixture('${root.path}/library_collections.db', 1200); + final actual = await LibraryCollectionsDatabase.readDatabaseSnapshot( + db, + forceLegacy: true, + ); + expect(db.pageSizes.every((size) => size <= 256), isTrue); + final expected = await _legacy(db); + expect(actual.wishlistRows, expected.wishlistRows); + expect(actual.lovedRows, expected.lovedRows); + expect(actual.favoriteArtistRows, expected.favoriteArtistRows); + expect(actual.playlistRows, expected.playlistRows); + expect(actual.playlistTrackRows, expected.playlistTrackRows); + db.pageSizes.clear(); + final tracks = + await LibraryCollectionsDatabase.readDatabasePlaylistTracks( + db, + '1', + forceLegacy: true, + ); + expect(db.pageSizes.every((size) => size <= 256), isTrue); + expect( + tracks.rows, + await db.rawQuery( + 'SELECT * FROM playlist_tracks WHERE playlist_id=? ORDER BY added_at ASC,rowid ASC', + ['1'], + ), + ); + }, + ); + + test( + 'small collection stays on bounded SQLite reads without native job', + () async { + final db = await _fixture('${root.path}/library_collections.db', 3); + final snapshot = await LibraryCollectionsDatabase.readDatabaseSnapshot( + db, + temporaryDirectory: root, + nativeJobRunner: (_) async => throw StateError('must not run'), + ); + expect(snapshot.nativeRowsPath, isNull); + expect(snapshot.wishlistRows.length, 3); + expect(db.pageSizes.every((size) => size <= 64), isTrue); + }, + ); + + test( + 'large read routes only probe and staged path, hydrates and cleans', + () async { + final db = await _fixture('${root.path}/library_collections.db', 600); + final expected = await _legacy(db); + db.pageSizes.clear(); + var calls = 0; + final snapshot = await LibraryCollectionsDatabase.readDatabaseSnapshot( + db, + temporaryDirectory: root, + nativeJobRunner: (request) async { + calls++; + expect(request['operation'], 'collections_snapshot'); + final projectedPath = request['collections_path'] as String; + expect(projectedPath, isNot(db.path)); + expect( + File(projectedPath).parent.path, + File(request['output_path'] as String).parent.path, + ); + final attached = await db.database.rawQuery('PRAGMA database_list'); + expect(attached.map((row) => row['name']), ['main']); + final projectedDb = _ReadDatabase( + await SqliteProcessDatabase.open(path: projectedPath), + ); + late LibraryCollectionsSnapshot projected; + try { + projected = await _legacy(projectedDb); + expect(projected.wishlistRows, expected.wishlistRows); + expect(projected.lovedRows, expected.lovedRows); + expect(projected.playlistRows, expected.playlistRows); + expect(projected.playlistTrackRows, expected.playlistTrackRows); + expect(projected.favoriteArtistRows, expected.favoriteArtistRows); + } finally { + await projectedDb.close(); + } + // Changing the live rows after DETACH cannot change this read's + // already captured projection or its hydrated membership snapshot. + await db.execute("DELETE FROM playlist_tracks WHERE playlist_id='0'"); + final rows = [ + for (final row in projected.wishlistRows) + {'kind': 'wishlist', 'row': row}, + for (final row in projected.lovedRows) + {'kind': 'loved', 'row': row}, + for (final row in projected.playlistRows) + {'kind': 'playlist', 'row': row}, + for (final row in projected.playlistTrackRows) + {'kind': 'membership', 'row': row}, + for (final row in projected.favoriteArtistRows) + {'kind': 'artist', 'row': row}, + ]; + await File( + request['output_path'] as String, + ).writeAsString(rows.map(jsonEncode).join('\n')); + return { + 'published': true, + 'rows_path': request['output_path'], + 'counts': {'rows': rows.length}, + }; + }, + ); + expect(calls, 1); + expect(db.pageSizes, [64]); + expect(snapshot.wishlistRows, isEmpty); + final stagedDirectory = snapshot.nativeRowsDirectory!; + final actual = await hydrateLibraryCollectionsSnapshot(snapshot); + expect( + jsonEncode(actual.toJson()), + jsonEncode(decodeLibraryCollectionsSnapshot(expected).toJson()), + ); + for (final playlist in actual.playlists) { + expect(playlist.trackCount, 150); + } + expect(await Directory(stagedDirectory).exists(), isFalse); + }, + ); + + test( + 'lazy read projects only the selected playlist and preserves ties', + () async { + final db = await _fixture('${root.path}/library_collections.db', 800); + final expected = await db.rawQuery( + 'SELECT * FROM playlist_tracks WHERE playlist_id=? ORDER BY added_at ASC,rowid ASC', + ['1'], + ); + final snapshot = + await LibraryCollectionsDatabase.readDatabasePlaylistTracks( + db, + '1', + temporaryDirectory: root, + nativeJobRunner: (request) async { + expect(request['playlist_id'], '1'); + final projectedPath = request['collections_path'] as String; + expect(projectedPath, isNot(db.path)); + final projected = await SqliteProcessDatabase.open( + path: projectedPath, + ); + try { + for (final table in [ + 'wishlist_tracks', + 'loved_tracks', + 'favorite_artists', + ]) { + expect( + await projected.rawQuery('SELECT * FROM $table'), + isEmpty, + ); + } + expect(await projected.rawQuery('SELECT id FROM playlists'), [ + {'id': '1'}, + ]); + expect(await projected.rawQuery('PRAGMA user_version'), [ + {'user_version': 2}, + ]); + final rows = await projected.rawQuery( + 'SELECT * FROM playlist_tracks ORDER BY added_at ASC,rowid ASC', + ); + expect(rows, expected); + await File(request['output_path'] as String).writeAsString( + rows + .map( + (row) => + jsonEncode({'kind': 'playlist_track', 'row': row}), + ) + .join('\n'), + ); + return { + 'published': true, + 'rows_path': request['output_path'], + 'counts': {'playlist_track': rows.length}, + }; + } finally { + await projected.close(); + } + }, + ); + final staged = snapshot.nativeRowsDirectory!; + final tracks = await hydratePlaylistTracksSnapshot( + snapshot, + knownKeys: expected.map((row) => row['track_key'] as String).toSet(), + ); + expect(tracks.membershipUnchanged, true); + expect(tracks.tracks, hasLength(200)); + expect(await Directory(staged).exists(), false); + }, + ); + + test( + 'failed projection detaches private database and removes staging', + () async { + final db = await _fixture('${root.path}/library_collections.db', 80); + db.failProjection = true; + await expectLater( + LibraryCollectionsDatabase.readDatabaseSnapshot( + db, + temporaryDirectory: root, + nativeJobRunner: (_) async => throw StateError('must not run'), + ), + throwsStateError, + ); + final databases = await db.database.rawQuery('PRAGMA database_list'); + expect(databases.map((row) => row['name']), ['main']); + expect( + await File(databases.single['file'] as String).resolveSymbolicLinks(), + await File(db.path).resolveSymbolicLinks(), + ); + expect(root.listSync().whereType(), isEmpty); + expect(await db.database.rawQuery('PRAGMA integrity_check'), [ + {'integrity_check': 'ok'}, + ]); + db.failProjection = false; + final rows = await LibraryCollectionsDatabase.readDatabaseSnapshot( + db, + forceLegacy: true, + ); + expect(rows.wishlistRows, hasLength(80)); + }, + ); + + test( + 'bad native result and native failure remove their staged directories', + () async { + final db = await _fixture('${root.path}/library_collections.db', 80); + for (final fail in [true, false]) { + await expectLater( + LibraryCollectionsDatabase.readDatabaseSnapshot( + db, + temporaryDirectory: root, + nativeJobRunner: (request) async { + await File( + request['output_path'] as String, + ).writeAsString('partial'); + if (fail) throw StateError('cancelled'); + return { + 'published': true, + 'rows_path': request['output_path'], + 'counts': {'rows': -1}, + }; + }, + ), + throwsA(anything), + ); + expect(root.listSync().whereType(), isEmpty); + } + }, + ); +} diff --git a/test/library_collections_hydration_test.dart b/test/library_collections_hydration_test.dart new file mode 100644 index 00000000..c7cd4d5c --- /dev/null +++ b/test/library_collections_hydration_test.dart @@ -0,0 +1,352 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/models/library_collections.dart'; +import 'package:spotiflac_android/models/track.dart'; +import 'package:spotiflac_android/services/library_collections_database.dart'; +import 'package:spotiflac_android/services/library_collections_hydration.dart'; + +Map trackRow(int index) => { + 'track_key': 'example:$index', + 'track_json': jsonEncode( + Track( + id: '$index', + name: 'Song $index', + artistName: 'Artist $index', + albumName: 'Album', + duration: 180, + source: 'example', + coverUrl: 'https://example.com/cover-$index.jpg', + availability: ServiceAvailability( + tidal: true, + qobuz: false, + amazon: false, + deezer: false, + ), + ).toJson(), + ), + 'added_at': '2026-01-02T03:04:05Z', +}; + +LibraryCollectionsSnapshot collectionFixture(int count) => + LibraryCollectionsSnapshot( + wishlistRows: List.generate(count, trackRow), + lovedRows: [ + trackRow(2), + {'track_key': 'bad', 'track_json': '{'}, + ], + favoriteArtistRows: [ + { + 'artist_key': 'example:artist', + 'artist_json': jsonEncode({ + 'artistId': 'artist', + 'providerId': 'example', + 'name': 'Artist', + 'imageUrl': 'https://example.com/artist.jpg', + 'addedAt': '2025-12-01T00:00:00Z', + }), + 'added_at': '2026-01-01T00:00:00Z', + }, + {'artist_key': 'bad', 'artist_json': '[]'}, + ], + playlistRows: [ + { + 'id': 'playlist', + 'name': 'Playlist', + 'created_at': '2026-01-01T00:00:00Z', + 'updated_at': 'invalid', + 'preview_track_json': jsonEncode({'coverUrl': 'cover'}), + }, + {'id': ''}, + { + 'id': 'other', + 'name': 'Other', + 'created_at': '2026-01-01T00:00:00Z', + 'preview_track_json': '{', + }, + ], + playlistTrackRows: [ + {'playlist_id': 'playlist', 'track_key': 'example:2'}, + {'playlist_id': 'playlist', 'track_key': 'example:2'}, + {'playlist_id': 'playlist', 'track_key': 'example:3'}, + {'playlist_id': 'orphan', 'track_key': 'example:orphan'}, + {'playlist_id': '', 'track_key': 'example:ignored'}, + {'playlist_id': 'other', 'track_key': ''}, + ], + ); + +void main() { + test( + 'native row file preserves snapshot parity and is always cleaned', + () async { + final snapshot = collectionFixture(80); + final temporary = await Directory.systemTemp.createTemp( + 'collection-read-test-', + ); + final file = File('${temporary.path}/rows.ndjson'); + final rows = [ + for (final row in snapshot.wishlistRows) + {'kind': 'wishlist', 'row': row}, + for (final row in snapshot.lovedRows) {'kind': 'loved', 'row': row}, + for (final row in snapshot.playlistRows) + {'kind': 'playlist', 'row': row}, + for (final row in snapshot.playlistTrackRows) + {'kind': 'membership', 'row': row}, + for (final row in snapshot.favoriteArtistRows) + {'kind': 'artist', 'row': row}, + ]; + await file.writeAsString(rows.map(jsonEncode).join('\n')); + final native = LibraryCollectionsSnapshot( + wishlistRows: const [], + lovedRows: const [], + playlistRows: const [], + playlistTrackRows: const [], + favoriteArtistRows: const [], + nativeRowsPath: file.path, + nativeRowsDirectory: temporary.path, + nativeRowCount: rows.length, + ); + final actual = await hydrateLibraryCollectionsSnapshot(native); + expect( + jsonEncode(actual.toJson()), + jsonEncode(decodeLibraryCollectionsSnapshot(snapshot).toJson()), + ); + expect(actual.playlistById('playlist')!.trackKeys, { + 'example:2', + 'example:3', + }); + expect(await temporary.exists(), isFalse); + for (final content in [ + 'not json', + '${jsonEncode(rows.first)}\n', + jsonEncode({'kind': 'unknown', 'row': {}}), + ]) { + final directory = await Directory.systemTemp.createTemp( + 'collection-read-test-', + ); + final path = '${directory.path}/rows.ndjson'; + await File(path).writeAsString(content); + await expectLater( + hydrateLibraryCollectionsSnapshot( + LibraryCollectionsSnapshot( + wishlistRows: const [], + lovedRows: const [], + playlistRows: const [], + playlistTrackRows: const [], + favoriteArtistRows: const [], + nativeRowsPath: path, + nativeRowsDirectory: directory.path, + nativeRowCount: rows.length, + ), + ), + throwsA(isA()), + ); + expect(await directory.exists(), isFalse); + } + }, + ); + + test( + 'native playlist rows preserve track index and remove their staging', + () async { + final directory = await Directory.systemTemp.createTemp( + 'collection-read-playlist-', + ); + final path = '${directory.path}/rows.ndjson'; + await File(path).writeAsString( + List.generate( + 80, + (index) => + jsonEncode({'kind': 'playlist_track', 'row': trackRow(index)}), + ).join('\n'), + ); + final known = List.generate(80, (index) => 'example:$index').toSet(); + final loaded = await hydratePlaylistTracksSnapshot( + LibraryPlaylistTracksSnapshot( + nativeRowsPath: path, + nativeRowsDirectory: directory.path, + nativeRowCount: 80, + ), + knownKeys: known, + ); + expect(loaded.tracks.length, 80); + expect(loaded.membershipUnchanged, isTrue); + expect(loaded.trackKeys, known); + expect(await directory.exists(), isFalse); + }, + ); + + test( + 'export worker keeps category/playlist/track order and original snapshot', + () async { + final base = decodeLibraryCollectionsSnapshot(collectionFixture(20000)); + final entries = base.wishlist.take(80).toList(); + final state = base.copyWith( + playlists: [ + base.playlists.first.copyWith( + tracks: entries, + coverImagePath: () => '/covers/custom.jpg', + ), + base.playlists.last.copyWith(tracks: entries.reversed.toList()), + ], + ); + final expected = jsonEncode(state.toJson()); + var ticks = 0; + final timer = Timer.periodic( + const Duration(milliseconds: 1), + (_) => ticks++, + ); + final pending = exportLibraryCollectionsState(state); + final newer = state.copyWith(wishlist: const [], playlists: const []); + final actual = await pending; + timer.cancel(); + expect(jsonEncode(actual), expected); + final json = await exportLibraryCollectionsJson(state); + expect(json, expected); + expect(jsonDecode(json), jsonDecode(expected)); + expect(newer.wishlistCount, 0); + expect(ticks, greaterThan(1)); + }, + ); + + test( + 'JSON fragment worker preserves escaping and small empty categories', + () async { + final state = LibraryCollectionsState( + wishlist: [ + CollectionTrackEntry( + key: 'example:escaped', + addedAt: DateTime.utc(2026), + track: const Track( + id: 'escaped', + name: '"音楽"\n\\line\t', + artistName: 'İstanbul 🎵', + albumName: '', + duration: 180, + comment: 'comma , colon : braces {} bracket []', + source: 'example', + ), + ), + ], + ); + final json = await exportLibraryCollectionsJson(state); + expect(json, jsonEncode(state.toJson())); + expect((jsonDecode(json) as Map)['loved'], isEmpty); + expect( + await exportLibraryCollectionsJson(LibraryCollectionsState()), + jsonEncode(LibraryCollectionsState().toJson()), + ); + }, + ); + + for (final count in [3, 80]) { + test( + 'collection hydration preserves row order and lazy indices ($count)', + () async { + final snapshot = collectionFixture(count); + final expected = decodeLibraryCollectionsSnapshot(snapshot); + final actual = await hydrateLibraryCollectionsSnapshot(snapshot); + expect(jsonEncode(actual.toJson()), jsonEncode(expected.toJson())); + expect( + actual.wishlist.map((entry) => entry.track.id), + List.generate(count, (index) => '$index'), + ); + expect(actual.loved.map((entry) => entry.key), ['example:2']); + expect(actual.favoriteArtists.single.addedAt, DateTime.utc(2025, 12)); + expect(actual.containsFavoriteArtistKey('example:artist'), isTrue); + final playlist = actual.playlistById('playlist')!; + expect(playlist.trackKeys, {'example:2', 'example:3'}); + expect(playlist.tracks, isEmpty); + expect(playlist.tracksLoaded, isFalse); + expect(playlist.previewCover, 'cover'); + expect(playlist.updatedAt, playlist.createdAt); + expect(actual.playlistById('other')!.previewCover, isNull); + expect(actual.isTrackInAnyPlaylist('example:2'), isTrue); + expect(actual.isTrackInAnyPlaylist('example:orphan'), isFalse); + expect(actual.isLoaded, isTrue); + }, + ); + } + + test( + 'playlist hydration retains duplicates and removes corrupt memberships', + () async { + final rows = [ + ...List.generate(80, trackRow), + trackRow(0), + {'track_key': 'corrupt', 'track_json': 'not json'}, + {'track_key': '', 'track_json': '{}'}, + {'track_key': 'array', 'track_json': '[]'}, + {'track_key': 'missing-fields', 'track_json': '{}'}, + ]; + final validKeys = List.generate(80, (index) => 'example:$index').toSet(); + final corrupted = await hydratePlaylistTracks( + rows, + knownKeys: {...validKeys, 'corrupt'}, + ); + expect(corrupted.tracks.length, 81); + expect(corrupted.tracks.last.key, 'example:0'); + expect(corrupted.trackKeys, validKeys); + expect(corrupted.membershipUnchanged, isFalse); + final valid = await hydratePlaylistTracks(rows, knownKeys: validKeys); + expect(valid.membershipUnchanged, isTrue); + final playlist = UserPlaylistCollection( + id: 'playlist', + name: 'Playlist', + createdAt: DateTime.utc(2026), + updatedAt: DateTime.utc(2026), + tracks: const [], + tracksLoaded: false, + trackKeys: validKeys, + ); + final loaded = playlist.withHydratedTracks( + tracks: valid.tracks, + trackKeys: valid.trackKeys, + membershipUnchanged: valid.membershipUnchanged, + ); + expect(loaded.sharesMembershipIndexWith(playlist), isTrue); + expect(loaded.tracksLoaded, isTrue); + expect(loaded.trackCount, 80); + }, + ); + + test('large collection parsing leaves caller timers able to run', () async { + final snapshot = collectionFixture(20000); + var ticks = 0; + final timer = Timer.periodic( + const Duration(milliseconds: 1), + (_) => ticks++, + ); + final loaded = await hydrateLibraryCollectionsSnapshot(snapshot); + timer.cancel(); + expect(loaded.wishlistCount, 20000); + expect(loaded.containsWishlistKey('example:19999'), isTrue); + // The old synchronous parsing path cannot service this timer while parsing. + expect(ticks, greaterThan(1)); + }); + + test( + 'worker failure releases a pending batch and allows subsequent loads', + () async { + final valid = collectionFixture(80); + final invalid = LibraryCollectionsSnapshot( + wishlistRows: valid.wishlistRows, + lovedRows: valid.lovedRows, + favoriteArtistRows: valid.favoriteArtistRows, + playlistRows: valid.playlistRows, + playlistTrackRows: [ + {'playlist_id': 123, 'track_key': 'invalid-type'}, + ], + ); + await expectLater( + hydrateLibraryCollectionsSnapshot(invalid), + throwsA(isA()), + ); + final subsequent = await hydrateLibraryCollectionsSnapshot(valid); + expect(subsequent.wishlistCount, 80); + expect(subsequent.playlistById('playlist')!.trackCount, 2); + }, + ); +} diff --git a/test/library_collections_mutation_test.dart b/test/library_collections_mutation_test.dart index df7980e7..eb634b74 100644 --- a/test/library_collections_mutation_test.dart +++ b/test/library_collections_mutation_test.dart @@ -1,4 +1,6 @@ import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -11,6 +13,13 @@ class _CollectionsDatabase implements LibraryCollectionsDatabase { final loved = >{}; final artists = >{}; final playlists = >{}; + final playlistTracks = >>{}; + final playlistReadStarted = Completer(); + final releasePlaylistRead = Completer(); + int playlistReads = 0; + LibraryPlaylistTracksSnapshot? stagedPlaylistRows; + Future? pendingSnapshot; + final snapshotReadStarted = Completer(); final events = []; final writeStarted = Completer(); final releaseWrite = Completer(); @@ -33,14 +42,56 @@ class _CollectionsDatabase implements LibraryCollectionsDatabase { Future migrateFromSharedPreferences() async => false; @override - Future loadSnapshot() async => - LibraryCollectionsSnapshot( - wishlistRows: wishlist.values.toList(), - lovedRows: loved.values.toList(), - playlistRows: playlists.values.toList(), - playlistTrackRows: const [], - favoriteArtistRows: artists.values.toList(), - ); + Future loadSnapshot() async { + if (!snapshotReadStarted.isCompleted) snapshotReadStarted.complete(); + return pendingSnapshot ?? + Future.value( + LibraryCollectionsSnapshot( + wishlistRows: wishlist.values.toList(), + lovedRows: loved.values.toList(), + playlistRows: playlists.values.toList(), + playlistTrackRows: [ + for (final entry in playlistTracks.entries) + for (final row in entry.value) + {'playlist_id': entry.key, 'track_key': row['track_key']}, + ], + favoriteArtistRows: artists.values.toList(), + ), + ); + } + + @override + Future>> loadPlaylistTracks( + String playlistId, + ) async { + playlistReads++; + if (!playlistReadStarted.isCompleted) { + playlistReadStarted.complete(); + await releasePlaylistRead.future; + } + return playlistTracks[playlistId] ?? const []; + } + + @override + Future loadPlaylistTracksSnapshot( + String playlistId, + ) async { + final rows = await loadPlaylistTracks(playlistId); + return stagedPlaylistRows ?? LibraryPlaylistTracksSnapshot(rows: rows); + } + + @override + Future renamePlaylist({ + required String playlistId, + required String name, + required String updatedAt, + }) => _write('rename:$playlistId', () { + playlists[playlistId] = { + ...playlists[playlistId]!, + 'name': name, + 'updated_at': updatedAt, + }; + }); Future _upsertTrack( Map> rows, @@ -138,6 +189,108 @@ ProviderContainer _container(_CollectionsDatabase db) { } void main() { + test('disposed snapshot read removes native rows before returning', () async { + final directory = await Directory.systemTemp.createTemp( + 'collection-read-disposed-', + ); + final path = '${directory.path}/rows.ndjson'; + await File(path).writeAsString('must never be parsed'); + final reply = Completer(); + final db = _CollectionsDatabase()..pendingSnapshot = reply.future; + final container = _container(db); + final notifier = container.read(libraryCollectionsProvider.notifier); + final done = notifier.ensurePlaylistLoaded('missing'); + await db.snapshotReadStarted.future; + container.dispose(); + reply.complete( + LibraryCollectionsSnapshot( + wishlistRows: const [], + lovedRows: const [], + playlistRows: const [], + playlistTrackRows: const [], + favoriteArtistRows: const [], + nativeRowsPath: path, + nativeRowsDirectory: directory.path, + nativeRowCount: 1, + ), + ); + await done; + expect(await directory.exists(), isFalse); + }); + + test( + 'playlist worker load is single-flight and retains intervening rename', + () async { + final db = _CollectionsDatabase(); + db.playlists['playlist'] = { + 'id': 'playlist', + 'name': 'Original', + 'created_at': '2026-01-01T00:00:00Z', + 'updated_at': '2026-01-01T00:00:00Z', + }; + db.playlistTracks['playlist'] = List.generate( + 100, + (index) => { + 'track_key': 'example:$index', + 'track_json': jsonEncode(_track('$index')), + 'added_at': '2026-01-01T00:00:00Z', + }, + ); + final container = _container(db); + final notifier = container.read(libraryCollectionsProvider.notifier); + await notifier.ensurePlaylistsLoaded(const []); + final first = notifier.ensurePlaylistLoaded('playlist'); + await db.playlistReadStarted.future; + final second = notifier.ensurePlaylistLoaded('playlist'); + final rename = notifier.renamePlaylist('playlist', 'Renamed'); + await db.writeStarted.future; + db.releaseWrite.complete(); + await rename; + db.releasePlaylistRead.complete(); + await Future.wait([first, second]); + final playlist = container + .read(libraryCollectionsProvider) + .playlistById('playlist')!; + expect(db.playlistReads, 1); + expect(playlist.name, 'Renamed'); + expect(playlist.tracksLoaded, isTrue); + expect(playlist.trackCount, 100); + expect( + playlist.tracks.map((entry) => entry.track.id), + List.generate(100, (index) => '$index'), + ); + }, + ); + + test('disposed playlist read cannot publish a worker result', () async { + final db = _CollectionsDatabase(); + final directory = await Directory.systemTemp.createTemp( + 'collection-read-disposed-', + ); + final path = '${directory.path}/rows.ndjson'; + await File(path).writeAsString('must never be parsed'); + db.stagedPlaylistRows = LibraryPlaylistTracksSnapshot( + nativeRowsPath: path, + nativeRowsDirectory: directory.path, + nativeRowCount: 1, + ); + db.playlists['playlist'] = { + 'id': 'playlist', + 'name': 'Playlist', + 'created_at': '2026-01-01T00:00:00Z', + }; + final container = _container(db); + final notifier = container.read(libraryCollectionsProvider.notifier); + await notifier.ensurePlaylistsLoaded(const []); + final loading = notifier.ensurePlaylistLoaded('playlist'); + await db.playlistReadStarted.future; + container.dispose(); + db.releasePlaylistRead.complete(); + await loading; + expect(db.playlistReads, 1); + expect(await directory.exists(), isFalse); + }); + test( 'late write completion does not publish or start a disposed mutation', () async { diff --git a/test/support/library_collections_benchmark.dart b/test/support/library_collections_benchmark.dart new file mode 100644 index 00000000..d08453ca --- /dev/null +++ b/test/support/library_collections_benchmark.dart @@ -0,0 +1,334 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:isolate'; + +import 'package:crypto/crypto.dart'; +import 'package:flutter/foundation.dart'; +import 'package:spotiflac_android/models/library_collections.dart'; +import 'package:spotiflac_android/models/track.dart'; +import 'package:spotiflac_android/services/library_collections_database.dart'; +import 'package:spotiflac_android/services/library_collections_hydration.dart'; +import 'package:spotiflac_android/services/library_playlist_search.dart'; +import 'package:spotiflac_android/services/library_search.dart'; + +LibraryCollectionsSnapshot _fixture(int count) => LibraryCollectionsSnapshot( + wishlistRows: [ + for (var index = 0; index < count; index++) + { + 'track_key': 'example:$index', + 'track_json': jsonEncode( + Track( + id: '$index', + name: 'Song $index', + artistName: 'Artist ${index % 997}', + albumName: 'Album ${index ~/ 12}', + duration: 180 + index % 240, + source: 'example', + coverUrl: 'https://example.test/$index.jpg', + isrc: 'XX000${index.toString().padLeft(7, '0')}', + ).toJson(), + ), + 'added_at': '2026-10-07T00:00:00Z', + }, + ], + lovedRows: const [], + favoriteArtistRows: const [], + playlistRows: [ + for (var index = 0; index < 4; index++) + { + 'id': 'playlist-$index', + 'name': 'Playlist $index', + 'created_at': '2026-10-07T00:00:00Z', + 'updated_at': '2026-10-07T00:00:00Z', + }, + ], + playlistTrackRows: [ + for (var index = 0; index < count * 2; index++) + {'playlist_id': 'playlist-${index % 4}', 'track_key': 'example:$index'}, + ], +); + +String _stateDigest(LibraryCollectionsState state) => sha256 + .convert( + utf8.encode( + jsonEncode({ + 'state': state.toJson(), + 'membership': [ + for (final playlist in state.playlists) + {'id': playlist.id, 'keys': playlist.trackKeys.toList()..sort()}, + ], + }), + ), + ) + .toString(); + +Future<({T value, Map timing})> measureCollectionOperation( + FutureOr Function() operation, +) async { + final watch = Stopwatch()..start(); + var beats = 0; + var last = 0; + var maximumGap = 0; + final timer = Timer.periodic(const Duration(milliseconds: 10), (_) { + final now = watch.elapsedMicroseconds; + if (now - last > maximumGap) maximumGap = now - last; + last = now; + beats++; + }); + late T value; + try { + value = await operation(); + } finally { + watch.stop(); + timer.cancel(); + } + if (watch.elapsedMicroseconds - last > maximumGap) { + maximumGap = watch.elapsedMicroseconds - last; + } + return ( + value: value, + timing: { + 'microseconds': watch.elapsedMicroseconds, + 'heartbeat_count': beats, + 'maximum_gap_us': maximumGap, + }, + ); +} + +Future> benchmarkCollectionHydration({ + int rows = 40000, +}) async { + final snapshot = await compute(_fixture, rows); + final baselineDigest = await compute( + _stateDigest, + decodeLibraryCollectionsSnapshot(snapshot), + ); + await hydrateLibraryCollectionsSnapshot(snapshot); + final baseline = >[]; + final worker = >[]; + for (var sample = 0; sample < 5; sample++) { + for (final useWorker in sample.isEven ? [false, true] : [true, false]) { + final result = await measureCollectionOperation( + () => useWorker + ? hydrateLibraryCollectionsSnapshot(snapshot) + : decodeLibraryCollectionsSnapshot(snapshot), + ); + final digest = await compute(_stateDigest, result.value); + if (digest != baselineDigest) { + throw StateError('Collection parity failed'); + } + (useWorker ? worker : baseline).add(result.timing); + } + } + return { + 'rows': rows, + 'membership_rows': rows * 2, + 'baseline': baseline, + 'worker': worker, + 'digest': baselineDigest, + 'note': 'Paired hydration latency and caller heartbeat, not displayed FPS.', + }; +} + +List _searchFixture(int count) => [ + for (var index = count - 1; index >= 0; index--) + UserPlaylistCollection( + id: '$index', + name: switch (index % 4) { + 0 => 'rock', + 1 => 'rock mix $index', + 2 => 'my rock $index', + _ => 'mix rock $index', + }, + createdAt: DateTime.utc(2026), + updatedAt: DateTime.utc(2026), + tracks: const [], + tracksLoaded: false, + trackKeys: {'example:$index'}, + ), +]; + +List _baselineSearch(List playlists) { + final query = LibrarySearchQuery('rock'); + final matches = playlists.where((p) => query.matches(p.name)).toList() + ..sort((a, b) { + final rank = query.rank(a.name).compareTo(query.rank(b.name)); + if (rank != 0) return rank; + final title = a.name.toLowerCase().compareTo(b.name.toLowerCase()); + return title != 0 ? title : a.id.compareTo(b.id); + }); + return matches + .skip(40) + .take(40) + .map( + (p) => LibrarySearchHit( + kind: LibrarySearchKind.playlists, + id: p.id, + title: p.name, + cover: p.coverImagePath ?? p.previewCover, + trackCount: p.trackCount, + ), + ) + .toList(growable: false); +} + +Future> benchmarkPlaylistSearch({int rows = 25000}) async { + final playlists = await compute(_searchFixture, rows); + final expected = _baselineSearch(playlists).map((hit) => hit.id).join(','); + await searchLibraryPlaylists( + playlists: playlists, + query: 'rock', + limit: 40, + offset: 40, + ); + final baseline = >[]; + final worker = >[]; + for (var sample = 0; sample < 5; sample++) { + for (final useWorker in sample.isEven ? [false, true] : [true, false]) { + final result = await measureCollectionOperation>( + () => useWorker + ? searchLibraryPlaylists( + playlists: playlists, + query: 'rock', + limit: 40, + offset: 40, + ) + : _baselineSearch(playlists), + ); + if (result.value.map((hit) => hit.id).join(',') != expected) { + throw StateError('Playlist search parity failed'); + } + (useWorker ? worker : baseline).add(result.timing); + } + } + return { + 'rows': rows, + 'offset': 40, + 'limit': 40, + 'baseline': baseline, + 'worker': worker, + 'note': 'Original full sort versus production bounded search and worker.', + }; +} + +Future> benchmarkPlaylistMetadataCopy({ + int rows = 100000, +}) async { + final state = await compute( + decodeLibraryCollectionsSnapshot, + await compute(_fixture, rows), + ); + final playlists = state.playlists; + final baseline = >[]; + final optimized = >[]; + for (var sample = 0; sample < 5; sample++) { + for (final reuse in sample.isEven ? [false, true] : [true, false]) { + final result = await measureCollectionOperation(() { + late LibraryCollectionsState updated; + for (var index = 0; index < 10; index++) { + final renamed = [ + playlists.first.copyWith(name: 'Renamed $index'), + ...playlists.skip(1), + ]; + updated = reuse + ? state.copyWith(playlists: renamed) + : LibraryCollectionsState( + wishlist: state.wishlist, + loved: state.loved, + favoriteArtists: state.favoriteArtists, + playlists: renamed, + isLoaded: state.isLoaded, + ); + } + return updated; + }); + if (!result.value.isTrackInAnyPlaylist('example:${rows * 2 - 1}') || + result.value.playlists.first.name != 'Renamed 9') { + throw StateError('Metadata membership parity failed'); + } + (reuse ? optimized : baseline).add(result.timing); + } + } + return { + 'membership_rows': rows * 2, + 'edits_per_sample': 10, + 'baseline': baseline, + 'optimized': optimized, + }; +} + +Future> benchmarkCollectionExport({ + int rows = 40000, +}) async { + final state = await compute( + decodeLibraryCollectionsSnapshot, + await compute(_fixture, rows), + ); + final expected = await compute(_encodedState, state); + await exportLibraryCollectionsState(state); + final baseline = >[]; + final worker = >[]; + for (var sample = 0; sample < 5; sample++) { + for (final useWorker in sample.isEven ? [false, true] : [true, false]) { + final result = await measureCollectionOperation>( + () => useWorker ? exportLibraryCollectionsState(state) : state.toJson(), + ); + final digest = await compute(_jsonDigest, result.value); + if (digest != expected) { + throw StateError('Collection export parity failed'); + } + (useWorker ? worker : baseline).add(result.timing); + } + } + return { + 'rows': rows, + 'baseline': baseline, + 'worker': worker, + 'note': + 'Model-to-map conversion; worker prioritizes caller responsiveness.', + }; +} + +String _encodedState(LibraryCollectionsState state) => + _jsonDigest(state.toJson()); +String _jsonDigest(Map value) => + sha256.convert(utf8.encode(jsonEncode(value))).toString(); + +Future _encodeExportMap(Map value) => + Isolate.run(() => jsonEncode(value)); +Future _legacyJsonExport(LibraryCollectionsState state) => + _encodeExportMap(state.toJson()); + +Future> benchmarkCollectionJsonExport({ + int rows = 40000, +}) async { + final state = await compute( + decodeLibraryCollectionsSnapshot, + await compute(_fixture, rows), + ); + final expected = await _legacyJsonExport(state); + await exportLibraryCollectionsJson(state); + final baseline = >[]; + final worker = >[]; + for (var sample = 0; sample < 5; sample++) { + for (final useWorker in sample.isEven ? [false, true] : [true, false]) { + final result = await measureCollectionOperation( + () => useWorker + ? exportLibraryCollectionsJson(state) + : _legacyJsonExport(state), + ); + if (result.value != expected) { + throw StateError('Collection JSON export parity failed'); + } + (useWorker ? worker : baseline).add(result.timing); + } + } + return { + 'rows': rows, + 'json_characters': expected.length, + 'baseline': baseline, + 'worker': worker, + 'note': + 'Full model-to-encoded-JSON: original UI map+encoding isolate versus bounded single worker JSON.', + }; +}