From faa08f5b56483284503c4e71fa94feca2aaf3dd6 Mon Sep 17 00:00:00 2001 From: zarzet <42882290+zarzet@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:47:56 +0700 Subject: [PATCH] fix(library): prune stale NAS rows outside failed scan paths --- lib/services/library_cleanup.dart | 56 ++++++++++++++++--- lib/services/library_ingestion_store.dart | 3 + lib/services/library_native_adapter.dart | 17 ++++-- lib/services/network_library_scanner.dart | 18 +++++- .../platform_bridge/bridge_library_scan.dart | 5 ++ test/library_native_adapter_test.dart | 46 +++++++++++++++ test/network_library_incremental_test.dart | 54 +++++++++++++++++- 7 files changed, 183 insertions(+), 16 deletions(-) diff --git a/lib/services/library_cleanup.dart b/lib/services/library_cleanup.dart index 29bda62b3..a9f9865d6 100644 --- a/lib/services/library_cleanup.dart +++ b/lib/services/library_cleanup.dart @@ -9,22 +9,64 @@ import 'package:mornye/utils/logger.dart'; /// Removes rows superseded by a staged scan inside the caller's transaction. /// On partial scans, an unreadable file is not evidence that it was deleted. +({String where, List args}) libraryScanReplacementPredicate( + String sourceId, { + required String stageTable, + required bool preserveMissing, + List? preserveMissingPaths, +}) { + final args = [sourceId]; + if (!preserveMissing) return (where: 'source_id = ?', args: args); + final replaced = + 'id IN (SELECT id FROM $stageTable) ' + 'OR file_path IN (SELECT file_path FROM $stageTable)'; + if (preserveMissingPaths == null || preserveMissingPaths.length > 128) { + return (where: 'source_id = ? AND ($replaced)', args: args); + } + if (preserveMissingPaths.isEmpty) { + return (where: 'source_id = ?', args: args); + } + final protected = []; + for (final path in preserveMissingPaths) { + if (path.endsWith('/')) { + // Literal prefixes preserve percent signs and underscores in NAS names. + protected.add('substr(file_path, 1, length(?)) = ?'); + args.addAll([path, path]); + } else { + protected.add('file_path = ?'); + args.add(path); + } + } + return ( + where: + 'source_id = ? AND ($replaced OR ' + 'NOT (${protected.join(' OR ')}))', + args: args, + ); +} + Future deleteReplacedLibraryScanRows( DatabaseExecutor db, String sourceId, { required String stageTable, required bool preserveMissing, + List? preserveMissingPaths, }) async { - final where = preserveMissing - ? 'source_id = ? AND (id IN (SELECT id FROM $stageTable) ' - 'OR file_path IN (SELECT file_path FROM $stageTable))' - : 'source_id = ?'; + final predicate = libraryScanReplacementPredicate( + sourceId, + stageTable: stageTable, + preserveMissing: preserveMissing, + preserveMissingPaths: preserveMissingPaths, + ); await db.rawDelete( 'DELETE FROM library_path_keys WHERE item_id IN ' - '(SELECT id FROM library WHERE $where)', - [sourceId], + '(SELECT id FROM library WHERE ${predicate.where})', + predicate.args, + ); + await db.rawDelete( + 'DELETE FROM library WHERE ${predicate.where}', + predicate.args, ); - await db.rawDelete('DELETE FROM library WHERE $where', [sourceId]); } /// Deletes a selection atomically without exceeding SQLite's parameter budget. diff --git a/lib/services/library_ingestion_store.dart b/lib/services/library_ingestion_store.dart index 697b0d936..28937b6ca 100644 --- a/lib/services/library_ingestion_store.dart +++ b/lib/services/library_ingestion_store.dart @@ -98,6 +98,7 @@ class LibraryIngestionStore { sourceId, validatedRows(), preserveMissing: scan.errorCount > 0, + preserveMissingPaths: scan.preserveMissingPaths, ); } @@ -362,6 +363,7 @@ class LibraryIngestionStore { Stream> items, { int batchSize = 300, bool preserveMissing = false, + List? preserveMissingPaths, }) async { if (batchSize <= 0) { throw ArgumentError.value(batchSize, 'batchSize', 'Must be positive'); @@ -459,6 +461,7 @@ class LibraryIngestionStore { sourceId, stageTable: _scanStageTable, preserveMissing: preserveMissing, + preserveMissingPaths: preserveMissingPaths, ); await txn.rawDelete(''' DELETE FROM library_path_keys diff --git a/lib/services/library_native_adapter.dart b/lib/services/library_native_adapter.dart index 7a5e888c5..7047f9404 100644 --- a/lib/services/library_native_adapter.dart +++ b/lib/services/library_native_adapter.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; import 'package:sqflite/sqflite.dart'; +import 'package:mornye/services/library_cleanup.dart'; import 'package:mornye/services/platform_bridge.dart'; import 'package:mornye/services/library_schema.dart'; import 'package:mornye/services/sqlite_helpers.dart' as sqlite; @@ -180,6 +181,7 @@ class LibraryNativeAdapter { }, incremental: false, preserveMissing: scan.errorCount > 0, + preserveMissingPaths: scan.preserveMissingPaths, requestId: requestId, isCancelled: isCancelled, ); @@ -210,6 +212,7 @@ class LibraryNativeAdapter { Map operation, { required bool incremental, required bool preserveMissing, + List? preserveMissingPaths, required String requestId, required bool Function() isCancelled, }) async { @@ -236,6 +239,7 @@ class LibraryNativeAdapter { result, incremental: incremental, preserveMissing: preserveMissing, + preserveMissingPaths: preserveMissingPaths, isCancelled: isCancelled, ); } on NativeSqliteSnapshotDetachException catch (error) { @@ -253,6 +257,7 @@ class LibraryNativeAdapter { Map nativeResult, { required bool incremental, required bool preserveMissing, + List? preserveMissingPaths, required bool Function() isCancelled, }) => _attached(privatePath, (alias) async { return sqlite.transactionWithBusyRetry(database, (txn) async { @@ -311,13 +316,13 @@ class LibraryNativeAdapter { [sourceId], ); } else { - await deleteWhere( - preserveMissing - ? 'source_id = ? AND (id IN(SELECT id FROM $alias.library) ' - 'OR file_path IN(SELECT file_path FROM $alias.library))' - : 'source_id = ?', - [sourceId], + final predicate = libraryScanReplacementPredicate( + sourceId, + stageTable: '$alias.library', + preserveMissing: preserveMissing, + preserveMissingPaths: preserveMissingPaths, ); + await deleteWhere(predicate.where, predicate.args); } _check(isCancelled); await txn.rawDelete( diff --git a/lib/services/network_library_scanner.dart b/lib/services/network_library_scanner.dart index 9f29c121e..180d6bd96 100644 --- a/lib/services/network_library_scanner.dart +++ b/lib/services/network_library_scanner.dart @@ -17,7 +17,7 @@ final _log = AppLogger('LocalLibrary'); /// Bounded tag prefetch; only NDJSON and artwork are stored locally. /// Failed listings never count as empty directories. Partial scans retain old -/// rows through the same ingestion policy used for local/SAF scans. +/// rows only under unreadable folders or at unreadable files. class NetworkLibraryScanner { NetworkLibraryScanner({ NetworkStorageService? storage, @@ -108,6 +108,19 @@ class NetworkLibraryScanner { // Fail before creating scan output if the root cannot be listed. List? first = await _storage.list(connection, root); await checkpoint(); + List? preserveMissingPaths = []; + void preserve(String path) { + final paths = preserveMissingPaths; + if (paths == null) return; + // Keep failed-scan memory and publication SQL bounded. Widespread + // failures fall back to retaining the whole previous source snapshot. + if (paths.length >= 128) { + preserveMissingPaths = null; + } else { + paths.add(NetworkStorageService.source(connection.id, path)); + } + } + final temp = await Directory( '${(await _temporaryDirectory()).path}/network_scans', ).create(recursive: true); @@ -171,6 +184,7 @@ class NetworkLibraryScanner { first = null; } catch (error) { errors++; + preserve(folder); _log.e('Network folder listing failed [$folder]', error); onProgress(processed, errors, folder); continue; @@ -333,6 +347,7 @@ class NetworkLibraryScanner { // Cancellation must abort the whole staging operation. await checkpoint(); errors++; + preserve(entry.path); _log.e('Network track scan failed [$trackSource]', error); } if (encodedRow != null) { @@ -374,6 +389,7 @@ class NetworkLibraryScanner { file: file, expectedCount: written, errorCount: errors, + preserveMissingPaths: preserveMissingPaths, ); } catch (_) { await metadata?.cancelPending(); diff --git a/lib/services/platform_bridge/bridge_library_scan.dart b/lib/services/platform_bridge/bridge_library_scan.dart index 03ef434a8..818ed3088 100644 --- a/lib/services/platform_bridge/bridge_library_scan.dart +++ b/lib/services/platform_bridge/bridge_library_scan.dart @@ -5,10 +5,15 @@ class LibraryScanNDJSONFile { final int expectedCount; final int errorCount; + /// Failed network files, or folder prefixes ending in '/'. Null retains the + /// conservative whole-source policy used by local and SAF scans. + final List? preserveMissingPaths; + const LibraryScanNDJSONFile({ required this.file, required this.expectedCount, this.errorCount = 0, + this.preserveMissingPaths, }); Stream> rows() async* { diff --git a/test/library_native_adapter_test.dart b/test/library_native_adapter_test.dart index 470a46947..5bc052b8e 100644 --- a/test/library_native_adapter_test.dart +++ b/test/library_native_adapter_test.dart @@ -218,6 +218,7 @@ void main() { Future scan( List> rows, { int errors = 0, + List? preserveMissingPaths, }) async { final file = File('${root.path}/scan.ndjson'); await file.writeAsString(rows.map(jsonEncode).join('\n')); @@ -225,6 +226,7 @@ void main() { file: file, expectedCount: rows.length, errorCount: errors, + preserveMissingPaths: preserveMissingPaths, ); } @@ -319,6 +321,50 @@ void main() { }, ); + test('partial NAS publication retains only unreadable scopes', () async { + const prefix = 'network://nas/music/'; + await addNativeLibraryTrack( + live, + nativeLibraryTrack('unreadable', path: '${prefix}unreadable.flac'), + ); + await addNativeLibraryTrack( + live, + nativeLibraryTrack('offline', path: '${prefix}offline%25_/keep.flac'), + ); + await addNativeLibraryTrack( + live, + nativeLibraryTrack('deleted', path: '${prefix}test/deleted.flac'), + ); + final refreshed = nativeLibraryTrack( + 'refreshed', + path: '${prefix}offline%25_/keep.flac', + ); + await adapter.importScanFile( + 'source', + await scan( + [refreshed], + errors: 2, + preserveMissingPaths: [ + '${prefix}offline%25_/', + '${prefix}unreadable.flac', + ], + ), + requestId: 'partial-nas', + isCancelled: () => false, + ); + expect( + (await live.query('library')).map((row) => row['id']), + unorderedEquals(['refreshed', 'unreadable', 'other']), + ); + expect( + (await live.query( + 'library_path_keys', + )).map((row) => row['item_id']).toSet(), + {'refreshed', 'unreadable', 'other'}, + ); + await cleaned(); + }); + test( 'incremental removes current downloads and exact source-scoped deleted paths', () async { diff --git a/test/network_library_incremental_test.dart b/test/network_library_incremental_test.dart index 776298c5d..21ec8e69c 100644 --- a/test/network_library_incremental_test.dart +++ b/test/network_library_incremental_test.dart @@ -94,6 +94,7 @@ void main() { Future scan({ bool force = false, bool commit = true, + String root = '', }) async { // A fresh reader per scan proves reuse comes from persisted Library // rows, rather than the playback service's short-lived memory cache. @@ -122,7 +123,7 @@ void main() { }, ); final result = await scanner.scan( - NetworkStorageService.source('nas', ''), + NetworkStorageService.source('nas', root), forceFullScan: force, checkpoint: () async { if (cancelled) throw StateError('cancelled'); @@ -134,6 +135,7 @@ void main() { 'source', result.rows(), preserveMissing: result.errorCount > 0, + preserveMissingPaths: result.preserveMissingPaths, ); } return result; @@ -263,7 +265,11 @@ void main() { final partial = await scan(); expect(partial.errorCount, 1); expect(reads, ['bad.flac']); - expect(await database.query('library'), hasLength(3)); + expect(await database.query('library'), hasLength(2)); + expect( + (await database.query('library')).map((row) => row['file_path']), + isNot(contains('network://nas/missing.flac')), + ); failedTracks.clear(); reads.clear(); await scan(); @@ -330,6 +336,50 @@ void main() { }, ); + test( + 'partial scan removes a deleted folder but retains failed scopes', + () async { + storage.tree[''] = [ + const NetworkEntry('test/', 'test', directory: true), + const NetworkEntry('offline%_/', 'offline%_', directory: true), + track('unreadable.flac'), + ]; + storage.tree['test/'] = [track('test/stale.flac')]; + storage.tree['offline%_/'] = [track('offline%_/keep.flac')]; + await scan(); + storage.tree['']!.removeAt(0); + storage.tree['']![1] = track('unreadable.flac', modified: 2000); + storage.failedFolders.add('offline%_/'); + failedTracks.add('unreadable.flac'); + final result = await scan(); + expect(result.errorCount, 2); + expect( + (await database.query('library')).map((row) => row['file_path']), + unorderedEquals([ + 'network://nas/offline%25_/keep.flac', + 'network://nas/unreadable.flac', + ]), + ); + }, + ); + + test('selected NAS folders stay scoped to their own subtree', () async { + storage.tree[''] = [ + const NetworkEntry('downloads/', 'downloads', directory: true), + const NetworkEntry('processed/', 'processed', directory: true), + ]; + storage.tree['downloads/'] = [track('downloads/original.flac')]; + storage.tree['processed/'] = [track('processed/converted.flac')]; + for (final folder in ['downloads/', 'processed/']) { + reads.clear(); + final result = await scan(root: folder, commit: false); + final rows = await result.rows().toList(); + expect(rows, hasLength(1)); + expect(rows.single['filePath'], startsWith('network://nas/$folder')); + expect(reads, hasLength(1)); + } + }); + test( 'cancellation after cache lookup discards staging and preserves rows', () async {