diff --git a/lib/providers/local_library_provider.dart b/lib/providers/local_library_provider.dart index ad39c0f69..358e38092 100644 --- a/lib/providers/local_library_provider.dart +++ b/lib/providers/local_library_provider.dart @@ -176,6 +176,7 @@ class LocalLibraryNotifier extends Notifier { bool _scanPauseRequested = false; bool _scanInProgress = false; Future? _scanCompletion; + NetworkLibraryScanner? _networkScanner; String? _nativeLibraryRequestId; final Set _pendingReconnectScans = {}; bool _drainingReconnectScans = false; @@ -200,6 +201,11 @@ class LocalLibraryNotifier extends Notifier { _scanCancelRequested = true; _scanPauseRequested = false; _pendingReconnectScans.clear(); + unawaited( + _networkScanner?.cancelPending().catchError((Object error) { + _log.w('Failed to cancel disposed network scan reads: $error'); + }), + ); if (_scanInProgress) { final requestId = _nativeLibraryRequestId; if (requestId != null) { @@ -508,6 +514,11 @@ class LocalLibraryNotifier extends Notifier { _log.i('Cancelling library scan'); _scanCancelRequested = true; + try { + await _networkScanner?.cancelPending(); + } catch (error) { + _log.w('Failed to cancel network scan reads: $error'); + } final requestId = _nativeLibraryRequestId; if (requestId != null) { try { diff --git a/lib/providers/local_library_provider_scan.dart b/lib/providers/local_library_provider_scan.dart index 916047950..9fae60cf4 100644 --- a/lib/providers/local_library_provider_scan.dart +++ b/lib/providers/local_library_provider_scan.dart @@ -48,6 +48,24 @@ extension _LocalLibraryScan on LocalLibraryNotifier { } } + Future _scanNetworkFolder( + String source, { + required Future Function() checkpoint, + required void Function(int processed, int errors, String name) onProgress, + }) async { + final scanner = NetworkLibraryScanner(); + _networkScanner = scanner; + try { + return await scanner.scan( + source, + checkpoint: checkpoint, + onProgress: onProgress, + ); + } finally { + if (identical(_networkScanner, scanner)) _networkScanner = null; + } + } + Future<({int inserted, int skipped})?> _replaceFromFullScanStream({ required String sourceId, required String folderPath, @@ -56,7 +74,7 @@ extension _LocalLibraryScan on LocalLibraryNotifier { }) async { if (_scanCancelRequested) return null; final scanFile = folderPath.startsWith('network://') - ? await NetworkLibraryScanner().scan( + ? await _scanNetworkFolder( folderPath, checkpoint: () async { while (_scanPauseRequested && diff --git a/lib/services/network_library_scanner.dart b/lib/services/network_library_scanner.dart index 003416636..39aec8087 100644 --- a/lib/services/network_library_scanner.dart +++ b/lib/services/network_library_scanner.dart @@ -39,11 +39,46 @@ class NetworkLibraryScanner { final int? _metadataConcurrency; final Future Function() _supportDirectory, _temporaryDirectory; final Future Function(File) _openOutput; + NetworkMetadataService? _activeMetadata; + bool _running = false; + bool _cancelled = false; + + /// Stop scan-owned native reads even while scan() is awaiting their result. + Future cancelPending() async { + _cancelled = true; + await _activeMetadata?.cancelPending(); + } Future scan( String source, { required Future Function() checkpoint, required void Function(int processed, int errors, String name) onProgress, + }) async { + if (_running) { + throw StateError('Network library scanner is already running'); + } + _running = true; + _cancelled = false; + try { + return await _scan( + source, + checkpoint: () async { + if (_cancelled) throw StateError('Network library scan cancelled'); + await checkpoint(); + if (_cancelled) throw StateError('Network library scan cancelled'); + }, + onProgress: onProgress, + ); + } finally { + _activeMetadata = null; + _running = false; + } + } + + Future _scan( + String source, { + required Future Function() checkpoint, + required void Function(int processed, int errors, String name) onProgress, }) async { final uri = Uri.parse(source); if (uri.scheme != 'network' || @@ -62,7 +97,8 @@ class NetworkLibraryScanner { } await checkpoint(); // Fail before creating scan output if the root cannot be listed. - final first = await _storage.list(connection, root); + List? first = await _storage.list(connection, root); + await checkpoint(); final temp = await Directory( '${(await _temporaryDirectory()).path}/network_scans', ).create(recursive: true); @@ -76,7 +112,6 @@ class NetworkLibraryScanner { var errors = 0; final pending = [root]; final visited = {}; - final seenFiles = {}; // Leave room for playback and avoid saturating a small NAS/router. final concurrency = _metadataConcurrency ?? @@ -88,6 +123,7 @@ class NetworkLibraryScanner { cacheDirectory: _temporaryDirectory, ) : null; + _activeMetadata = metadata; final read = _readMetadata ?? ((String source) => metadata!.read(source, forceRefresh: true)); @@ -115,11 +151,14 @@ class NetworkLibraryScanner { if (visited.length > 100000) { throw StateError('Too many network folders'); } - final List entries; + List? entries; try { entries = folder == root - ? first + ? first! : await _storage.list(connection, folder); + // The first listing can be very large. Release it before descending + // into children instead of retaining it for the entire scan. + first = null; } catch (error) { errors++; _log.e('Network folder listing failed [$folder]', error); @@ -131,14 +170,23 @@ class NetworkLibraryScanner { '${entries.where((entry) => entry.directory).length} folders, ' '${entries.where((entry) => entry.audio).length} audio files', ); - final iterator = entries.iterator; + // Each directory is visited once and only its direct children are + // accepted, so file deduplication need not retain every scanned path. + final seenFiles = {}; + Iterator? iterator = entries.iterator; + entries = null; Future fill() async { - while (prefetched.length < concurrency && iterator.moveNext()) { + while (prefetched.length < concurrency && iterator!.moveNext()) { await checkpoint(); - final entry = iterator.current; + final entry = iterator!.current; final path = networkPath(entry.path); // Reject provider parent links and paths outside this folder. if (!path.startsWith(folder) || path == folder) continue; + final child = path.substring(folder.length); + final name = entry.directory && child.endsWith('/') + ? child.substring(0, child.length - 1) + : child; + if (name.isEmpty || name.contains('/')) continue; if (entry.directory) { if (!path.endsWith('/')) { throw const FormatException('Invalid folder entry'); @@ -242,6 +290,10 @@ class NetworkLibraryScanner { onProgress(processed, errors, entry.name); await fill(); } + // Async closures may otherwise keep the completed iterator's backing + // list alive while awaiting the next directory listing. + iterator = null; + seenFiles.clear(); } await checkpoint(); await flush(); diff --git a/test/library_memory_release_test.dart b/test/library_memory_release_test.dart new file mode 100644 index 000000000..0e11d0ea1 --- /dev/null +++ b/test/library_memory_release_test.dart @@ -0,0 +1,95 @@ +import 'dart:developer' as developer; +import 'dart:io'; + +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/providers/download_history_provider.dart'; +import 'package:spotiflac_android/providers/library_browse_provider.dart'; +import 'package:spotiflac_android/providers/local_library_provider.dart'; +import 'package:spotiflac_android/providers/settings_provider.dart'; + +import 'support/fixed_settings.dart'; +import 'support/vm_service_probe.dart'; + +class _History extends DownloadHistoryNotifier { + @override + DownloadHistoryState build() => DownloadHistoryState(); +} + +class _Local extends LocalLibraryNotifier { + @override + LocalLibraryState build() => LocalLibraryState(); +} + +Future _browseAndClose(ProviderContainer container, String search) async { + final provider = libraryBrowseProvider(( + artists: false, + artist: null, + search: search, + sort: 'a-z', + limit: 40, + completeness: null, + )); + final subscription = container.listen(provider, (_, _) {}); + try { + await container.read(provider.future); + for (var page = 0; page < 9; page++) { + await container.read(provider.notifier).loadMore(); + } + } finally { + subscription.close(); + } + await container.pump(); + expect(container.exists(provider), isFalse); +} + +Future main() async { + TestWidgetsFlutterBinding.ensureInitialized(); + final service = (await developer.Service.getInfo()).serverUri; + test( + 'Library pages and old filter results are collectible after closing', + () async { + final overrides = HttpOverrides.current; + HttpOverrides.global = null; + addTearDown(() => HttpOverrides.global = overrides); + final entries = >[]; + final container = ProviderContainer( + overrides: [ + settingsProvider.overrideWith(FixedSettingsNotifier.new), + downloadHistoryProvider.overrideWith(_History.new), + localLibraryProvider.overrideWith(_Local.new), + libraryBrowsePageLoaderProvider.overrideWithValue((page) async { + final rows = [ + for (var i = page.offset; i < page.offset + 40 && i < 400; i++) + LibraryBrowseEntry( + source: 'local', + key: '$i', + name: 'Album $i', + artist: 'Artist', + samplePath: '/music/$i.flac', + trackCount: 1, + ), + ]; + entries.addAll(rows.map(WeakReference.new)); + return LibraryBrowsePage(rows); + }), + ], + ); + // Debug DevTools intentionally retain provider event snapshots. Exclude + // that history to measure the application-owned lifecycle as in release. + container.observers.clear(); + addTearDown(container.dispose); + for (var i = 0; i < 5; i++) { + await _browseAndClose(container, 'filter $i'); + } + expect(entries, hasLength(2000)); + await collectTestGarbage(service!); + expect( + entries.where((entry) => entry.target != null).length, + 0, + reason: 'Closed Library queries must not retain their loaded rows', + ); + }, + skip: service == null ? 'Requires flutter test --enable-vmservice' : false, + ); +} diff --git a/test/network_library_memory_test.dart b/test/network_library_memory_test.dart new file mode 100644 index 000000000..8f8b8a343 --- /dev/null +++ b/test/network_library_memory_test.dart @@ -0,0 +1,91 @@ +import 'dart:developer' as developer; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/services/network_library_scanner.dart'; +import 'package:spotiflac_android/services/network_storage_service.dart'; + +import 'support/vm_service_probe.dart'; + +class _TransientListingStorage extends NetworkStorageService { + _TransientListingStorage(this.protocol, this.collect); + + final NetworkProtocol protocol; + final Future Function() collect; + WeakReference>? rootListing; + bool? rootWasReleased; + + @override + Future connection(String id) async => NetworkConnection( + id: id, + name: 'Fixture', + protocol: protocol, + address: protocol == NetworkProtocol.smb + ? 'smb://example.test/music/' + : 'https://example.test/music/', + ); + + @override + Future> list( + NetworkConnection connection, [ + String path = '', + ]) async { + if (path.isEmpty) { + final entries = [ + const NetworkEntry('child/', 'child', directory: true), + for (var i = 0; i < 4000; i++) NetworkEntry('$i.flac', '$i.flac'), + ]; + rootListing = WeakReference(entries); + return entries; + } + await collect(); + rootWasReleased = rootListing!.target == null; + return [const NetworkEntry('child/song.flac', 'song.flac')]; + } +} + +Future main() async { + TestWidgetsFlutterBinding.ensureInitialized(); + final service = (await developer.Service.getInfo()).serverUri; + for (final protocol in [NetworkProtocol.smb, NetworkProtocol.webdav]) { + test( + '${protocol.name} scan releases completed directory listings before finishing', + () async { + final overrides = HttpOverrides.current; + HttpOverrides.global = null; + addTearDown(() => HttpOverrides.global = overrides); + final directory = await Directory.systemTemp.createTemp( + 'network-library-memory-', + ); + addTearDown(() => directory.delete(recursive: true)); + final storage = _TransientListingStorage( + protocol, + () => collectTestGarbage(service!), + ); + addTearDown(storage.dispose); + final result = + await NetworkLibraryScanner( + storage: storage, + readMetadata: (_) async => {'title': 'Track'}, + supportDirectory: () async => directory, + temporaryDirectory: () async => directory, + ).scan( + NetworkStorageService.source('fixture', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(result.expectedCount, 4001); + expect(result.errorCount, 0); + expect( + storage.rootWasReleased, + isTrue, + reason: 'A completed root listing must not stay alive in child scans', + ); + await result.delete(); + }, + skip: service == null + ? 'Requires flutter test --enable-vmservice' + : false, + ); + } +} diff --git a/test/network_library_scanner_test.dart b/test/network_library_scanner_test.dart index 79c64801f..2b6c8bc7d 100644 --- a/test/network_library_scanner_test.dart +++ b/test/network_library_scanner_test.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:spotiflac_android/services/network_library_scanner.dart'; import 'package:spotiflac_android/services/network_storage_service.dart'; @@ -72,6 +73,107 @@ void main() { ); } + test( + 'cancel interrupts active native metadata reads and drains their work', + () async { + const channel = MethodChannel('com.zarz.spotiflac/backend'); + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final jobs = >{}; + final cancelled = {}; + final started = Completer(); + var blockReads = true; + messenger.setMockMethodCallHandler(channel, (call) async { + final arguments = call.arguments as Map; + final id = arguments['request_id'] as String; + if (call.method == 'cancelNativeDataJob') { + final job = jobs[id]; + if (job != null && !job.isCompleted) { + cancelled.add(id); + job.completeError(PlatformException(code: 'CANCELLED')); + } + return null; + } + expect(call.method, 'runNativeDataJob'); + if (!blockReads) { + return jsonEncode({ + 'metadata': {'title': 'Track'}, + 'cacheable': true, + }); + } + final job = Completer(); + jobs[id] = job; + if (jobs.length == 2) started.complete(); + return job.future; + }); + addTearDown(() => messenger.setMockMethodCallHandler(channel, null)); + final storage = tracks(20); + addTearDown(storage.dispose); + final scanner = NetworkLibraryScanner( + storage: storage, + supportDirectory: () async => directory, + temporaryDirectory: () async => directory, + metadataConcurrency: 2, + ); + final scan = scanner.scan( + NetworkStorageService.source('nas', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + final stopped = expectLater(scan, throwsStateError); + await started.future; + await scanner.cancelPending(); + await stopped; + expect(jobs, hasLength(2)); + expect(cancelled, jobs.keys.toSet()); + expect(jobs.values.every((job) => job.isCompleted), isTrue); + await expectNoOutput(); + + // A completed cancellation does not poison a later scan or retain its reader. + blockReads = false; + final retry = await scanner.scan( + NetworkStorageService.source('nas', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(retry.expectedCount, 20); + expect(retry.errorCount, 0); + }, + ); + + test( + 'directory-scoped deduplication rejects cross-folder listing entries', + () async { + final storage = _Storage({ + '': [ + const NetworkEntry('root.flac', 'root.flac'), + const NetworkEntry('root.flac', 'root.flac'), + const NetworkEntry('nested/song.flac', 'song.flac'), + const NetworkEntry('nested/', 'nested', directory: true), + const NetworkEntry('nested/', 'nested', directory: true), + ], + 'nested/': [ + const NetworkEntry('nested/song.flac', 'song.flac'), + const NetworkEntry('nested/song.flac', 'song.flac'), + const NetworkEntry('root.flac', 'root.flac'), + ], + }); + final reads = []; + final result = + await scanner(storage, (source) async { + reads.add(Uri.parse(source).pathSegments.join('/')); + return {'title': 'Track'}; + }).scan( + NetworkStorageService.source('nas', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(reads, ['root.flac', 'nested/song.flac']); + expect(result.expectedCount, 2); + expect(result.errorCount, 0); + }, + ); + test( 'bounded prefetch retains listing order after out-of-order failures', () async { diff --git a/test/network_smb_memory_test.dart b/test/network_smb_memory_test.dart new file mode 100644 index 000000000..943118dc9 --- /dev/null +++ b/test/network_smb_memory_test.dart @@ -0,0 +1,108 @@ +// Host-only injection of the same native library used by the SMB plugin. +// ignore_for_file: invalid_use_of_internal_member + +import 'dart:developer' as developer; +import 'dart:io'; + +// ignore: implementation_imports +import 'package:dart_smb2/src/ffi/native_lib.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/services/network_library_scanner.dart'; +import 'package:spotiflac_android/services/network_smb_client.dart'; +import 'package:spotiflac_android/services/network_storage_service.dart'; + +import 'support/vm_service_probe.dart'; + +Future _smbWorkers(Uri service) async { + final vm = await callTestVmService(service, 'getVM'); + return (vm['isolates'] as List) + .where((isolate) => (isolate as Map)['name'] == 'workerMain') + .length; +} + +Future main() async { + TestWidgetsFlutterBinding.ensureInitialized(); + final service = (await developer.Service.getInfo()).serverUri; + final library = Platform.environment['TEST_SMB_LIBRARY']; + final address = Platform.environment['TEST_SMB_ADDRESS']; + test( + '12 native SMB scans release workers, handles and range streams', + () async { + final overrides = HttpOverrides.current; + HttpOverrides.global = null; + addTearDown(() => HttpOverrides.global = overrides); + debugLibSmb2PathOverride = library; + addTearDown(() => debugLibSmb2PathOverride = null); + final directory = await Directory.systemTemp.createTemp('smb-memory-'); + addTearDown(() => directory.delete(recursive: true)); + final storage = NetworkStorageService( + read: () async => null, + write: (_) async {}, + ); + addTearDown(storage.dispose); + await storage.save( + NetworkConnection( + id: 'fixture', + name: 'SMB fixture', + protocol: NetworkProtocol.smb, + address: address!, + username: 'test', + password: 'secret', + ), + ); + final client = HttpClient(); + addTearDown(() => client.close(force: true)); + final scanner = NetworkLibraryScanner( + storage: storage, + supportDirectory: () async => directory, + temporaryDirectory: () async => directory, + readMetadata: (source) async { + final url = await storage.resolve(source); + final request = await client.getUrl(Uri.parse(url)); + request.headers.set(HttpHeaders.rangeHeader, 'bytes=0-31'); + final response = await request.close(); + expect(response.statusCode, 206); + final bytes = await response.fold>( + [], + (bytes, chunk) => bytes..addAll(chunk), + ); + expect(bytes, List.generate(32, (i) => i)); + return {'title': 'Fixture'}; + }, + ); + final workersBefore = await _smbWorkers(service!); + final probe = await NetworkSmbClient.connect( + Uri.parse(address), + username: 'test', + password: 'secret', + domain: '', + timeout: const Duration(seconds: 5), + ); + try { + expect(await _smbWorkers(service), greaterThan(workersBefore)); + } finally { + await probe.close(); + } + for (var run = 0; run < 12; run++) { + final result = await scanner.scan( + NetworkStorageService.source('fixture', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(result.expectedCount, 1); + expect(result.errorCount, 0); + await result.delete(); + await collectTestGarbage(service); + expect(await _smbWorkers(service), workersBefore); + } + expect( + await Directory('${directory.path}/network_scans').list().toList(), + isEmpty, + ); + }, + timeout: const Timeout(Duration(minutes: 2)), + skip: library == null || address == null || service == null + ? 'Requires TEST_SMB_LIBRARY, TEST_SMB_ADDRESS and --enable-vmservice' + : false, + ); +} diff --git a/test/support/vm_service_probe.dart b/test/support/vm_service_probe.dart new file mode 100644 index 000000000..5ad300b58 --- /dev/null +++ b/test/support/vm_service_probe.dart @@ -0,0 +1,34 @@ +import 'dart:convert'; +import 'dart:developer' as developer; +import 'dart:io'; +import 'dart:isolate'; + +Future> callTestVmService( + Uri service, + String method, [ + Map parameters = const {}, +]) async { + final client = HttpClient(); + try { + final request = await client.getUrl( + service.resolve(method).replace(queryParameters: parameters), + ); + final response = await request.close(); + final decoded = + jsonDecode(await response.transform(utf8.decoder).join()) + as Map; + if (decoded['error'] != null) { + throw StateError('VM service failed: ${decoded['error']}'); + } + return decoded['result'] as Map; + } finally { + client.close(force: true); + } +} + +Future collectTestGarbage(Uri service) async { + await callTestVmService(service, 'getAllocationProfile', { + 'isolateId': developer.Service.getIsolateId(Isolate.current)!, + 'gc': 'true', + }); +}