From e58217c8cdfe36091a6a7465424b0fb15842e453 Mon Sep 17 00:00:00 2001 From: zarzet <42882290+zarzet@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:40:01 +0700 Subject: [PATCH] fix(library): release network scan memory and cancel active metadata reads Release completed directory listings and iterators during SMB and WebDAV scans. Scope file deduplication to direct directory children so large libraries do not retain every scanned path. Cancel scan-owned metadata reads immediately on cancellation or provider disposal, and clear scanner ownership on completion. Add forced-GC regression coverage for directory listings and paged library rows, cancellation coverage, and repeated native SMB scan worker cleanup checks. --- lib/providers/local_library_provider.dart | 11 ++ .../local_library_provider_scan.dart | 20 +++- lib/services/network_library_scanner.dart | 66 +++++++++-- test/library_memory_release_test.dart | 95 +++++++++++++++ test/network_library_memory_test.dart | 91 +++++++++++++++ test/network_library_scanner_test.dart | 102 +++++++++++++++++ test/network_smb_memory_test.dart | 108 ++++++++++++++++++ test/support/vm_service_probe.dart | 34 ++++++ 8 files changed, 519 insertions(+), 8 deletions(-) create mode 100644 test/library_memory_release_test.dart create mode 100644 test/network_library_memory_test.dart create mode 100644 test/network_smb_memory_test.dart create mode 100644 test/support/vm_service_probe.dart diff --git a/lib/providers/local_library_provider.dart b/lib/providers/local_library_provider.dart index ad39c0f69..358e38092 100644 --- a/lib/providers/local_library_provider.dart +++ b/lib/providers/local_library_provider.dart @@ -176,6 +176,7 @@ class LocalLibraryNotifier extends Notifier { bool _scanPauseRequested = false; bool _scanInProgress = false; Future? _scanCompletion; + NetworkLibraryScanner? _networkScanner; String? _nativeLibraryRequestId; final Set _pendingReconnectScans = {}; bool _drainingReconnectScans = false; @@ -200,6 +201,11 @@ class LocalLibraryNotifier extends Notifier { _scanCancelRequested = true; _scanPauseRequested = false; _pendingReconnectScans.clear(); + unawaited( + _networkScanner?.cancelPending().catchError((Object error) { + _log.w('Failed to cancel disposed network scan reads: $error'); + }), + ); if (_scanInProgress) { final requestId = _nativeLibraryRequestId; if (requestId != null) { @@ -508,6 +514,11 @@ class LocalLibraryNotifier extends Notifier { _log.i('Cancelling library scan'); _scanCancelRequested = true; + try { + await _networkScanner?.cancelPending(); + } catch (error) { + _log.w('Failed to cancel network scan reads: $error'); + } final requestId = _nativeLibraryRequestId; if (requestId != null) { try { diff --git a/lib/providers/local_library_provider_scan.dart b/lib/providers/local_library_provider_scan.dart index 916047950..9fae60cf4 100644 --- a/lib/providers/local_library_provider_scan.dart +++ b/lib/providers/local_library_provider_scan.dart @@ -48,6 +48,24 @@ extension _LocalLibraryScan on LocalLibraryNotifier { } } + Future _scanNetworkFolder( + String source, { + required Future Function() checkpoint, + required void Function(int processed, int errors, String name) onProgress, + }) async { + final scanner = NetworkLibraryScanner(); + _networkScanner = scanner; + try { + return await scanner.scan( + source, + checkpoint: checkpoint, + onProgress: onProgress, + ); + } finally { + if (identical(_networkScanner, scanner)) _networkScanner = null; + } + } + Future<({int inserted, int skipped})?> _replaceFromFullScanStream({ required String sourceId, required String folderPath, @@ -56,7 +74,7 @@ extension _LocalLibraryScan on LocalLibraryNotifier { }) async { if (_scanCancelRequested) return null; final scanFile = folderPath.startsWith('network://') - ? await NetworkLibraryScanner().scan( + ? await _scanNetworkFolder( folderPath, checkpoint: () async { while (_scanPauseRequested && diff --git a/lib/services/network_library_scanner.dart b/lib/services/network_library_scanner.dart index 003416636..39aec8087 100644 --- a/lib/services/network_library_scanner.dart +++ b/lib/services/network_library_scanner.dart @@ -39,11 +39,46 @@ class NetworkLibraryScanner { final int? _metadataConcurrency; final Future Function() _supportDirectory, _temporaryDirectory; final Future Function(File) _openOutput; + NetworkMetadataService? _activeMetadata; + bool _running = false; + bool _cancelled = false; + + /// Stop scan-owned native reads even while scan() is awaiting their result. + Future cancelPending() async { + _cancelled = true; + await _activeMetadata?.cancelPending(); + } Future scan( String source, { required Future Function() checkpoint, required void Function(int processed, int errors, String name) onProgress, + }) async { + if (_running) { + throw StateError('Network library scanner is already running'); + } + _running = true; + _cancelled = false; + try { + return await _scan( + source, + checkpoint: () async { + if (_cancelled) throw StateError('Network library scan cancelled'); + await checkpoint(); + if (_cancelled) throw StateError('Network library scan cancelled'); + }, + onProgress: onProgress, + ); + } finally { + _activeMetadata = null; + _running = false; + } + } + + Future _scan( + String source, { + required Future Function() checkpoint, + required void Function(int processed, int errors, String name) onProgress, }) async { final uri = Uri.parse(source); if (uri.scheme != 'network' || @@ -62,7 +97,8 @@ class NetworkLibraryScanner { } await checkpoint(); // Fail before creating scan output if the root cannot be listed. - final first = await _storage.list(connection, root); + List? first = await _storage.list(connection, root); + await checkpoint(); final temp = await Directory( '${(await _temporaryDirectory()).path}/network_scans', ).create(recursive: true); @@ -76,7 +112,6 @@ class NetworkLibraryScanner { var errors = 0; final pending = [root]; final visited = {}; - final seenFiles = {}; // Leave room for playback and avoid saturating a small NAS/router. final concurrency = _metadataConcurrency ?? @@ -88,6 +123,7 @@ class NetworkLibraryScanner { cacheDirectory: _temporaryDirectory, ) : null; + _activeMetadata = metadata; final read = _readMetadata ?? ((String source) => metadata!.read(source, forceRefresh: true)); @@ -115,11 +151,14 @@ class NetworkLibraryScanner { if (visited.length > 100000) { throw StateError('Too many network folders'); } - final List entries; + List? entries; try { entries = folder == root - ? first + ? first! : await _storage.list(connection, folder); + // The first listing can be very large. Release it before descending + // into children instead of retaining it for the entire scan. + first = null; } catch (error) { errors++; _log.e('Network folder listing failed [$folder]', error); @@ -131,14 +170,23 @@ class NetworkLibraryScanner { '${entries.where((entry) => entry.directory).length} folders, ' '${entries.where((entry) => entry.audio).length} audio files', ); - final iterator = entries.iterator; + // Each directory is visited once and only its direct children are + // accepted, so file deduplication need not retain every scanned path. + final seenFiles = {}; + Iterator? iterator = entries.iterator; + entries = null; Future fill() async { - while (prefetched.length < concurrency && iterator.moveNext()) { + while (prefetched.length < concurrency && iterator!.moveNext()) { await checkpoint(); - final entry = iterator.current; + final entry = iterator!.current; final path = networkPath(entry.path); // Reject provider parent links and paths outside this folder. if (!path.startsWith(folder) || path == folder) continue; + final child = path.substring(folder.length); + final name = entry.directory && child.endsWith('/') + ? child.substring(0, child.length - 1) + : child; + if (name.isEmpty || name.contains('/')) continue; if (entry.directory) { if (!path.endsWith('/')) { throw const FormatException('Invalid folder entry'); @@ -242,6 +290,10 @@ class NetworkLibraryScanner { onProgress(processed, errors, entry.name); await fill(); } + // Async closures may otherwise keep the completed iterator's backing + // list alive while awaiting the next directory listing. + iterator = null; + seenFiles.clear(); } await checkpoint(); await flush(); diff --git a/test/library_memory_release_test.dart b/test/library_memory_release_test.dart new file mode 100644 index 000000000..0e11d0ea1 --- /dev/null +++ b/test/library_memory_release_test.dart @@ -0,0 +1,95 @@ +import 'dart:developer' as developer; +import 'dart:io'; + +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/providers/download_history_provider.dart'; +import 'package:spotiflac_android/providers/library_browse_provider.dart'; +import 'package:spotiflac_android/providers/local_library_provider.dart'; +import 'package:spotiflac_android/providers/settings_provider.dart'; + +import 'support/fixed_settings.dart'; +import 'support/vm_service_probe.dart'; + +class _History extends DownloadHistoryNotifier { + @override + DownloadHistoryState build() => DownloadHistoryState(); +} + +class _Local extends LocalLibraryNotifier { + @override + LocalLibraryState build() => LocalLibraryState(); +} + +Future _browseAndClose(ProviderContainer container, String search) async { + final provider = libraryBrowseProvider(( + artists: false, + artist: null, + search: search, + sort: 'a-z', + limit: 40, + completeness: null, + )); + final subscription = container.listen(provider, (_, _) {}); + try { + await container.read(provider.future); + for (var page = 0; page < 9; page++) { + await container.read(provider.notifier).loadMore(); + } + } finally { + subscription.close(); + } + await container.pump(); + expect(container.exists(provider), isFalse); +} + +Future main() async { + TestWidgetsFlutterBinding.ensureInitialized(); + final service = (await developer.Service.getInfo()).serverUri; + test( + 'Library pages and old filter results are collectible after closing', + () async { + final overrides = HttpOverrides.current; + HttpOverrides.global = null; + addTearDown(() => HttpOverrides.global = overrides); + final entries = >[]; + final container = ProviderContainer( + overrides: [ + settingsProvider.overrideWith(FixedSettingsNotifier.new), + downloadHistoryProvider.overrideWith(_History.new), + localLibraryProvider.overrideWith(_Local.new), + libraryBrowsePageLoaderProvider.overrideWithValue((page) async { + final rows = [ + for (var i = page.offset; i < page.offset + 40 && i < 400; i++) + LibraryBrowseEntry( + source: 'local', + key: '$i', + name: 'Album $i', + artist: 'Artist', + samplePath: '/music/$i.flac', + trackCount: 1, + ), + ]; + entries.addAll(rows.map(WeakReference.new)); + return LibraryBrowsePage(rows); + }), + ], + ); + // Debug DevTools intentionally retain provider event snapshots. Exclude + // that history to measure the application-owned lifecycle as in release. + container.observers.clear(); + addTearDown(container.dispose); + for (var i = 0; i < 5; i++) { + await _browseAndClose(container, 'filter $i'); + } + expect(entries, hasLength(2000)); + await collectTestGarbage(service!); + expect( + entries.where((entry) => entry.target != null).length, + 0, + reason: 'Closed Library queries must not retain their loaded rows', + ); + }, + skip: service == null ? 'Requires flutter test --enable-vmservice' : false, + ); +} diff --git a/test/network_library_memory_test.dart b/test/network_library_memory_test.dart new file mode 100644 index 000000000..8f8b8a343 --- /dev/null +++ b/test/network_library_memory_test.dart @@ -0,0 +1,91 @@ +import 'dart:developer' as developer; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/services/network_library_scanner.dart'; +import 'package:spotiflac_android/services/network_storage_service.dart'; + +import 'support/vm_service_probe.dart'; + +class _TransientListingStorage extends NetworkStorageService { + _TransientListingStorage(this.protocol, this.collect); + + final NetworkProtocol protocol; + final Future Function() collect; + WeakReference>? rootListing; + bool? rootWasReleased; + + @override + Future connection(String id) async => NetworkConnection( + id: id, + name: 'Fixture', + protocol: protocol, + address: protocol == NetworkProtocol.smb + ? 'smb://example.test/music/' + : 'https://example.test/music/', + ); + + @override + Future> list( + NetworkConnection connection, [ + String path = '', + ]) async { + if (path.isEmpty) { + final entries = [ + const NetworkEntry('child/', 'child', directory: true), + for (var i = 0; i < 4000; i++) NetworkEntry('$i.flac', '$i.flac'), + ]; + rootListing = WeakReference(entries); + return entries; + } + await collect(); + rootWasReleased = rootListing!.target == null; + return [const NetworkEntry('child/song.flac', 'song.flac')]; + } +} + +Future main() async { + TestWidgetsFlutterBinding.ensureInitialized(); + final service = (await developer.Service.getInfo()).serverUri; + for (final protocol in [NetworkProtocol.smb, NetworkProtocol.webdav]) { + test( + '${protocol.name} scan releases completed directory listings before finishing', + () async { + final overrides = HttpOverrides.current; + HttpOverrides.global = null; + addTearDown(() => HttpOverrides.global = overrides); + final directory = await Directory.systemTemp.createTemp( + 'network-library-memory-', + ); + addTearDown(() => directory.delete(recursive: true)); + final storage = _TransientListingStorage( + protocol, + () => collectTestGarbage(service!), + ); + addTearDown(storage.dispose); + final result = + await NetworkLibraryScanner( + storage: storage, + readMetadata: (_) async => {'title': 'Track'}, + supportDirectory: () async => directory, + temporaryDirectory: () async => directory, + ).scan( + NetworkStorageService.source('fixture', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(result.expectedCount, 4001); + expect(result.errorCount, 0); + expect( + storage.rootWasReleased, + isTrue, + reason: 'A completed root listing must not stay alive in child scans', + ); + await result.delete(); + }, + skip: service == null + ? 'Requires flutter test --enable-vmservice' + : false, + ); + } +} diff --git a/test/network_library_scanner_test.dart b/test/network_library_scanner_test.dart index 79c64801f..2b6c8bc7d 100644 --- a/test/network_library_scanner_test.dart +++ b/test/network_library_scanner_test.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; +import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:spotiflac_android/services/network_library_scanner.dart'; import 'package:spotiflac_android/services/network_storage_service.dart'; @@ -72,6 +73,107 @@ void main() { ); } + test( + 'cancel interrupts active native metadata reads and drains their work', + () async { + const channel = MethodChannel('com.zarz.spotiflac/backend'); + final messenger = + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger; + final jobs = >{}; + final cancelled = {}; + final started = Completer(); + var blockReads = true; + messenger.setMockMethodCallHandler(channel, (call) async { + final arguments = call.arguments as Map; + final id = arguments['request_id'] as String; + if (call.method == 'cancelNativeDataJob') { + final job = jobs[id]; + if (job != null && !job.isCompleted) { + cancelled.add(id); + job.completeError(PlatformException(code: 'CANCELLED')); + } + return null; + } + expect(call.method, 'runNativeDataJob'); + if (!blockReads) { + return jsonEncode({ + 'metadata': {'title': 'Track'}, + 'cacheable': true, + }); + } + final job = Completer(); + jobs[id] = job; + if (jobs.length == 2) started.complete(); + return job.future; + }); + addTearDown(() => messenger.setMockMethodCallHandler(channel, null)); + final storage = tracks(20); + addTearDown(storage.dispose); + final scanner = NetworkLibraryScanner( + storage: storage, + supportDirectory: () async => directory, + temporaryDirectory: () async => directory, + metadataConcurrency: 2, + ); + final scan = scanner.scan( + NetworkStorageService.source('nas', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + final stopped = expectLater(scan, throwsStateError); + await started.future; + await scanner.cancelPending(); + await stopped; + expect(jobs, hasLength(2)); + expect(cancelled, jobs.keys.toSet()); + expect(jobs.values.every((job) => job.isCompleted), isTrue); + await expectNoOutput(); + + // A completed cancellation does not poison a later scan or retain its reader. + blockReads = false; + final retry = await scanner.scan( + NetworkStorageService.source('nas', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(retry.expectedCount, 20); + expect(retry.errorCount, 0); + }, + ); + + test( + 'directory-scoped deduplication rejects cross-folder listing entries', + () async { + final storage = _Storage({ + '': [ + const NetworkEntry('root.flac', 'root.flac'), + const NetworkEntry('root.flac', 'root.flac'), + const NetworkEntry('nested/song.flac', 'song.flac'), + const NetworkEntry('nested/', 'nested', directory: true), + const NetworkEntry('nested/', 'nested', directory: true), + ], + 'nested/': [ + const NetworkEntry('nested/song.flac', 'song.flac'), + const NetworkEntry('nested/song.flac', 'song.flac'), + const NetworkEntry('root.flac', 'root.flac'), + ], + }); + final reads = []; + final result = + await scanner(storage, (source) async { + reads.add(Uri.parse(source).pathSegments.join('/')); + return {'title': 'Track'}; + }).scan( + NetworkStorageService.source('nas', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(reads, ['root.flac', 'nested/song.flac']); + expect(result.expectedCount, 2); + expect(result.errorCount, 0); + }, + ); + test( 'bounded prefetch retains listing order after out-of-order failures', () async { diff --git a/test/network_smb_memory_test.dart b/test/network_smb_memory_test.dart new file mode 100644 index 000000000..943118dc9 --- /dev/null +++ b/test/network_smb_memory_test.dart @@ -0,0 +1,108 @@ +// Host-only injection of the same native library used by the SMB plugin. +// ignore_for_file: invalid_use_of_internal_member + +import 'dart:developer' as developer; +import 'dart:io'; + +// ignore: implementation_imports +import 'package:dart_smb2/src/ffi/native_lib.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/services/network_library_scanner.dart'; +import 'package:spotiflac_android/services/network_smb_client.dart'; +import 'package:spotiflac_android/services/network_storage_service.dart'; + +import 'support/vm_service_probe.dart'; + +Future _smbWorkers(Uri service) async { + final vm = await callTestVmService(service, 'getVM'); + return (vm['isolates'] as List) + .where((isolate) => (isolate as Map)['name'] == 'workerMain') + .length; +} + +Future main() async { + TestWidgetsFlutterBinding.ensureInitialized(); + final service = (await developer.Service.getInfo()).serverUri; + final library = Platform.environment['TEST_SMB_LIBRARY']; + final address = Platform.environment['TEST_SMB_ADDRESS']; + test( + '12 native SMB scans release workers, handles and range streams', + () async { + final overrides = HttpOverrides.current; + HttpOverrides.global = null; + addTearDown(() => HttpOverrides.global = overrides); + debugLibSmb2PathOverride = library; + addTearDown(() => debugLibSmb2PathOverride = null); + final directory = await Directory.systemTemp.createTemp('smb-memory-'); + addTearDown(() => directory.delete(recursive: true)); + final storage = NetworkStorageService( + read: () async => null, + write: (_) async {}, + ); + addTearDown(storage.dispose); + await storage.save( + NetworkConnection( + id: 'fixture', + name: 'SMB fixture', + protocol: NetworkProtocol.smb, + address: address!, + username: 'test', + password: 'secret', + ), + ); + final client = HttpClient(); + addTearDown(() => client.close(force: true)); + final scanner = NetworkLibraryScanner( + storage: storage, + supportDirectory: () async => directory, + temporaryDirectory: () async => directory, + readMetadata: (source) async { + final url = await storage.resolve(source); + final request = await client.getUrl(Uri.parse(url)); + request.headers.set(HttpHeaders.rangeHeader, 'bytes=0-31'); + final response = await request.close(); + expect(response.statusCode, 206); + final bytes = await response.fold>( + [], + (bytes, chunk) => bytes..addAll(chunk), + ); + expect(bytes, List.generate(32, (i) => i)); + return {'title': 'Fixture'}; + }, + ); + final workersBefore = await _smbWorkers(service!); + final probe = await NetworkSmbClient.connect( + Uri.parse(address), + username: 'test', + password: 'secret', + domain: '', + timeout: const Duration(seconds: 5), + ); + try { + expect(await _smbWorkers(service), greaterThan(workersBefore)); + } finally { + await probe.close(); + } + for (var run = 0; run < 12; run++) { + final result = await scanner.scan( + NetworkStorageService.source('fixture', ''), + checkpoint: () async {}, + onProgress: (_, _, _) {}, + ); + expect(result.expectedCount, 1); + expect(result.errorCount, 0); + await result.delete(); + await collectTestGarbage(service); + expect(await _smbWorkers(service), workersBefore); + } + expect( + await Directory('${directory.path}/network_scans').list().toList(), + isEmpty, + ); + }, + timeout: const Timeout(Duration(minutes: 2)), + skip: library == null || address == null || service == null + ? 'Requires TEST_SMB_LIBRARY, TEST_SMB_ADDRESS and --enable-vmservice' + : false, + ); +} diff --git a/test/support/vm_service_probe.dart b/test/support/vm_service_probe.dart new file mode 100644 index 000000000..5ad300b58 --- /dev/null +++ b/test/support/vm_service_probe.dart @@ -0,0 +1,34 @@ +import 'dart:convert'; +import 'dart:developer' as developer; +import 'dart:io'; +import 'dart:isolate'; + +Future> callTestVmService( + Uri service, + String method, [ + Map parameters = const {}, +]) async { + final client = HttpClient(); + try { + final request = await client.getUrl( + service.resolve(method).replace(queryParameters: parameters), + ); + final response = await request.close(); + final decoded = + jsonDecode(await response.transform(utf8.decoder).join()) + as Map; + if (decoded['error'] != null) { + throw StateError('VM service failed: ${decoded['error']}'); + } + return decoded['result'] as Map; + } finally { + client.close(force: true); + } +} + +Future collectTestGarbage(Uri service) async { + await callTestVmService(service, 'getAllocationProfile', { + 'isolateId': developer.Service.getIsolateId(Isolate.current)!, + 'gc': 'true', + }); +}