fix(library): release network scan memory and cancel active metadata reads

Release completed directory listings and iterators during SMB and WebDAV scans. Scope file deduplication to direct directory children so large libraries do not retain every scanned path.

Cancel scan-owned metadata reads immediately on cancellation or provider disposal, and clear scanner ownership on completion.

Add forced-GC regression coverage for directory listings and paged library rows, cancellation coverage, and repeated native SMB scan worker cleanup checks.
This commit is contained in:
zarzet committed 2026-10-19 20:40:01 +07:00
1 parent 205e178e5b
commit e58217c8cd
8 files changed
+519 -8

No files matched your search

+11
View File
@@ -176,6 +176,7 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
bool _scanPauseRequested = false;
bool _scanInProgress = false;
Future<void>? _scanCompletion;
NetworkLibraryScanner? _networkScanner;
String? _nativeLibraryRequestId;
final Set<String> _pendingReconnectScans = {};
bool _drainingReconnectScans = false;
@@ -200,6 +201,11 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
_scanCancelRequested = true;
_scanPauseRequested = false;
_pendingReconnectScans.clear();
unawaited(
_networkScanner?.cancelPending().catchError((Object error) {
_log.w('Failed to cancel disposed network scan reads: $error');
}),
);
if (_scanInProgress) {
final requestId = _nativeLibraryRequestId;
if (requestId != null) {
@@ -508,6 +514,11 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
_log.i('Cancelling library scan');
_scanCancelRequested = true;
try {
await _networkScanner?.cancelPending();
} catch (error) {
_log.w('Failed to cancel network scan reads: $error');
}
final requestId = _nativeLibraryRequestId;
if (requestId != null) {
try {
+19 -1
View File
@@ -48,6 +48,24 @@ extension _LocalLibraryScan on LocalLibraryNotifier {
}
}
Future<LibraryScanNDJSONFile> _scanNetworkFolder(
String source, {
required Future<void> Function() checkpoint,
required void Function(int processed, int errors, String name) onProgress,
}) async {
final scanner = NetworkLibraryScanner();
_networkScanner = scanner;
try {
return await scanner.scan(
source,
checkpoint: checkpoint,
onProgress: onProgress,
);
} finally {
if (identical(_networkScanner, scanner)) _networkScanner = null;
}
}
Future<({int inserted, int skipped})?> _replaceFromFullScanStream({
required String sourceId,
required String folderPath,
@@ -56,7 +74,7 @@ extension _LocalLibraryScan on LocalLibraryNotifier {
}) async {
if (_scanCancelRequested) return null;
final scanFile = folderPath.startsWith('network://')
? await NetworkLibraryScanner().scan(
? await _scanNetworkFolder(
folderPath,
checkpoint: () async {
while (_scanPauseRequested &&
+59 -7
View File
@@ -39,11 +39,46 @@ class NetworkLibraryScanner {
final int? _metadataConcurrency;
final Future<Directory> Function() _supportDirectory, _temporaryDirectory;
final Future<RandomAccessFile> Function(File) _openOutput;
NetworkMetadataService? _activeMetadata;
bool _running = false;
bool _cancelled = false;
/// Stop scan-owned native reads even while scan() is awaiting their result.
Future<void> cancelPending() async {
_cancelled = true;
await _activeMetadata?.cancelPending();
}
Future<LibraryScanNDJSONFile> scan(
String source, {
required Future<void> Function() checkpoint,
required void Function(int processed, int errors, String name) onProgress,
}) async {
if (_running) {
throw StateError('Network library scanner is already running');
}
_running = true;
_cancelled = false;
try {
return await _scan(
source,
checkpoint: () async {
if (_cancelled) throw StateError('Network library scan cancelled');
await checkpoint();
if (_cancelled) throw StateError('Network library scan cancelled');
},
onProgress: onProgress,
);
} finally {
_activeMetadata = null;
_running = false;
}
}
Future<LibraryScanNDJSONFile> _scan(
String source, {
required Future<void> Function() checkpoint,
required void Function(int processed, int errors, String name) onProgress,
}) async {
final uri = Uri.parse(source);
if (uri.scheme != 'network' ||
@@ -62,7 +97,8 @@ class NetworkLibraryScanner {
}
await checkpoint();
// Fail before creating scan output if the root cannot be listed.
final first = await _storage.list(connection, root);
List<NetworkEntry>? first = await _storage.list(connection, root);
await checkpoint();
final temp = await Directory(
'${(await _temporaryDirectory()).path}/network_scans',
).create(recursive: true);
@@ -76,7 +112,6 @@ class NetworkLibraryScanner {
var errors = 0;
final pending = <String>[root];
final visited = <String>{};
final seenFiles = <String>{};
// Leave room for playback and avoid saturating a small NAS/router.
final concurrency =
_metadataConcurrency ??
@@ -88,6 +123,7 @@ class NetworkLibraryScanner {
cacheDirectory: _temporaryDirectory,
)
: null;
_activeMetadata = metadata;
final read =
_readMetadata ??
((String source) => metadata!.read(source, forceRefresh: true));
@@ -115,11 +151,14 @@ class NetworkLibraryScanner {
if (visited.length > 100000) {
throw StateError('Too many network folders');
}
final List<NetworkEntry> entries;
List<NetworkEntry>? entries;
try {
entries = folder == root
? first
? first!
: await _storage.list(connection, folder);
// The first listing can be very large. Release it before descending
// into children instead of retaining it for the entire scan.
first = null;
} catch (error) {
errors++;
_log.e('Network folder listing failed [$folder]', error);
@@ -131,14 +170,23 @@ class NetworkLibraryScanner {
'${entries.where((entry) => entry.directory).length} folders, '
'${entries.where((entry) => entry.audio).length} audio files',
);
final iterator = entries.iterator;
// Each directory is visited once and only its direct children are
// accepted, so file deduplication need not retain every scanned path.
final seenFiles = <String>{};
Iterator<NetworkEntry>? iterator = entries.iterator;
entries = null;
Future<void> fill() async {
while (prefetched.length < concurrency && iterator.moveNext()) {
while (prefetched.length < concurrency && iterator!.moveNext()) {
await checkpoint();
final entry = iterator.current;
final entry = iterator!.current;
final path = networkPath(entry.path);
// Reject provider parent links and paths outside this folder.
if (!path.startsWith(folder) || path == folder) continue;
final child = path.substring(folder.length);
final name = entry.directory && child.endsWith('/')
? child.substring(0, child.length - 1)
: child;
if (name.isEmpty || name.contains('/')) continue;
if (entry.directory) {
if (!path.endsWith('/')) {
throw const FormatException('Invalid folder entry');
@@ -242,6 +290,10 @@ class NetworkLibraryScanner {
onProgress(processed, errors, entry.name);
await fill();
}
// Async closures may otherwise keep the completed iterator's backing
// list alive while awaiting the next directory listing.
iterator = null;
seenFiles.clear();
}
await checkpoint();
await flush();
+95
View File
@@ -0,0 +1,95 @@
import 'dart:developer' as developer;
import 'dart:io';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:spotiflac_android/providers/download_history_provider.dart';
import 'package:spotiflac_android/providers/library_browse_provider.dart';
import 'package:spotiflac_android/providers/local_library_provider.dart';
import 'package:spotiflac_android/providers/settings_provider.dart';
import 'support/fixed_settings.dart';
import 'support/vm_service_probe.dart';
class _History extends DownloadHistoryNotifier {
@override
DownloadHistoryState build() => DownloadHistoryState();
}
class _Local extends LocalLibraryNotifier {
@override
LocalLibraryState build() => LocalLibraryState();
}
Future<void> _browseAndClose(ProviderContainer container, String search) async {
final provider = libraryBrowseProvider((
artists: false,
artist: null,
search: search,
sort: 'a-z',
limit: 40,
completeness: null,
));
final subscription = container.listen(provider, (_, _) {});
try {
await container.read(provider.future);
for (var page = 0; page < 9; page++) {
await container.read(provider.notifier).loadMore();
}
} finally {
subscription.close();
}
await container.pump();
expect(container.exists(provider), isFalse);
}
Future<void> main() async {
TestWidgetsFlutterBinding.ensureInitialized();
final service = (await developer.Service.getInfo()).serverUri;
test(
'Library pages and old filter results are collectible after closing',
() async {
final overrides = HttpOverrides.current;
HttpOverrides.global = null;
addTearDown(() => HttpOverrides.global = overrides);
final entries = <WeakReference<LibraryBrowseEntry>>[];
final container = ProviderContainer(
overrides: [
settingsProvider.overrideWith(FixedSettingsNotifier.new),
downloadHistoryProvider.overrideWith(_History.new),
localLibraryProvider.overrideWith(_Local.new),
libraryBrowsePageLoaderProvider.overrideWithValue((page) async {
final rows = [
for (var i = page.offset; i < page.offset + 40 && i < 400; i++)
LibraryBrowseEntry(
source: 'local',
key: '$i',
name: 'Album $i',
artist: 'Artist',
samplePath: '/music/$i.flac',
trackCount: 1,
),
];
entries.addAll(rows.map(WeakReference.new));
return LibraryBrowsePage(rows);
}),
],
);
// Debug DevTools intentionally retain provider event snapshots. Exclude
// that history to measure the application-owned lifecycle as in release.
container.observers.clear();
addTearDown(container.dispose);
for (var i = 0; i < 5; i++) {
await _browseAndClose(container, 'filter $i');
}
expect(entries, hasLength(2000));
await collectTestGarbage(service!);
expect(
entries.where((entry) => entry.target != null).length,
0,
reason: 'Closed Library queries must not retain their loaded rows',
);
},
skip: service == null ? 'Requires flutter test --enable-vmservice' : false,
);
}
+91
View File
@@ -0,0 +1,91 @@
import 'dart:developer' as developer;
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:spotiflac_android/services/network_library_scanner.dart';
import 'package:spotiflac_android/services/network_storage_service.dart';
import 'support/vm_service_probe.dart';
class _TransientListingStorage extends NetworkStorageService {
_TransientListingStorage(this.protocol, this.collect);
final NetworkProtocol protocol;
final Future<void> Function() collect;
WeakReference<List<NetworkEntry>>? rootListing;
bool? rootWasReleased;
@override
Future<NetworkConnection> connection(String id) async => NetworkConnection(
id: id,
name: 'Fixture',
protocol: protocol,
address: protocol == NetworkProtocol.smb
? 'smb://example.test/music/'
: 'https://example.test/music/',
);
@override
Future<List<NetworkEntry>> list(
NetworkConnection connection, [
String path = '',
]) async {
if (path.isEmpty) {
final entries = [
const NetworkEntry('child/', 'child', directory: true),
for (var i = 0; i < 4000; i++) NetworkEntry('$i.flac', '$i.flac'),
];
rootListing = WeakReference(entries);
return entries;
}
await collect();
rootWasReleased = rootListing!.target == null;
return [const NetworkEntry('child/song.flac', 'song.flac')];
}
}
Future<void> main() async {
TestWidgetsFlutterBinding.ensureInitialized();
final service = (await developer.Service.getInfo()).serverUri;
for (final protocol in [NetworkProtocol.smb, NetworkProtocol.webdav]) {
test(
'${protocol.name} scan releases completed directory listings before finishing',
() async {
final overrides = HttpOverrides.current;
HttpOverrides.global = null;
addTearDown(() => HttpOverrides.global = overrides);
final directory = await Directory.systemTemp.createTemp(
'network-library-memory-',
);
addTearDown(() => directory.delete(recursive: true));
final storage = _TransientListingStorage(
protocol,
() => collectTestGarbage(service!),
);
addTearDown(storage.dispose);
final result =
await NetworkLibraryScanner(
storage: storage,
readMetadata: (_) async => {'title': 'Track'},
supportDirectory: () async => directory,
temporaryDirectory: () async => directory,
).scan(
NetworkStorageService.source('fixture', ''),
checkpoint: () async {},
onProgress: (_, _, _) {},
);
expect(result.expectedCount, 4001);
expect(result.errorCount, 0);
expect(
storage.rootWasReleased,
isTrue,
reason: 'A completed root listing must not stay alive in child scans',
);
await result.delete();
},
skip: service == null
? 'Requires flutter test --enable-vmservice'
: false,
);
}
}
+102
View File
@@ -2,6 +2,7 @@ import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:flutter/services.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:spotiflac_android/services/network_library_scanner.dart';
import 'package:spotiflac_android/services/network_storage_service.dart';
@@ -72,6 +73,107 @@ void main() {
);
}
test(
'cancel interrupts active native metadata reads and drains their work',
() async {
const channel = MethodChannel('com.zarz.spotiflac/backend');
final messenger =
TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger;
final jobs = <String, Completer<String>>{};
final cancelled = <String>{};
final started = Completer<void>();
var blockReads = true;
messenger.setMockMethodCallHandler(channel, (call) async {
final arguments = call.arguments as Map;
final id = arguments['request_id'] as String;
if (call.method == 'cancelNativeDataJob') {
final job = jobs[id];
if (job != null && !job.isCompleted) {
cancelled.add(id);
job.completeError(PlatformException(code: 'CANCELLED'));
}
return null;
}
expect(call.method, 'runNativeDataJob');
if (!blockReads) {
return jsonEncode({
'metadata': {'title': 'Track'},
'cacheable': true,
});
}
final job = Completer<String>();
jobs[id] = job;
if (jobs.length == 2) started.complete();
return job.future;
});
addTearDown(() => messenger.setMockMethodCallHandler(channel, null));
final storage = tracks(20);
addTearDown(storage.dispose);
final scanner = NetworkLibraryScanner(
storage: storage,
supportDirectory: () async => directory,
temporaryDirectory: () async => directory,
metadataConcurrency: 2,
);
final scan = scanner.scan(
NetworkStorageService.source('nas', ''),
checkpoint: () async {},
onProgress: (_, _, _) {},
);
final stopped = expectLater(scan, throwsStateError);
await started.future;
await scanner.cancelPending();
await stopped;
expect(jobs, hasLength(2));
expect(cancelled, jobs.keys.toSet());
expect(jobs.values.every((job) => job.isCompleted), isTrue);
await expectNoOutput();
// A completed cancellation does not poison a later scan or retain its reader.
blockReads = false;
final retry = await scanner.scan(
NetworkStorageService.source('nas', ''),
checkpoint: () async {},
onProgress: (_, _, _) {},
);
expect(retry.expectedCount, 20);
expect(retry.errorCount, 0);
},
);
test(
'directory-scoped deduplication rejects cross-folder listing entries',
() async {
final storage = _Storage({
'': [
const NetworkEntry('root.flac', 'root.flac'),
const NetworkEntry('root.flac', 'root.flac'),
const NetworkEntry('nested/song.flac', 'song.flac'),
const NetworkEntry('nested/', 'nested', directory: true),
const NetworkEntry('nested/', 'nested', directory: true),
],
'nested/': [
const NetworkEntry('nested/song.flac', 'song.flac'),
const NetworkEntry('nested/song.flac', 'song.flac'),
const NetworkEntry('root.flac', 'root.flac'),
],
});
final reads = <String>[];
final result =
await scanner(storage, (source) async {
reads.add(Uri.parse(source).pathSegments.join('/'));
return {'title': 'Track'};
}).scan(
NetworkStorageService.source('nas', ''),
checkpoint: () async {},
onProgress: (_, _, _) {},
);
expect(reads, ['root.flac', 'nested/song.flac']);
expect(result.expectedCount, 2);
expect(result.errorCount, 0);
},
);
test(
'bounded prefetch retains listing order after out-of-order failures',
() async {
+108
View File
@@ -0,0 +1,108 @@
// Host-only injection of the same native library used by the SMB plugin.
// ignore_for_file: invalid_use_of_internal_member
import 'dart:developer' as developer;
import 'dart:io';
// ignore: implementation_imports
import 'package:dart_smb2/src/ffi/native_lib.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:spotiflac_android/services/network_library_scanner.dart';
import 'package:spotiflac_android/services/network_smb_client.dart';
import 'package:spotiflac_android/services/network_storage_service.dart';
import 'support/vm_service_probe.dart';
Future<int> _smbWorkers(Uri service) async {
final vm = await callTestVmService(service, 'getVM');
return (vm['isolates'] as List)
.where((isolate) => (isolate as Map)['name'] == 'workerMain')
.length;
}
Future<void> main() async {
TestWidgetsFlutterBinding.ensureInitialized();
final service = (await developer.Service.getInfo()).serverUri;
final library = Platform.environment['TEST_SMB_LIBRARY'];
final address = Platform.environment['TEST_SMB_ADDRESS'];
test(
'12 native SMB scans release workers, handles and range streams',
() async {
final overrides = HttpOverrides.current;
HttpOverrides.global = null;
addTearDown(() => HttpOverrides.global = overrides);
debugLibSmb2PathOverride = library;
addTearDown(() => debugLibSmb2PathOverride = null);
final directory = await Directory.systemTemp.createTemp('smb-memory-');
addTearDown(() => directory.delete(recursive: true));
final storage = NetworkStorageService(
read: () async => null,
write: (_) async {},
);
addTearDown(storage.dispose);
await storage.save(
NetworkConnection(
id: 'fixture',
name: 'SMB fixture',
protocol: NetworkProtocol.smb,
address: address!,
username: 'test',
password: 'secret',
),
);
final client = HttpClient();
addTearDown(() => client.close(force: true));
final scanner = NetworkLibraryScanner(
storage: storage,
supportDirectory: () async => directory,
temporaryDirectory: () async => directory,
readMetadata: (source) async {
final url = await storage.resolve(source);
final request = await client.getUrl(Uri.parse(url));
request.headers.set(HttpHeaders.rangeHeader, 'bytes=0-31');
final response = await request.close();
expect(response.statusCode, 206);
final bytes = await response.fold<List<int>>(
[],
(bytes, chunk) => bytes..addAll(chunk),
);
expect(bytes, List.generate(32, (i) => i));
return {'title': 'Fixture'};
},
);
final workersBefore = await _smbWorkers(service!);
final probe = await NetworkSmbClient.connect(
Uri.parse(address),
username: 'test',
password: 'secret',
domain: '',
timeout: const Duration(seconds: 5),
);
try {
expect(await _smbWorkers(service), greaterThan(workersBefore));
} finally {
await probe.close();
}
for (var run = 0; run < 12; run++) {
final result = await scanner.scan(
NetworkStorageService.source('fixture', ''),
checkpoint: () async {},
onProgress: (_, _, _) {},
);
expect(result.expectedCount, 1);
expect(result.errorCount, 0);
await result.delete();
await collectTestGarbage(service);
expect(await _smbWorkers(service), workersBefore);
}
expect(
await Directory('${directory.path}/network_scans').list().toList(),
isEmpty,
);
},
timeout: const Timeout(Duration(minutes: 2)),
skip: library == null || address == null || service == null
? 'Requires TEST_SMB_LIBRARY, TEST_SMB_ADDRESS and --enable-vmservice'
: false,
);
}
+34
View File
@@ -0,0 +1,34 @@
import 'dart:convert';
import 'dart:developer' as developer;
import 'dart:io';
import 'dart:isolate';
Future<Map<String, dynamic>> callTestVmService(
Uri service,
String method, [
Map<String, String> parameters = const {},
]) async {
final client = HttpClient();
try {
final request = await client.getUrl(
service.resolve(method).replace(queryParameters: parameters),
);
final response = await request.close();
final decoded =
jsonDecode(await response.transform(utf8.decoder).join())
as Map<String, dynamic>;
if (decoded['error'] != null) {
throw StateError('VM service failed: ${decoded['error']}');
}
return decoded['result'] as Map<String, dynamic>;
} finally {
client.close(force: true);
}
}
Future<void> collectTestGarbage(Uri service) async {
await callTestVmService(service, 'getAllocationProfile', {
'isolateId': developer.Service.getIsolateId(Isolate.current)!,
'gc': 'true',
});
}