fix(library): prune stale NAS rows outside failed scan paths

This commit is contained in:
zarzet committed 2026-10-21 01:47:56 +07:00
1 parent 6bbc0823d4
commit faa08f5b56
7 files changed
+183 -16

No files matched your search

+49 -7
View File
@@ -9,22 +9,64 @@ import 'package:mornye/utils/logger.dart';
/// Removes rows superseded by a staged scan inside the caller's transaction.
/// On partial scans, an unreadable file is not evidence that it was deleted.
({String where, List<Object?> args}) libraryScanReplacementPredicate(
String sourceId, {
required String stageTable,
required bool preserveMissing,
List<String>? preserveMissingPaths,
}) {
final args = <Object?>[sourceId];
if (!preserveMissing) return (where: 'source_id = ?', args: args);
final replaced =
'id IN (SELECT id FROM $stageTable) '
'OR file_path IN (SELECT file_path FROM $stageTable)';
if (preserveMissingPaths == null || preserveMissingPaths.length > 128) {
return (where: 'source_id = ? AND ($replaced)', args: args);
}
if (preserveMissingPaths.isEmpty) {
return (where: 'source_id = ?', args: args);
}
final protected = <String>[];
for (final path in preserveMissingPaths) {
if (path.endsWith('/')) {
// Literal prefixes preserve percent signs and underscores in NAS names.
protected.add('substr(file_path, 1, length(?)) = ?');
args.addAll([path, path]);
} else {
protected.add('file_path = ?');
args.add(path);
}
}
return (
where:
'source_id = ? AND ($replaced OR '
'NOT (${protected.join(' OR ')}))',
args: args,
);
}
Future<void> deleteReplacedLibraryScanRows(
DatabaseExecutor db,
String sourceId, {
required String stageTable,
required bool preserveMissing,
List<String>? preserveMissingPaths,
}) async {
final where = preserveMissing
? 'source_id = ? AND (id IN (SELECT id FROM $stageTable) '
'OR file_path IN (SELECT file_path FROM $stageTable))'
: 'source_id = ?';
final predicate = libraryScanReplacementPredicate(
sourceId,
stageTable: stageTable,
preserveMissing: preserveMissing,
preserveMissingPaths: preserveMissingPaths,
);
await db.rawDelete(
'DELETE FROM library_path_keys WHERE item_id IN '
'(SELECT id FROM library WHERE $where)',
[sourceId],
'(SELECT id FROM library WHERE ${predicate.where})',
predicate.args,
);
await db.rawDelete(
'DELETE FROM library WHERE ${predicate.where}',
predicate.args,
);
await db.rawDelete('DELETE FROM library WHERE $where', [sourceId]);
}
/// Deletes a selection atomically without exceeding SQLite's parameter budget.
@@ -98,6 +98,7 @@ class LibraryIngestionStore {
sourceId,
validatedRows(),
preserveMissing: scan.errorCount > 0,
preserveMissingPaths: scan.preserveMissingPaths,
);
}
@@ -362,6 +363,7 @@ class LibraryIngestionStore {
Stream<Map<String, dynamic>> items, {
int batchSize = 300,
bool preserveMissing = false,
List<String>? preserveMissingPaths,
}) async {
if (batchSize <= 0) {
throw ArgumentError.value(batchSize, 'batchSize', 'Must be positive');
@@ -459,6 +461,7 @@ class LibraryIngestionStore {
sourceId,
stageTable: _scanStageTable,
preserveMissing: preserveMissing,
preserveMissingPaths: preserveMissingPaths,
);
await txn.rawDelete('''
DELETE FROM library_path_keys
+11 -6
View File
@@ -3,6 +3,7 @@ import 'dart:io';
import 'package:path/path.dart' as p;
import 'package:path_provider/path_provider.dart';
import 'package:sqflite/sqflite.dart';
import 'package:mornye/services/library_cleanup.dart';
import 'package:mornye/services/platform_bridge.dart';
import 'package:mornye/services/library_schema.dart';
import 'package:mornye/services/sqlite_helpers.dart' as sqlite;
@@ -180,6 +181,7 @@ class LibraryNativeAdapter {
},
incremental: false,
preserveMissing: scan.errorCount > 0,
preserveMissingPaths: scan.preserveMissingPaths,
requestId: requestId,
isCancelled: isCancelled,
);
@@ -210,6 +212,7 @@ class LibraryNativeAdapter {
Map<String, dynamic> operation, {
required bool incremental,
required bool preserveMissing,
List<String>? preserveMissingPaths,
required String requestId,
required bool Function() isCancelled,
}) async {
@@ -236,6 +239,7 @@ class LibraryNativeAdapter {
result,
incremental: incremental,
preserveMissing: preserveMissing,
preserveMissingPaths: preserveMissingPaths,
isCancelled: isCancelled,
);
} on NativeSqliteSnapshotDetachException catch (error) {
@@ -253,6 +257,7 @@ class LibraryNativeAdapter {
Map<String, dynamic> nativeResult, {
required bool incremental,
required bool preserveMissing,
List<String>? preserveMissingPaths,
required bool Function() isCancelled,
}) => _attached(privatePath, (alias) async {
return sqlite.transactionWithBusyRetry(database, (txn) async {
@@ -311,13 +316,13 @@ class LibraryNativeAdapter {
[sourceId],
);
} else {
await deleteWhere(
preserveMissing
? 'source_id = ? AND (id IN(SELECT id FROM $alias.library) '
'OR file_path IN(SELECT file_path FROM $alias.library))'
: 'source_id = ?',
[sourceId],
final predicate = libraryScanReplacementPredicate(
sourceId,
stageTable: '$alias.library',
preserveMissing: preserveMissing,
preserveMissingPaths: preserveMissingPaths,
);
await deleteWhere(predicate.where, predicate.args);
}
_check(isCancelled);
await txn.rawDelete(
+17 -1
View File
@@ -17,7 +17,7 @@ final _log = AppLogger('LocalLibrary');
/// Bounded tag prefetch; only NDJSON and artwork are stored locally.
/// Failed listings never count as empty directories. Partial scans retain old
/// rows through the same ingestion policy used for local/SAF scans.
/// rows only under unreadable folders or at unreadable files.
class NetworkLibraryScanner {
NetworkLibraryScanner({
NetworkStorageService? storage,
@@ -108,6 +108,19 @@ class NetworkLibraryScanner {
// Fail before creating scan output if the root cannot be listed.
List<NetworkEntry>? first = await _storage.list(connection, root);
await checkpoint();
List<String>? preserveMissingPaths = [];
void preserve(String path) {
final paths = preserveMissingPaths;
if (paths == null) return;
// Keep failed-scan memory and publication SQL bounded. Widespread
// failures fall back to retaining the whole previous source snapshot.
if (paths.length >= 128) {
preserveMissingPaths = null;
} else {
paths.add(NetworkStorageService.source(connection.id, path));
}
}
final temp = await Directory(
'${(await _temporaryDirectory()).path}/network_scans',
).create(recursive: true);
@@ -171,6 +184,7 @@ class NetworkLibraryScanner {
first = null;
} catch (error) {
errors++;
preserve(folder);
_log.e('Network folder listing failed [$folder]', error);
onProgress(processed, errors, folder);
continue;
@@ -333,6 +347,7 @@ class NetworkLibraryScanner {
// Cancellation must abort the whole staging operation.
await checkpoint();
errors++;
preserve(entry.path);
_log.e('Network track scan failed [$trackSource]', error);
}
if (encodedRow != null) {
@@ -374,6 +389,7 @@ class NetworkLibraryScanner {
file: file,
expectedCount: written,
errorCount: errors,
preserveMissingPaths: preserveMissingPaths,
);
} catch (_) {
await metadata?.cancelPending();
@@ -5,10 +5,15 @@ class LibraryScanNDJSONFile {
final int expectedCount;
final int errorCount;
/// Failed network files, or folder prefixes ending in '/'. Null retains the
/// conservative whole-source policy used by local and SAF scans.
final List<String>? preserveMissingPaths;
const LibraryScanNDJSONFile({
required this.file,
required this.expectedCount,
this.errorCount = 0,
this.preserveMissingPaths,
});
Stream<Map<String, dynamic>> rows() async* {
+46
View File
@@ -218,6 +218,7 @@ void main() {
Future<LibraryScanNDJSONFile> scan(
List<Map<String, dynamic>> rows, {
int errors = 0,
List<String>? preserveMissingPaths,
}) async {
final file = File('${root.path}/scan.ndjson');
await file.writeAsString(rows.map(jsonEncode).join('\n'));
@@ -225,6 +226,7 @@ void main() {
file: file,
expectedCount: rows.length,
errorCount: errors,
preserveMissingPaths: preserveMissingPaths,
);
}
@@ -319,6 +321,50 @@ void main() {
},
);
test('partial NAS publication retains only unreadable scopes', () async {
const prefix = 'network://nas/music/';
await addNativeLibraryTrack(
live,
nativeLibraryTrack('unreadable', path: '${prefix}unreadable.flac'),
);
await addNativeLibraryTrack(
live,
nativeLibraryTrack('offline', path: '${prefix}offline%25_/keep.flac'),
);
await addNativeLibraryTrack(
live,
nativeLibraryTrack('deleted', path: '${prefix}test/deleted.flac'),
);
final refreshed = nativeLibraryTrack(
'refreshed',
path: '${prefix}offline%25_/keep.flac',
);
await adapter.importScanFile(
'source',
await scan(
[refreshed],
errors: 2,
preserveMissingPaths: [
'${prefix}offline%25_/',
'${prefix}unreadable.flac',
],
),
requestId: 'partial-nas',
isCancelled: () => false,
);
expect(
(await live.query('library')).map((row) => row['id']),
unorderedEquals(['refreshed', 'unreadable', 'other']),
);
expect(
(await live.query(
'library_path_keys',
)).map((row) => row['item_id']).toSet(),
{'refreshed', 'unreadable', 'other'},
);
await cleaned();
});
test(
'incremental removes current downloads and exact source-scoped deleted paths',
() async {
+52 -2
View File
@@ -94,6 +94,7 @@ void main() {
Future<LibraryScanNDJSONFile> scan({
bool force = false,
bool commit = true,
String root = '',
}) async {
// A fresh reader per scan proves reuse comes from persisted Library
// rows, rather than the playback service's short-lived memory cache.
@@ -122,7 +123,7 @@ void main() {
},
);
final result = await scanner.scan(
NetworkStorageService.source('nas', ''),
NetworkStorageService.source('nas', root),
forceFullScan: force,
checkpoint: () async {
if (cancelled) throw StateError('cancelled');
@@ -134,6 +135,7 @@ void main() {
'source',
result.rows(),
preserveMissing: result.errorCount > 0,
preserveMissingPaths: result.preserveMissingPaths,
);
}
return result;
@@ -263,7 +265,11 @@ void main() {
final partial = await scan();
expect(partial.errorCount, 1);
expect(reads, ['bad.flac']);
expect(await database.query('library'), hasLength(3));
expect(await database.query('library'), hasLength(2));
expect(
(await database.query('library')).map((row) => row['file_path']),
isNot(contains('network://nas/missing.flac')),
);
failedTracks.clear();
reads.clear();
await scan();
@@ -330,6 +336,50 @@ void main() {
},
);
test(
'partial scan removes a deleted folder but retains failed scopes',
() async {
storage.tree[''] = [
const NetworkEntry('test/', 'test', directory: true),
const NetworkEntry('offline%_/', 'offline%_', directory: true),
track('unreadable.flac'),
];
storage.tree['test/'] = [track('test/stale.flac')];
storage.tree['offline%_/'] = [track('offline%_/keep.flac')];
await scan();
storage.tree['']!.removeAt(0);
storage.tree['']![1] = track('unreadable.flac', modified: 2000);
storage.failedFolders.add('offline%_/');
failedTracks.add('unreadable.flac');
final result = await scan();
expect(result.errorCount, 2);
expect(
(await database.query('library')).map((row) => row['file_path']),
unorderedEquals([
'network://nas/offline%25_/keep.flac',
'network://nas/unreadable.flac',
]),
);
},
);
test('selected NAS folders stay scoped to their own subtree', () async {
storage.tree[''] = [
const NetworkEntry('downloads/', 'downloads', directory: true),
const NetworkEntry('processed/', 'processed', directory: true),
];
storage.tree['downloads/'] = [track('downloads/original.flac')];
storage.tree['processed/'] = [track('processed/converted.flac')];
for (final folder in ['downloads/', 'processed/']) {
reads.clear();
final result = await scan(root: folder, commit: false);
final rows = await result.rows().toList();
expect(rows, hasLength(1));
expect(rows.single['filePath'], startsWith('network://nas/$folder'));
expect(reads, hasLength(1));
}
});
test(
'cancellation after cache lookup discards staging and preserves rows',
() async {