mirror of
https://github.com/zarzet/SpotiFLAC-Mobile.git
synced 2026-11-21 20:00:49 +01:00
fix(library): prune stale NAS rows outside failed scan paths
This commit is contained in:
1 parent
6bbc0823d4
commit
faa08f5b56
7 files changed
+183
-16
No files matched your search
@@ -9,22 +9,64 @@ import 'package:mornye/utils/logger.dart';
|
||||
|
||||
/// Removes rows superseded by a staged scan inside the caller's transaction.
|
||||
/// On partial scans, an unreadable file is not evidence that it was deleted.
|
||||
({String where, List<Object?> args}) libraryScanReplacementPredicate(
|
||||
String sourceId, {
|
||||
required String stageTable,
|
||||
required bool preserveMissing,
|
||||
List<String>? preserveMissingPaths,
|
||||
}) {
|
||||
final args = <Object?>[sourceId];
|
||||
if (!preserveMissing) return (where: 'source_id = ?', args: args);
|
||||
final replaced =
|
||||
'id IN (SELECT id FROM $stageTable) '
|
||||
'OR file_path IN (SELECT file_path FROM $stageTable)';
|
||||
if (preserveMissingPaths == null || preserveMissingPaths.length > 128) {
|
||||
return (where: 'source_id = ? AND ($replaced)', args: args);
|
||||
}
|
||||
if (preserveMissingPaths.isEmpty) {
|
||||
return (where: 'source_id = ?', args: args);
|
||||
}
|
||||
final protected = <String>[];
|
||||
for (final path in preserveMissingPaths) {
|
||||
if (path.endsWith('/')) {
|
||||
// Literal prefixes preserve percent signs and underscores in NAS names.
|
||||
protected.add('substr(file_path, 1, length(?)) = ?');
|
||||
args.addAll([path, path]);
|
||||
} else {
|
||||
protected.add('file_path = ?');
|
||||
args.add(path);
|
||||
}
|
||||
}
|
||||
return (
|
||||
where:
|
||||
'source_id = ? AND ($replaced OR '
|
||||
'NOT (${protected.join(' OR ')}))',
|
||||
args: args,
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> deleteReplacedLibraryScanRows(
|
||||
DatabaseExecutor db,
|
||||
String sourceId, {
|
||||
required String stageTable,
|
||||
required bool preserveMissing,
|
||||
List<String>? preserveMissingPaths,
|
||||
}) async {
|
||||
final where = preserveMissing
|
||||
? 'source_id = ? AND (id IN (SELECT id FROM $stageTable) '
|
||||
'OR file_path IN (SELECT file_path FROM $stageTable))'
|
||||
: 'source_id = ?';
|
||||
final predicate = libraryScanReplacementPredicate(
|
||||
sourceId,
|
||||
stageTable: stageTable,
|
||||
preserveMissing: preserveMissing,
|
||||
preserveMissingPaths: preserveMissingPaths,
|
||||
);
|
||||
await db.rawDelete(
|
||||
'DELETE FROM library_path_keys WHERE item_id IN '
|
||||
'(SELECT id FROM library WHERE $where)',
|
||||
[sourceId],
|
||||
'(SELECT id FROM library WHERE ${predicate.where})',
|
||||
predicate.args,
|
||||
);
|
||||
await db.rawDelete(
|
||||
'DELETE FROM library WHERE ${predicate.where}',
|
||||
predicate.args,
|
||||
);
|
||||
await db.rawDelete('DELETE FROM library WHERE $where', [sourceId]);
|
||||
}
|
||||
|
||||
/// Deletes a selection atomically without exceeding SQLite's parameter budget.
|
||||
|
||||
@@ -98,6 +98,7 @@ class LibraryIngestionStore {
|
||||
sourceId,
|
||||
validatedRows(),
|
||||
preserveMissing: scan.errorCount > 0,
|
||||
preserveMissingPaths: scan.preserveMissingPaths,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -362,6 +363,7 @@ class LibraryIngestionStore {
|
||||
Stream<Map<String, dynamic>> items, {
|
||||
int batchSize = 300,
|
||||
bool preserveMissing = false,
|
||||
List<String>? preserveMissingPaths,
|
||||
}) async {
|
||||
if (batchSize <= 0) {
|
||||
throw ArgumentError.value(batchSize, 'batchSize', 'Must be positive');
|
||||
@@ -459,6 +461,7 @@ class LibraryIngestionStore {
|
||||
sourceId,
|
||||
stageTable: _scanStageTable,
|
||||
preserveMissing: preserveMissing,
|
||||
preserveMissingPaths: preserveMissingPaths,
|
||||
);
|
||||
await txn.rawDelete('''
|
||||
DELETE FROM library_path_keys
|
||||
|
||||
@@ -3,6 +3,7 @@ import 'dart:io';
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import 'package:mornye/services/library_cleanup.dart';
|
||||
import 'package:mornye/services/platform_bridge.dart';
|
||||
import 'package:mornye/services/library_schema.dart';
|
||||
import 'package:mornye/services/sqlite_helpers.dart' as sqlite;
|
||||
@@ -180,6 +181,7 @@ class LibraryNativeAdapter {
|
||||
},
|
||||
incremental: false,
|
||||
preserveMissing: scan.errorCount > 0,
|
||||
preserveMissingPaths: scan.preserveMissingPaths,
|
||||
requestId: requestId,
|
||||
isCancelled: isCancelled,
|
||||
);
|
||||
@@ -210,6 +212,7 @@ class LibraryNativeAdapter {
|
||||
Map<String, dynamic> operation, {
|
||||
required bool incremental,
|
||||
required bool preserveMissing,
|
||||
List<String>? preserveMissingPaths,
|
||||
required String requestId,
|
||||
required bool Function() isCancelled,
|
||||
}) async {
|
||||
@@ -236,6 +239,7 @@ class LibraryNativeAdapter {
|
||||
result,
|
||||
incremental: incremental,
|
||||
preserveMissing: preserveMissing,
|
||||
preserveMissingPaths: preserveMissingPaths,
|
||||
isCancelled: isCancelled,
|
||||
);
|
||||
} on NativeSqliteSnapshotDetachException catch (error) {
|
||||
@@ -253,6 +257,7 @@ class LibraryNativeAdapter {
|
||||
Map<String, dynamic> nativeResult, {
|
||||
required bool incremental,
|
||||
required bool preserveMissing,
|
||||
List<String>? preserveMissingPaths,
|
||||
required bool Function() isCancelled,
|
||||
}) => _attached(privatePath, (alias) async {
|
||||
return sqlite.transactionWithBusyRetry(database, (txn) async {
|
||||
@@ -311,13 +316,13 @@ class LibraryNativeAdapter {
|
||||
[sourceId],
|
||||
);
|
||||
} else {
|
||||
await deleteWhere(
|
||||
preserveMissing
|
||||
? 'source_id = ? AND (id IN(SELECT id FROM $alias.library) '
|
||||
'OR file_path IN(SELECT file_path FROM $alias.library))'
|
||||
: 'source_id = ?',
|
||||
[sourceId],
|
||||
final predicate = libraryScanReplacementPredicate(
|
||||
sourceId,
|
||||
stageTable: '$alias.library',
|
||||
preserveMissing: preserveMissing,
|
||||
preserveMissingPaths: preserveMissingPaths,
|
||||
);
|
||||
await deleteWhere(predicate.where, predicate.args);
|
||||
}
|
||||
_check(isCancelled);
|
||||
await txn.rawDelete(
|
||||
|
||||
@@ -17,7 +17,7 @@ final _log = AppLogger('LocalLibrary');
|
||||
|
||||
/// Bounded tag prefetch; only NDJSON and artwork are stored locally.
|
||||
/// Failed listings never count as empty directories. Partial scans retain old
|
||||
/// rows through the same ingestion policy used for local/SAF scans.
|
||||
/// rows only under unreadable folders or at unreadable files.
|
||||
class NetworkLibraryScanner {
|
||||
NetworkLibraryScanner({
|
||||
NetworkStorageService? storage,
|
||||
@@ -108,6 +108,19 @@ class NetworkLibraryScanner {
|
||||
// Fail before creating scan output if the root cannot be listed.
|
||||
List<NetworkEntry>? first = await _storage.list(connection, root);
|
||||
await checkpoint();
|
||||
List<String>? preserveMissingPaths = [];
|
||||
void preserve(String path) {
|
||||
final paths = preserveMissingPaths;
|
||||
if (paths == null) return;
|
||||
// Keep failed-scan memory and publication SQL bounded. Widespread
|
||||
// failures fall back to retaining the whole previous source snapshot.
|
||||
if (paths.length >= 128) {
|
||||
preserveMissingPaths = null;
|
||||
} else {
|
||||
paths.add(NetworkStorageService.source(connection.id, path));
|
||||
}
|
||||
}
|
||||
|
||||
final temp = await Directory(
|
||||
'${(await _temporaryDirectory()).path}/network_scans',
|
||||
).create(recursive: true);
|
||||
@@ -171,6 +184,7 @@ class NetworkLibraryScanner {
|
||||
first = null;
|
||||
} catch (error) {
|
||||
errors++;
|
||||
preserve(folder);
|
||||
_log.e('Network folder listing failed [$folder]', error);
|
||||
onProgress(processed, errors, folder);
|
||||
continue;
|
||||
@@ -333,6 +347,7 @@ class NetworkLibraryScanner {
|
||||
// Cancellation must abort the whole staging operation.
|
||||
await checkpoint();
|
||||
errors++;
|
||||
preserve(entry.path);
|
||||
_log.e('Network track scan failed [$trackSource]', error);
|
||||
}
|
||||
if (encodedRow != null) {
|
||||
@@ -374,6 +389,7 @@ class NetworkLibraryScanner {
|
||||
file: file,
|
||||
expectedCount: written,
|
||||
errorCount: errors,
|
||||
preserveMissingPaths: preserveMissingPaths,
|
||||
);
|
||||
} catch (_) {
|
||||
await metadata?.cancelPending();
|
||||
|
||||
@@ -5,10 +5,15 @@ class LibraryScanNDJSONFile {
|
||||
final int expectedCount;
|
||||
final int errorCount;
|
||||
|
||||
/// Failed network files, or folder prefixes ending in '/'. Null retains the
|
||||
/// conservative whole-source policy used by local and SAF scans.
|
||||
final List<String>? preserveMissingPaths;
|
||||
|
||||
const LibraryScanNDJSONFile({
|
||||
required this.file,
|
||||
required this.expectedCount,
|
||||
this.errorCount = 0,
|
||||
this.preserveMissingPaths,
|
||||
});
|
||||
|
||||
Stream<Map<String, dynamic>> rows() async* {
|
||||
|
||||
@@ -218,6 +218,7 @@ void main() {
|
||||
Future<LibraryScanNDJSONFile> scan(
|
||||
List<Map<String, dynamic>> rows, {
|
||||
int errors = 0,
|
||||
List<String>? preserveMissingPaths,
|
||||
}) async {
|
||||
final file = File('${root.path}/scan.ndjson');
|
||||
await file.writeAsString(rows.map(jsonEncode).join('\n'));
|
||||
@@ -225,6 +226,7 @@ void main() {
|
||||
file: file,
|
||||
expectedCount: rows.length,
|
||||
errorCount: errors,
|
||||
preserveMissingPaths: preserveMissingPaths,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -319,6 +321,50 @@ void main() {
|
||||
},
|
||||
);
|
||||
|
||||
test('partial NAS publication retains only unreadable scopes', () async {
|
||||
const prefix = 'network://nas/music/';
|
||||
await addNativeLibraryTrack(
|
||||
live,
|
||||
nativeLibraryTrack('unreadable', path: '${prefix}unreadable.flac'),
|
||||
);
|
||||
await addNativeLibraryTrack(
|
||||
live,
|
||||
nativeLibraryTrack('offline', path: '${prefix}offline%25_/keep.flac'),
|
||||
);
|
||||
await addNativeLibraryTrack(
|
||||
live,
|
||||
nativeLibraryTrack('deleted', path: '${prefix}test/deleted.flac'),
|
||||
);
|
||||
final refreshed = nativeLibraryTrack(
|
||||
'refreshed',
|
||||
path: '${prefix}offline%25_/keep.flac',
|
||||
);
|
||||
await adapter.importScanFile(
|
||||
'source',
|
||||
await scan(
|
||||
[refreshed],
|
||||
errors: 2,
|
||||
preserveMissingPaths: [
|
||||
'${prefix}offline%25_/',
|
||||
'${prefix}unreadable.flac',
|
||||
],
|
||||
),
|
||||
requestId: 'partial-nas',
|
||||
isCancelled: () => false,
|
||||
);
|
||||
expect(
|
||||
(await live.query('library')).map((row) => row['id']),
|
||||
unorderedEquals(['refreshed', 'unreadable', 'other']),
|
||||
);
|
||||
expect(
|
||||
(await live.query(
|
||||
'library_path_keys',
|
||||
)).map((row) => row['item_id']).toSet(),
|
||||
{'refreshed', 'unreadable', 'other'},
|
||||
);
|
||||
await cleaned();
|
||||
});
|
||||
|
||||
test(
|
||||
'incremental removes current downloads and exact source-scoped deleted paths',
|
||||
() async {
|
||||
|
||||
@@ -94,6 +94,7 @@ void main() {
|
||||
Future<LibraryScanNDJSONFile> scan({
|
||||
bool force = false,
|
||||
bool commit = true,
|
||||
String root = '',
|
||||
}) async {
|
||||
// A fresh reader per scan proves reuse comes from persisted Library
|
||||
// rows, rather than the playback service's short-lived memory cache.
|
||||
@@ -122,7 +123,7 @@ void main() {
|
||||
},
|
||||
);
|
||||
final result = await scanner.scan(
|
||||
NetworkStorageService.source('nas', ''),
|
||||
NetworkStorageService.source('nas', root),
|
||||
forceFullScan: force,
|
||||
checkpoint: () async {
|
||||
if (cancelled) throw StateError('cancelled');
|
||||
@@ -134,6 +135,7 @@ void main() {
|
||||
'source',
|
||||
result.rows(),
|
||||
preserveMissing: result.errorCount > 0,
|
||||
preserveMissingPaths: result.preserveMissingPaths,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
@@ -263,7 +265,11 @@ void main() {
|
||||
final partial = await scan();
|
||||
expect(partial.errorCount, 1);
|
||||
expect(reads, ['bad.flac']);
|
||||
expect(await database.query('library'), hasLength(3));
|
||||
expect(await database.query('library'), hasLength(2));
|
||||
expect(
|
||||
(await database.query('library')).map((row) => row['file_path']),
|
||||
isNot(contains('network://nas/missing.flac')),
|
||||
);
|
||||
failedTracks.clear();
|
||||
reads.clear();
|
||||
await scan();
|
||||
@@ -330,6 +336,50 @@ void main() {
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'partial scan removes a deleted folder but retains failed scopes',
|
||||
() async {
|
||||
storage.tree[''] = [
|
||||
const NetworkEntry('test/', 'test', directory: true),
|
||||
const NetworkEntry('offline%_/', 'offline%_', directory: true),
|
||||
track('unreadable.flac'),
|
||||
];
|
||||
storage.tree['test/'] = [track('test/stale.flac')];
|
||||
storage.tree['offline%_/'] = [track('offline%_/keep.flac')];
|
||||
await scan();
|
||||
storage.tree['']!.removeAt(0);
|
||||
storage.tree['']![1] = track('unreadable.flac', modified: 2000);
|
||||
storage.failedFolders.add('offline%_/');
|
||||
failedTracks.add('unreadable.flac');
|
||||
final result = await scan();
|
||||
expect(result.errorCount, 2);
|
||||
expect(
|
||||
(await database.query('library')).map((row) => row['file_path']),
|
||||
unorderedEquals([
|
||||
'network://nas/offline%25_/keep.flac',
|
||||
'network://nas/unreadable.flac',
|
||||
]),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
test('selected NAS folders stay scoped to their own subtree', () async {
|
||||
storage.tree[''] = [
|
||||
const NetworkEntry('downloads/', 'downloads', directory: true),
|
||||
const NetworkEntry('processed/', 'processed', directory: true),
|
||||
];
|
||||
storage.tree['downloads/'] = [track('downloads/original.flac')];
|
||||
storage.tree['processed/'] = [track('processed/converted.flac')];
|
||||
for (final folder in ['downloads/', 'processed/']) {
|
||||
reads.clear();
|
||||
final result = await scan(root: folder, commit: false);
|
||||
final rows = await result.rows().toList();
|
||||
expect(rows, hasLength(1));
|
||||
expect(rows.single['filePath'], startsWith('network://nas/$folder'));
|
||||
expect(reads, hasLength(1));
|
||||
}
|
||||
});
|
||||
|
||||
test(
|
||||
'cancellation after cache lookup discards staging and preserves rows',
|
||||
() async {
|
||||
|
||||
Reference in new issue
Block a user