diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt index 2859a6c3..40aeee6e 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt @@ -571,11 +571,11 @@ private const val SAF_SCAN_PAUSE_POLL_MS = 100L * Holds a SAF scan at its checkpoint while paused, keeping its position in * memory. Returns true when the scan should stop because it was cancelled. */ -internal fun MainActivity.safScanStopRequested(): Boolean { - while (safScanPaused && !safScanCancel) { +internal fun MainActivity.safScanStopRequested(cancelled: () -> Boolean = { false }): Boolean { + while (safScanPaused && !safScanCancel && !cancelled()) { Thread.sleep(SAF_SCAN_PAUSE_POLL_MS) } - return safScanCancel + return safScanCancel || cancelled() } /** @@ -1287,6 +1287,7 @@ internal fun MainActivity.scanSafTreeIncremental(treeUriStr: String, existingFil internal fun MainActivity.scanSafTreeIncremental( treeUriStr: String, existingFiles: Map, + cancelled: () -> Boolean = { false }, ): Any { val (_, root, rootChildren) = resolveReadableSafTreeOrThrow(treeUriStr) @@ -1329,7 +1330,7 @@ internal fun MainActivity.scanSafTreeIncremental( val lister = SafTreeLister(this) while (queue.isNotEmpty()) { - if (safScanStopRequested()) { + if (safScanStopRequested(cancelled)) { updateSafScanProgress { it.isComplete = true } val result = JSONObject() result.put("files", JSONArray()) @@ -1361,7 +1362,7 @@ internal fun MainActivity.scanSafTreeIncremental( rememberCueDirectoryListing(dir, children, safChildLookupCache) for (child in children) { - if (safScanStopRequested()) { + if (safScanStopRequested(cancelled)) { updateSafScanProgress { it.isComplete = true } val result = JSONObject() result.put("files", JSONArray()) @@ -1462,7 +1463,7 @@ internal fun MainActivity.scanSafTreeIncremental( val cueReferencedAudioUris = mutableSetOf() for ((cueDoc, parentDir, cueName, cueLastModified) in cueFilesToScan) { - if (safScanStopRequested()) { + if (safScanStopRequested(cancelled)) { updateSafScanProgress { it.isComplete = true } spill.abandon() val result = JSONObject() @@ -1602,7 +1603,7 @@ internal fun MainActivity.scanSafTreeIncremental( val pendingAudio = mutableListOf() for (audio in audioFiles) { - if (safScanStopRequested()) return cancelledIncrementalResult() + if (safScanStopRequested(cancelled)) return cancelledIncrementalResult() if (cueReferencedAudioUris.contains(audio.doc.uri.toString())) { scanned++ reportProcessed() @@ -1613,7 +1614,7 @@ internal fun MainActivity.scanSafTreeIncremental( val completed = runSafReadsInOrder( pendingAudio, - cancelled = { safScanStopRequested() }, + cancelled = { safScanStopRequested(cancelled) }, task = { audio -> val ext = audio.name.substringAfterLast('.', "").lowercase(Locale.ROOT) val fallbackExt = if (ext.isNotBlank()) ".${ext}" else null diff --git a/android/app/src/rust/kotlin/com/zarz/spotiflac/NativeDataJobs.kt b/android/app/src/rust/kotlin/com/zarz/spotiflac/NativeDataJobs.kt index 1c4eecf1..9dab7d2d 100644 --- a/android/app/src/rust/kotlin/com/zarz/spotiflac/NativeDataJobs.kt +++ b/android/app/src/rust/kotlin/com/zarz/spotiflac/NativeDataJobs.kt @@ -3,6 +3,8 @@ package com.zarz.spotiflac import com.spotiflac.backend.CancellationDomain import com.spotiflac.backend.CancellationRegistry import com.spotiflac.backend.RequestLease +import java.io.File +import org.json.JSONArray import org.json.JSONObject // Independent of extension initialization: palette/metadata jobs can start @@ -45,5 +47,63 @@ internal fun MainActivity.runNativeDataJobPlatform(arguments: Any?, lease: Reque val args = arguments as? Map<*, *> ?: error("Missing data job arguments") val request = JSONObject(args["request_json"] as? String ?: error("Missing data job request")) val bytes = args["bytes"] as? ByteArray ?: byteArrayOf() - return bridgeJsonResult(RustCoreBackend.runDataJob(request, bytes, lease)) + var spilledDelta: File? = null + return run { + fun checkActive() { check(!lease.isCancelled()) { "Native data job cancelled" } } + try { + checkActive() + if (request.optString("operation") == "library_snapshot") { + // SAF timestamps require ContentResolver, but DB paging and all + // row mapping stay native. One lease spans every batch. + val backfilled = JSONObject() + var cursor = "" + do { + checkActive() + val pageRequest = JSONObject(request.toString()) + .put("operation", "library_snapshot_candidates") + .put("cursor", cursor).put("limit", 500) + val page = JSONObject(RustCoreBackend.runDataJob(pageRequest, byteArrayOf(), lease)) + val paths = page.getJSONArray("paths") + val uris = JSONArray() + for (index in 0 until paths.length()) uris.put(paths.getJSONObject(index).getString("file_path")) + val times = JSONObject(getSafFileModTimes(uris.toString())) + val keys = times.keys() + while (keys.hasNext()) { + val path = keys.next() + backfilled.put(path, times.getLong(path)) + } + cursor = page.getString("cursor") + } while (page.getBoolean("has_more")) + request.put("backfilled_mod_times", backfilled) + } + if (request.optString("operation") == "library_scan_incremental" && !request.has("tree_uri")) { + safScanActive = false + } + if (request.optString("operation") == "library_scan_incremental" && request.has("tree_uri")) { + checkActive() + val delta = scanSafTreeIncremental( + request.getString("tree_uri"), + loadExistingFilesFromSnapshot(request.optString("snapshot_path")), + cancelled = { lease.isCancelled() }, + ) + val deltaPath = if (delta is String) null else { + (delta as? Map<*, *>)?.get("__json_file") as? String + ?: error("Invalid SAF scan result") + } + // Own a spilled response before checking cancellation, so the + // finally block also removes a just-completed cancelled scan. + spilledDelta = deltaPath?.let(::File) + checkActive() + request.put("operation", "library_incremental_import") + if (delta is String) { + request.put("delta_json", delta) + } else { + request.put("delta_path", deltaPath) + } + } + bridgeJsonResult(RustCoreBackend.runDataJob(request, bytes, lease)) + } finally { + spilledDelta?.delete() + } + } } diff --git a/android/app/src/rust/kotlin/com/zarz/spotiflac/SelectedCoreBackend.kt b/android/app/src/rust/kotlin/com/zarz/spotiflac/SelectedCoreBackend.kt index ecd579df..20f7ad6a 100644 --- a/android/app/src/rust/kotlin/com/zarz/spotiflac/SelectedCoreBackend.kt +++ b/android/app/src/rust/kotlin/com/zarz/spotiflac/SelectedCoreBackend.kt @@ -137,7 +137,21 @@ internal object RustCoreBackend : CoreBackend { owner() to requestRegistry().acquire(id) internal fun runDataJob(request: JSONObject, bytes: ByteArray, lease: RequestLease): String { - return com.spotiflac.backend.runNativeDataJob(request.toString(), bytes, lease) + if (request.optString("operation") != "library_scan_incremental") { + return com.spotiflac.backend.runNativeDataJob(request.toString(), bytes, lease) + } + val folder = request.getString("folder_path") + return withLibraryDirectories(listOf(folder)) { current -> + val snapshot = request.optString("snapshot_path") + val staged = if (snapshot.isEmpty()) null else workspace.createTemporaryFile("library_snapshot_", ".tsv") + try { + if (staged != null) { + File(snapshot).copyTo(staged, overwrite = true) + request.put("snapshot_path", staged.canonicalPath) + } + current.runNativeDataJob(request.toString(), bytes, lease) + } finally { staged?.delete() } + } } @Synchronized diff --git a/ios/Runner/CoreBackend.swift b/ios/Runner/CoreBackend.swift index e0921479..d9482bea 100644 --- a/ios/Runner/CoreBackend.swift +++ b/ios/Runner/CoreBackend.swift @@ -302,10 +302,26 @@ final class RustCoreBackend: CoreBackend { throw failure("Missing native data job lease") } defer { finishDataJob(id, lease) } - guard let raw = args["request_json"] as? String else { + guard let raw = args["request_json"] as? String, + var request = try JSONSerialization.jsonObject(with: Data(raw.utf8)) as? [String: Any] else { throw failure("Missing native data job request") } let bytes = (args["bytes"] as? FlutterStandardTypedData)?.data ?? Data() + if request["operation"] as? String == "library_scan_incremental" { + guard let folder = request["folder_path"] as? String else { throw failure("Missing library folder") } + return try withLibraryDirectories([folder]) { current in + let staged = root.appendingPathComponent("files", isDirectory: true) + .appendingPathComponent("library_snapshot_\(UUID().uuidString).tsv") + defer { try? FileManager.default.removeItem(at: staged) } + if let snapshot = request["snapshot_path"] as? String, !snapshot.isEmpty { + try FileManager.default.copyItem(atPath: snapshot, toPath: staged.path) + request["snapshot_path"] = staged.path + } + request["folder_path"] = URL(fileURLWithPath: folder).resolvingSymlinksInPath().standardizedFileURL.path + let encoded = String(decoding: try JSONSerialization.data(withJSONObject: request), as: UTF8.self) + return try current.runNativeDataJob(requestJson: encoded, bytes: bytes, lease: lease) + } + } return try SpotiFLACBackend.runNativeDataJob(requestJson: raw, bytes: bytes, lease: lease) } diff --git a/lib/providers/local_library_provider.dart b/lib/providers/local_library_provider.dart index 9072b9a3..6629680b 100644 --- a/lib/providers/local_library_provider.dart +++ b/lib/providers/local_library_provider.dart @@ -170,6 +170,7 @@ class LocalLibraryNotifier extends Notifier { bool _scanPauseRequested = false; bool _scanInProgress = false; Future? _scanCompletion; + String? _nativeLibraryRequestId; final Set _pendingReconnectScans = {}; bool _drainingReconnectScans = false; StreamSubscription? _storageEventsSubscription; @@ -194,6 +195,16 @@ class LocalLibraryNotifier extends Notifier { _scanPauseRequested = false; _pendingReconnectScans.clear(); if (_scanInProgress) { + final requestId = _nativeLibraryRequestId; + if (requestId != null) { + unawaited( + PlatformBridge.cancelNativeDataJob(requestId).catchError(( + Object error, + ) { + _log.w('Failed to cancel disposed library ingestion: $error'); + }), + ); + } unawaited( PlatformBridge.cancelLibraryScan().catchError((Object error) { _log.w('Failed to cancel disposed library scan: $error'); @@ -783,27 +794,11 @@ class LocalLibraryNotifier extends Notifier { scannedFiles: scanFile.expectedCount, scanErrorCount: scanFile.errorCount, ); - Stream> validatedRows() async* { - var decodedRows = 0; - await for (final json in scanFile.rows()) { - if (_scanCancelRequested) { - throw StateError('Library scan cancelled during ingestion'); - } - decodedRows++; - yield json; - } - if (decodedRows != scanFile.expectedCount) { - throw FormatException( - 'Library scan row count mismatch: decoded $decodedRows, ' - 'expected ${scanFile.expectedCount}', - ); - } - } - - final result = await _db.replaceSourceStream( + final result = await _db.replaceSourceScanFile( sourceId, - validatedRows(), - preserveMissing: scanFile.errorCount > 0, + scanFile, + requestId: _nativeLibraryRequestId ?? 'desktop-library-scan', + isCancelled: () => _scanCancelRequested || !ref.mounted, ); _log.i( 'Stream-ingested ${result.inserted}/${scanFile.expectedCount} scan rows ' @@ -836,6 +831,9 @@ class LocalLibraryNotifier extends Notifier { if (sourceId != null) _pendingReconnectScans.remove(sourceId); _scanCancelRequested = false; _scanPauseRequested = false; + _nativeLibraryRequestId = PlatformBridge.supportsCoreBackend + ? 'library-scan-${DateTime.now().microsecondsSinceEpoch}' + : null; try { await _runScan( folderPath, @@ -854,6 +852,7 @@ class LocalLibraryNotifier extends Notifier { } _scanInProgress = false; _scanCompletion = null; + _nativeLibraryRequestId = null; completion.complete(); } } @@ -991,6 +990,12 @@ class LocalLibraryNotifier extends Notifier { ); if (!ref.mounted) return; if (scanResult == null || _scanCancelRequested) { + if (scanResult != null) { + await _refreshSummaryFromStorage( + excludedDownloadedCount: scanResult.skipped, + ); + } + if (!ref.mounted) return; state = state.copyWith( isScanning: false, scanIsFinalizing: false, @@ -1060,26 +1065,34 @@ class LocalLibraryNotifier extends Notifier { errorCount: state.scanErrorCount, ); } else { - final existingFiles = await _db.getFileModTimes( - sourceId: activeSourceId, - ); - _log.i( - 'Incremental scan: ${existingFiles.length} existing files in database', - ); - - final backfilledModTimes = await _backfillLegacyFileModTimes( - isSaf: isSaf, - existingFiles: existingFiles, - ); - if (backfilledModTimes.isNotEmpty) { - await _db.updateFileModTimes(backfilledModTimes); - existingFiles.addAll(backfilledModTimes); - _log.i('Backfilled ${backfilledModTimes.length} legacy mod times'); + final nativeIngestion = PlatformBridge.supportsCoreBackend; + final existingFiles = nativeIngestion + ? const {} + : await _db.getFileModTimes(sourceId: activeSourceId); + if (!nativeIngestion) { + _log.i( + 'Incremental scan: ${existingFiles.length} existing files in database', + ); + final backfilledModTimes = await _backfillLegacyFileModTimes( + isSaf: isSaf, + existingFiles: existingFiles, + ); + if (backfilledModTimes.isNotEmpty) { + await _db.updateFileModTimes(backfilledModTimes); + existingFiles.addAll(backfilledModTimes); + _log.i('Backfilled ${backfilledModTimes.length} legacy mod times'); + } } final useSnapshotBridge = Platform.isAndroid && existingFiles.isNotEmpty; - final snapshotPath = useSnapshotBridge + final snapshotPath = nativeIngestion + ? await _db.writeNativeFileModTimesSnapshot( + activeSourceId, + requestId: _nativeLibraryRequestId!, + isCancelled: () => _scanCancelRequested || !ref.mounted, + ) + : useSnapshotBridge ? await _db.writeFileModTimesSnapshot(existingFiles) : null; @@ -1090,7 +1103,16 @@ class LocalLibraryNotifier extends Notifier { !ref.mounted) { return; } - if (isSaf) { + if (nativeIngestion) { + result = await _db.scanIncrementalNative( + activeSourceId, + effectiveFolderPath, + snapshotPath!, + isSaf: isSaf, + requestId: _nativeLibraryRequestId!, + isCancelled: () => _scanCancelRequested || !ref.mounted, + ); + } else if (isSaf) { result = useSnapshotBridge && snapshotPath != null ? await PlatformBridge.scanSafTreeIncrementalFromSnapshot( effectiveFolderPath, @@ -1121,6 +1143,12 @@ class LocalLibraryNotifier extends Notifier { if (!ref.mounted) return; if (_scanCancelRequested || result['cancelled'] == true) { + if (result['committed'] == true) { + await _refreshSummaryFromStorage( + excludedDownloadedCount: (result['skipped'] as num?)?.toInt(), + ); + } + if (!ref.mounted) return; state = state.copyWith( isScanning: false, scanIsFinalizing: false, @@ -1151,15 +1179,24 @@ class LocalLibraryNotifier extends Notifier { []; final skippedCount = result['skippedCount'] as int? ?? 0; final totalFiles = result['totalFiles'] as int? ?? 0; + final scannedCount = nativeIngestion + ? (result['scanned_count'] as num).toInt() + : scannedList.length; + final deletedCount = nativeIngestion + ? (result['deleted_count'] as num).toInt() + : deletedPaths.length; + if (nativeIngestion && result['committed'] != true) { + throw StateError('Native Library scan did not commit its index'); + } _log.i( - 'Incremental result: ${scannedList.length} scanned, ' - '$skippedCount skipped, ${deletedPaths.length} deleted, $totalFiles total', + 'Incremental result: $scannedCount scanned, ' + '$skippedCount skipped, $deletedCount deleted, $totalFiles total', ); - final removedDownloaded = await _db.deleteDownloadedRowsForSource( - activeSourceId, - ); + final removedDownloaded = nativeIngestion + ? 0 + : await _db.deleteDownloadedRowsForSource(activeSourceId); if (removedDownloaded > 0) { _log.i( 'Removed $removedDownloaded downloaded tracks already present in ' @@ -1168,8 +1205,10 @@ class LocalLibraryNotifier extends Notifier { } final updatedItems = []; - var skippedDownloads = removedDownloaded; - if (scannedList.isNotEmpty) { + var skippedDownloads = nativeIngestion + ? (result['skipped'] as num).toInt() + : removedDownloaded; + if (!nativeIngestion && scannedList.isNotEmpty) { for (final json in scannedList) { final map = json as Map; final item = LocalLibraryItem.fromJson(map); @@ -1190,7 +1229,7 @@ class LocalLibraryNotifier extends Notifier { } } - if (deletedPaths.isNotEmpty) { + if (!nativeIngestion && deletedPaths.isNotEmpty) { final deleteCount = await _db.deleteByPaths(deletedPaths); _log.i('Deleted $deleteCount items from database'); } @@ -1229,8 +1268,8 @@ class LocalLibraryNotifier extends Notifier { final summary = 'Incremental scan complete: ${state.totalCount} total tracks ' - '(${scannedList.length} new/updated, $skippedCount unchanged, ' - '${deletedPaths.length} removed, $skippedDownloads already in downloads, ' + '($scannedCount new/updated, $skippedCount unchanged, ' + '$deletedCount removed, $skippedDownloads already in downloads, ' '${state.scanErrorCount} errors)'; if (state.scanErrorCount > 0) { _log.e(summary); @@ -1408,6 +1447,14 @@ class LocalLibraryNotifier extends Notifier { _log.i('Cancelling library scan'); _scanCancelRequested = true; + final requestId = _nativeLibraryRequestId; + if (requestId != null) { + try { + await PlatformBridge.cancelNativeDataJob(requestId); + } catch (error) { + _log.w('Failed to cancel library data job: $error'); + } + } _pendingReconnectScans.clear(); _availabilityRequestVersion++; _scanReconnectedOnRefresh = false; diff --git a/lib/services/library_database.dart b/lib/services/library_database.dart index f2f4a4b3..16e5830a 100644 --- a/lib/services/library_database.dart +++ b/lib/services/library_database.dart @@ -13,6 +13,7 @@ import 'package:spotiflac_android/services/library_search.dart'; import 'package:spotiflac_android/services/library_database_models.dart'; import 'package:spotiflac_android/services/library_row_mapper.dart'; import 'package:spotiflac_android/services/library_schema.dart'; +import 'package:spotiflac_android/services/platform_bridge.dart'; import 'package:spotiflac_android/services/sqlite_helpers.dart' as sqlite; export 'library_database_models.dart'; @@ -76,6 +77,118 @@ class LibraryDatabase { bool get searchFtsAvailable => _searchFtsAvailable ?? false; + Future> _nativeScanRequest(String sourceId) async { + final db = await database; + final history = await HistoryDatabase.instance.database; + final sources = await db.query( + 'library_sources', + columns: ['path', 'enabled'], + where: 'id = ?', + whereArgs: [sourceId], + limit: 1, + ); + if (sources.isEmpty || sources.single['enabled'] != 1) { + throw StateError('Library source changed or was removed during scan'); + } + return { + 'library_path': db.path, + 'history_path': history.path, + 'source_id': sourceId, + 'expected_source_path': sources.single['path'], + 'platform': Platform.isAndroid ? 'android' : 'ios', + }; + } + + /// Keeps native scan output native through parsing, indexing and the atomic + /// source swap. The stream implementation remains the desktop/test adapter. + Future<({int inserted, int skipped})> replaceSourceScanFile( + String sourceId, + LibraryScanNDJSONFile scan, { + required String requestId, + required bool Function() isCancelled, + }) async { + if (isCancelled()) throw StateError('Library scan cancelled'); + if (PlatformBridge.supportsCoreBackend) { + final request = await _nativeScanRequest(sourceId); + if (isCancelled()) throw StateError('Library scan cancelled'); + final result = await PlatformBridge.runNativeDataJob({ + ...request, + 'operation': 'library_full_import', + 'ndjson_path': scan.file.path, + 'expected_count': scan.expectedCount, + 'error_count': scan.errorCount, + }, requestId: requestId); + return ( + inserted: (result['inserted'] as num).toInt(), + skipped: (result['skipped'] as num).toInt(), + ); + } + Stream> validatedRows() async* { + var count = 0; + await for (final row in scan.rows()) { + if (isCancelled()) { + throw StateError('Library scan cancelled during ingestion'); + } + count++; + yield row; + } + if (count != scan.expectedCount) { + throw FormatException( + 'Library scan row count mismatch: decoded $count, ' + 'expected ${scan.expectedCount}', + ); + } + } + + return replaceSourceStream( + sourceId, + validatedRows(), + preserveMissing: scan.errorCount > 0, + ); + } + + /// Exports timestamps directly from the DB on the worker. Android's adapter + /// backfills zero SAF timestamps in bounded ContentResolver batches first. + Future writeNativeFileModTimesSnapshot( + String sourceId, { + required String requestId, + required bool Function() isCancelled, + }) async { + final request = await _nativeScanRequest(sourceId); + if (isCancelled()) throw StateError('Library scan cancelled'); + final directory = await getTemporaryDirectory(); + if (isCancelled()) throw StateError('Library scan cancelled'); + final result = await PlatformBridge.runNativeDataJob({ + ...request, + 'operation': 'library_snapshot', + 'snapshot_path': join( + directory.path, + 'library_file_mod_times_${DateTime.now().microsecondsSinceEpoch}.tsv', + ), + }, requestId: requestId); + return result['path'] as String; + } + + /// Native folder scanning and delta ingestion share one operation, so delta + /// track maps never return to Flutter. SAF traversal stays in its OS adapter. + Future> scanIncrementalNative( + String sourceId, + String folderPath, + String snapshotPath, { + required bool isSaf, + required String requestId, + required bool Function() isCancelled, + }) async { + final request = await _nativeScanRequest(sourceId); + if (isCancelled()) throw StateError('Library scan cancelled'); + return PlatformBridge.runNativeDataJob({ + ...request, + 'operation': 'library_scan_incremental', + if (isSaf) 'tree_uri': folderPath else 'folder_path': folderPath, + 'snapshot_path': snapshotPath, + }, requestId: requestId); + } + Future> searchLibrary({ required String query, required LibrarySearchKind kind, diff --git a/rust_backend/crates/mobile/src/data_jobs.rs b/rust_backend/crates/mobile/src/data_jobs.rs index 0ac83551..7311d51b 100644 --- a/rust_backend/crates/mobile/src/data_jobs.rs +++ b/rust_backend/crates/mobile/src/data_jobs.rs @@ -55,7 +55,7 @@ fn execute( request: &Value, bytes: &[u8], check: &(dyn Fn() -> Result<(), String> + Sync), - _manager: Option<&ExtensionManager>, + manager: Option<&ExtensionManager>, ) -> Result { let operation = string(request, "operation")?; match operation { @@ -120,6 +120,14 @@ fn execute( )?; Ok(json!({"cutoff": cutoff})) } + "library_scan_incremental" => crate::native_library::scan_incremental( + manager.ok_or("Library scan requires the initialized backend")?, + request, + check, + ), + operation if operation.starts_with("library_") => { + crate::native_library::execute(request, check) + } _ => Err(format!("Unknown native data operation: {operation}")), } } diff --git a/rust_backend/crates/mobile/src/lib.rs b/rust_backend/crates/mobile/src/lib.rs index 5aa0a566..2a76089f 100644 --- a/rust_backend/crates/mobile/src/lib.rs +++ b/rust_backend/crates/mobile/src/lib.rs @@ -12,6 +12,7 @@ mod lyrics; mod manager; mod metadata; mod native_id3; +mod native_library; mod native_listing; mod native_lyrics; mod native_playlists; diff --git a/rust_backend/crates/mobile/src/native_library.rs b/rust_backend/crates/mobile/src/native_library.rs new file mode 100644 index 00000000..7753a662 --- /dev/null +++ b/rust_backend/crates/mobile/src/native_library.rs @@ -0,0 +1,683 @@ +//! App-owned Library persistence. The schema remains owned by Dart; scans enter +//! through one worker call and never transfer a library-sized row list to UI. + +pub(crate) mod dart_date; +pub(crate) mod path_keys; + +use crate::manager::ExtensionManager; +use rusqlite::{Connection, OpenFlags, TransactionBehavior, params, params_from_iter}; +use serde_json::{Map, Value, json}; +use std::{ + fs::{self, File, OpenOptions}, + io::{BufRead, BufReader, BufWriter, Read, Write}, + path::Path, + time::Duration, +}; + +const SCHEMA_VERSION: i64 = 17; +const SCAN_VERSION: i64 = 4; +const BATCH_SIZE: usize = 300; +const MAX_ROW_BYTES: usize = 16 * 1024 * 1024; +const COLUMNS: &[(&str, &str)] = &[ + ("id", "id"), + ("trackName", "track_name"), + ("artistName", "artist_name"), + ("albumName", "album_name"), + ("albumArtist", "album_artist"), + ("filePath", "file_path"), + ("coverPath", "cover_path"), + ("scannedAt", "scanned_at"), + ("fileModTime", "file_mod_time"), + ("isrc", "isrc"), + ("trackNumber", "track_number"), + ("totalTracks", "total_tracks"), + ("discNumber", "disc_number"), + ("totalDiscs", "total_discs"), + ("duration", "duration"), + ("releaseDate", "release_date"), + ("bitDepth", "bit_depth"), + ("sampleRate", "sample_rate"), + ("bitrate", "bitrate"), + ("genre", "genre"), + ("composer", "composer"), + ("label", "label"), + ("copyright", "copyright"), + ("format", "format"), +]; + +type Check<'a> = &'a dyn Fn() -> Result<(), String>; + +fn error(context: &str, error: impl std::fmt::Display) -> String { + format!("{context}: {error}") +} + +fn required<'a>(request: &'a Value, key: &str) -> Result<&'a str, String> { + request[key] + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| format!("Library request requires {key}")) +} + +fn open(request: &Value, with_history: bool) -> Result { + let path = required(request, "library_path")?; + if !Path::new(path).is_absolute() + || Path::new(path).file_name().and_then(|v| v.to_str()) != Some("local_library.db") + { + return Err("Library database path must reference local_library.db".into()); + } + // Never create a missing DB or migrate it from a worker. Dart initializes + // the current schema and optional FTS/triggers before handing us this path. + let db = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE) + .map_err(|e| error("open Library", e))?; + db.busy_timeout(Duration::from_secs(5)) + .map_err(|e| error("Library busy timeout", e))?; + let version: i64 = db + .pragma_query_value(None, "user_version", |row| row.get(0)) + .map_err(|e| error("Library schema version", e))?; + if version != SCHEMA_VERSION { + return Err(format!( + "Library schema v{version} is incompatible with native v{SCHEMA_VERSION}" + )); + } + db.execute_batch("PRAGMA recursive_triggers = ON; PRAGMA synchronous = NORMAL;") + .map_err(|e| error("configure Library", e))?; + if with_history { + let history = required(request, "history_path")?; + if !Path::new(history).is_absolute() + || Path::new(history).file_name().and_then(|v| v.to_str()) != Some("history.db") + || !Path::new(history).is_file() + { + return Err("History database must already exist".into()); + } + db.execute("ATTACH DATABASE ? AS history_db", [history]) + .map_err(|e| error("attach History", e))?; + } + Ok(db) +} + +fn source_current(db: &Connection, request: &Value) -> Result<(), String> { + let source = required(request, "source_id")?; + let path = required(request, "expected_source_path")?; + let current: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM library_sources WHERE id = ? AND path = ? AND enabled = 1)", + params![source, path], |row| row.get(0), + ).map_err(|e| error("check Library source", e))?; + if !current { + return Err("Library source changed or was removed during scan".into()); + } + Ok(()) +} + +/// Unified native-data dispatcher entry point. All filesystem/SQLite work must +/// execute on the platform worker, never on its method-channel/main thread. +pub(crate) fn execute(request: &Value, check: Check<'_>) -> Result { + check()?; + match required(request, "operation")? { + "library_full_import" => import(request, None, check), + "library_incremental_import" => { + let owned; + let delta = if let Some(path) = request["delta_path"].as_str() { + let file = File::open(path).map_err(|e| error("open scan delta", e))?; + owned = serde_json::from_reader(BufReader::new(file)) + .map_err(|e| error("decode scan delta file", e))?; + &owned + } else if let Some(text) = request["delta_json"].as_str() { + owned = serde_json::from_str(text).map_err(|e| error("decode scan delta", e))?; + &owned + } else { + &request["delta"] + }; + if !delta.is_object() { + return Err("Library delta must be an object".into()); + } + import(request, Some(delta), check) + } + "library_snapshot" => snapshot(request, check), + "library_snapshot_candidates" => snapshot_candidates(request, check), + _ => Err("Unknown native Library operation".into()), + } +} + +/// The existing native scanner's delta stays native through ingestion. The +/// platform SAF adapter uses library_incremental_import with its native delta. +pub(crate) fn scan_incremental( + manager: &ExtensionManager, + request: &Value, + check: &(dyn Fn() -> Result<(), String> + Sync), +) -> Result { + let delta = manager + .inner + .scan_library_folder_incremental_from_snapshot( + required(request, "folder_path")?, + required(request, "snapshot_path")?, + check, + )?; + check()?; + if delta["cancelled"] == true { + return Err("Library scan cancelled".into()); + } + import(request, Some(&delta), check) +} + +fn normalized(value: &str) -> String { + path_keys::dart_lower(path_keys::dart_trim(value)) +} + +fn text<'a>(row: &'a Value, key: &str) -> Result<&'a str, String> { + match &row[key] { + Value::Null => Ok(""), + Value::String(value) => Ok(value), + _ => Err(format!("Library row {key} must be a string")), + } +} + +fn flag(value: &Value) -> i64 { + i64::from(value == &Value::Bool(true) || value.as_f64() == Some(1.0)) +} + +fn has_replaygain(row: &Value) -> bool { + if flag(&row["hasReplayGain"]) != 0 { + return true; + } + for key in ["replaygain_track_gain", "replaygain_album_gain"] { + let value = match &row[key] { + Value::String(s) => s.clone(), + Value::Number(n) => n.to_string(), + _ => continue, + }; + let trimmed = path_keys::dart_trim(&value); + let lowered = trimmed.to_lowercase(); + let number = if lowered.ends_with("db") { + &trimmed[..trimmed.len() - 2] + } else { + trimmed + }; + if number.trim().parse::().is_ok_and(f64::is_finite) { + return true; + } + } + false +} + +fn numeric(value: &Value) -> Option { + value + .as_i64() + .or_else(|| value.as_f64().filter(|n| n.is_finite()).map(|n| n as i64)) +} + +fn scan_timestamp(value: &str) -> i64 { + dart_date::parse(value) + .map(|date| date.instant.timestamp_millis()) + .unwrap_or(0) +} + +fn incremental_row(row: &Value) -> Result { + let mut canonical = row.clone(); + let timestamp = text(row, "scannedAt")?; + let parsed = + dart_date::parse(timestamp).ok_or("Invalid incremental Library scannedAt timestamp")?; + let formatted = dart_date::iso(&parsed); + canonical["scannedAt"] = formatted.into(); + // Preserve the existing LocalLibraryItem.fromJson(...).toJson() contract, + // including its numeric truncation and scan-version defaults. Full-scan + // imports retain the raw scan flags through LibraryRowMapper instead. + let object = canonical + .as_object_mut() + .ok_or("Library row must be an object")?; + object.remove("audioMetadataScanVersion"); + object.remove("metadataFromFilename"); + for key in [ + "fileModTime", + "trackNumber", + "totalTracks", + "discNumber", + "totalDiscs", + "duration", + "bitDepth", + "sampleRate", + "bitrate", + ] { + if let Some(value) = object.get(key) + && !value.is_null() + { + let value = numeric(value) + .ok_or_else(|| format!("Incremental Library {key} must be numeric"))?; + object.insert(key.into(), value.into()); + } + } + Ok(canonical) +} + +fn encode(row: &Value, source: &str) -> Result, String> { + if !row.is_object() || path_keys::dart_trim(text(row, "id")?).is_empty() { + return Err("Library scan row has no valid id".into()); + } + let mut values = Map::new(); + for key in ["audioMetadataScanVersion", "fileModTime"] { + if !row[key].is_null() && !row[key].is_number() { + return Err(format!("Library {key} must be numeric")); + } + } + for (json_key, column) in COLUMNS { + values.insert((*column).into(), row[*json_key].clone()); + } + // Staging CTAS does not retain NOT NULL constraints. Validate required + // storage fields before staging so corrupt input cannot survive to swap. + for key in [ + "trackName", + "artistName", + "albumName", + "filePath", + "scannedAt", + ] { + if !row[key].is_string() { + return Err(format!("Library scan row requires string {key}")); + } + } + values.insert("source_id".into(), source.into()); + values.insert("explicit".into(), flag(&row["explicit"]).into()); + values.insert("has_lyrics".into(), flag(&row["hasLyrics"]).into()); + values.insert( + "has_replaygain".into(), + i64::from(has_replaygain(row)).into(), + ); + values.insert( + "audio_metadata_scan_version".into(), + numeric(&row["audioMetadataScanVersion"]) + .unwrap_or(if row["metadataFromFilename"] == true { + 0 + } else { + SCAN_VERSION + }) + .into(), + ); + let track = normalized(text(row, "trackName")?); + let artist = normalized(text(row, "artistName")?); + let album = normalized(text(row, "albumName")?); + let album_artist = if row["albumArtist"].is_null() { + artist.clone() + } else { + normalized(text(row, "albumArtist")?) + }; + for (key, value) in [ + ("track_name_norm", track.clone()), + ("artist_name_norm", artist.clone()), + ("album_name_norm", album.clone()), + ("album_artist_norm", album_artist.clone()), + ("match_key", format!("{track}|{artist}")), + ("album_key", format!("{album}|{album_artist}")), + ] { + values.insert(key.into(), value.into()); + } + let search_album_artist = if path_keys::dart_trim(text(row, "albumArtist")?).is_empty() { + artist.clone() + } else { + album_artist + }; + values.insert( + "search_text".into(), + [track, artist, album, search_album_artist] + .into_iter() + .filter(|value| !value.is_empty()) + .collect::>() + .join(" ") + .into(), + ); + values.insert("sort_genre".into(), normalized(text(row, "genre")?).into()); + values.insert( + "sort_release".into(), + path_keys::dart_trim(text(row, "releaseDate")?).into(), + ); + values.insert( + "sort_added".into(), + numeric(&row["fileModTime"]) + .unwrap_or_else(|| scan_timestamp(text(row, "scannedAt").unwrap_or(""))) + .into(), + ); + Ok(values) +} + +fn sql_value(value: &Value) -> Result { + use rusqlite::types::Value as SqlValue; + match value { + Value::Null => Ok(SqlValue::Null), + Value::String(s) => Ok(SqlValue::Text(s.clone())), + Value::Bool(v) => Ok(SqlValue::Integer(i64::from(*v))), + Value::Number(n) => n + .as_i64() + .map(SqlValue::Integer) + .or_else(|| n.as_f64().map(SqlValue::Real)) + .ok_or("Invalid Library number".into()), + _ => Err("Library storage field cannot contain an object or array".into()), + } +} + +fn import(request: &Value, delta: Option<&Value>, check: Check<'_>) -> Result { + let mut db = open(request, true)?; + source_current(&db, request)?; + let source = required(request, "source_id")?; + let columns = db + .prepare("PRAGMA table_info(library)") + .map_err(|e| error("Library columns", e))? + .query_map([], |row| row.get::<_, String>(1)) + .map_err(|e| error("Library columns", e))? + .collect::, _>>() + .map_err(|e| error("Library columns", e))?; + let column_list = columns + .iter() + .map(|name| format!("\"{}\"", name.replace('"', "\"\""))) + .collect::>() + .join(", "); + db.execute_batch("CREATE TEMP TABLE native_library_stage AS SELECT * FROM library WHERE 0; + CREATE UNIQUE INDEX native_library_stage_id ON native_library_stage(id); + CREATE UNIQUE INDEX native_library_stage_path ON native_library_stage(file_path); + CREATE TEMP TABLE native_library_keys(item_id TEXT NOT NULL, path_key TEXT NOT NULL, PRIMARY KEY(item_id,path_key)); + CREATE INDEX native_library_keys_key ON native_library_keys(path_key);") + .map_err(|e| error("stage Library scan", e))?; + let placeholders = vec!["?"; columns.len()].join(", "); + let insert = format!( + "INSERT OR REPLACE INTO native_library_stage ({column_list}) VALUES ({placeholders})" + ); + let android = request["platform"] == "android"; + let mut streamed = 0_usize; + let mut pending = Vec::with_capacity(BATCH_SIZE); + let mut flush = |rows: &mut Vec| -> Result<(), String> { + if rows.is_empty() { + return Ok(()); + } + check()?; + let transaction = db + .transaction() + .map_err(|e| error("stage transaction", e))?; + for row in rows.drain(..) { + check()?; + let row = if delta.is_some() { + incremental_row(&row)? + } else { + row + }; + let mapped = encode(&row, source)?; + let fields = columns + .iter() + .map(|column| sql_value(mapped.get(column).unwrap_or(&Value::Null))) + .collect::, _>>()?; + transaction + .prepare_cached(&insert) + .map_err(|e| error("prepare staged row", e))? + .execute(params_from_iter(fields)) + .map_err(|e| error("stage Library row", e))?; + let id = text(&row, "id")?; + transaction + .execute("DELETE FROM native_library_keys WHERE item_id = ?", [id]) + .map_err(|e| error("replace staged keys", e))?; + for key in path_keys::build(text(&row, "filePath")?, android)? { + transaction + .prepare_cached("INSERT OR IGNORE INTO native_library_keys VALUES (?, ?)") + .map_err(|e| error("prepare staged keys", e))? + .execute(params![id, key]) + .map_err(|e| error("stage path key", e))?; + } + } + transaction + .commit() + .map_err(|e| error("commit staged batch", e))?; + Ok(()) + }; + if let Some(delta) = delta { + let rows = delta["files"] + .as_array() + .or_else(|| delta["scanned"].as_array()) + .ok_or("Native incremental scan requires a rows array")?; + for row in rows { + pending.push(row.clone()); + streamed += 1; + if pending.len() == BATCH_SIZE { + flush(&mut pending)?; + } + } + } else { + let expected = request["expected_count"] + .as_u64() + .ok_or("Full scan requires expected_count")?; + let mut reader = BufReader::new( + File::open(required(request, "ndjson_path")?) + .map_err(|e| error("open scan NDJSON", e))?, + ); + let mut line = String::new(); + loop { + check()?; + line.clear(); + let bytes = reader + .by_ref() + .take((MAX_ROW_BYTES + 1) as u64) + .read_line(&mut line) + .map_err(|e| error("read scan NDJSON", e))?; + if bytes == 0 { + break; + } + if bytes > MAX_ROW_BYTES { + return Err("Library scan NDJSON row is too large".into()); + } + if path_keys::dart_trim(&line).is_empty() { + continue; + } + pending.push(serde_json::from_str(&line).map_err(|e| error("decode scan NDJSON", e))?); + streamed += 1; + if pending.len() == BATCH_SIZE { + flush(&mut pending)?; + } + } + if streamed as u64 != expected { + return Err(format!( + "Library scan row count mismatch: decoded {streamed}, expected {expected}" + )); + } + } + flush(&mut pending)?; + check()?; + let transaction = db + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| error("begin Library swap", e))?; + source_current(&transaction, request)?; + let history_match = "EXISTS(SELECT 1 FROM native_library_keys sk JOIN history_db.history_path_keys hk ON hk.path_key = sk.path_key WHERE sk.item_id = s.id)"; + let staged: i64 = transaction + .query_row("SELECT COUNT(*) FROM native_library_stage", [], |row| { + row.get(0) + }) + .map_err(|e| error("count staged rows", e))?; + let skipped: i64 = transaction + .query_row( + &format!("SELECT COUNT(*) FROM native_library_stage s WHERE {history_match}"), + [], + |row| row.get(0), + ) + .map_err(|e| error("count excluded downloads", e))?; + let mut deleted = 0; + let mut excluded_existing = 0; + if let Some(delta) = delta { + // Remove files which became downloads, then apply this source's delta + // atomically. All deletes remain source-scoped even if aliases overlap. + let excluded = "source_id = ? AND EXISTS(SELECT 1 FROM library_path_keys lk JOIN history_db.history_path_keys hk ON hk.path_key = lk.path_key WHERE lk.item_id = library.id)"; + transaction.execute(&format!("CREATE TEMP TABLE native_library_excluded AS SELECT id FROM library WHERE {excluded}"), [source]).map_err(|e| error("select downloaded rows", e))?; + transaction.execute("DELETE FROM library_path_keys WHERE item_id IN(SELECT id FROM native_library_excluded)", []).map_err(|e| error("remove downloaded path keys", e))?; + excluded_existing += transaction + .execute( + "DELETE FROM library WHERE id IN(SELECT id FROM native_library_excluded)", + [], + ) + .map_err(|e| error("remove downloaded rows", e))?; + let paths = delta["removedUris"] + .as_array() + .or_else(|| delta["deletedPaths"].as_array()); + if let Some(paths) = paths { + for path in paths { + check()?; + let path = path + .as_str() + .ok_or("Deleted Library path must be a string")?; + transaction.execute("DELETE FROM library_path_keys WHERE item_id IN(SELECT id FROM library WHERE source_id = ? AND file_path = ?)", params![source,path]).map_err(|e| error("delete removed path keys", e))?; + deleted += transaction + .execute( + "DELETE FROM library WHERE source_id = ? AND file_path = ?", + params![source, path], + ) + .map_err(|e| error("delete removed track", e))?; + } + } + // staged/download overlap is the same anti-join used by full scans. + let ids = format!( + "SELECT id FROM library WHERE source_id = ? AND file_path IN(SELECT file_path FROM native_library_stage s WHERE {history_match})" + ); + transaction + .execute( + &format!("DELETE FROM library_path_keys WHERE item_id IN({ids})"), + [source], + ) + .map_err(|e| error("remove excluded keys", e))?; + excluded_existing += transaction + .execute(&format!("DELETE FROM library WHERE id IN({ids})"), [source]) + .map_err(|e| error("remove excluded rows", e))?; + } else { + let errors = request["error_count"] + .as_u64() + .ok_or("Full scan requires nonnegative error_count")?; + let where_clause = if errors > 0 { + "source_id = ? AND (id IN(SELECT id FROM native_library_stage) OR file_path IN(SELECT file_path FROM native_library_stage))" + } else { + "source_id = ?" + }; + transaction.execute(&format!("DELETE FROM library_path_keys WHERE item_id IN(SELECT id FROM library WHERE {where_clause})"), [source]).map_err(|e| error("replace source keys", e))?; + transaction + .execute( + &format!("DELETE FROM library WHERE {where_clause}"), + [source], + ) + .map_err(|e| error("replace source rows", e))?; + } + check()?; + transaction.execute(&format!("DELETE FROM library_path_keys WHERE item_id IN(SELECT s.id FROM native_library_stage s WHERE NOT {history_match})"), []).map_err(|e| error("replace imported keys", e))?; + transaction.execute(&format!("INSERT OR REPLACE INTO library({column_list}) SELECT {} FROM native_library_stage s WHERE NOT {history_match}", columns.iter().map(|c| format!("s.\"{c}\"")).collect::>().join(", ")), []).map_err(|e| error("swap imported rows", e))?; + transaction.execute(&format!("INSERT OR IGNORE INTO library_path_keys SELECT sk.item_id,sk.path_key FROM native_library_keys sk JOIN native_library_stage s ON s.id=sk.item_id WHERE NOT {history_match}"), []).map_err(|e| error("swap imported keys", e))?; + check()?; + transaction + .commit() + .map_err(|e| error("commit Library swap", e))?; + Ok( + json!({"committed":true,"inserted":staged-skipped,"upserted":staged-skipped,"skipped":skipped+excluded_existing as i64,"scanned_count":streamed,"deleted_count":deleted,"skippedCount":delta.map(|d|d["skippedCount"].clone()).unwrap_or(json!(0)),"totalFiles":delta.map(|d|d["totalFiles"].clone()).unwrap_or(json!(streamed))}), + ) +} + +fn snapshot_candidates(request: &Value, check: Check<'_>) -> Result { + let db = open(request, false)?; + source_current(&db, request)?; + let cursor = request["cursor"].as_str().unwrap_or(""); + let limit = request["limit"].as_i64().unwrap_or(500).clamp(1, 500); + let mut statement = db.prepare("SELECT id,file_path FROM library WHERE source_id=? AND id>? AND COALESCE(file_mod_time,0)=0 AND audio_metadata_scan_version>=4 AND file_path LIKE 'content://%' ORDER BY id LIMIT ?").map_err(|e| error("prepare snapshot backfill", e))?; + let rows = statement + .query_map( + params![required(request, "source_id")?, cursor, limit + 1], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .map_err(|e| error("query snapshot backfill", e))? + .collect::, _>>() + .map_err(|e| error("read snapshot backfill", e))?; + check()?; + let has_more = rows.len() > limit as usize; + let rows = rows.into_iter().take(limit as usize).collect::>(); + Ok( + json!({"paths":rows.iter().map(|(id,path)|json!({"id":id,"file_path":path})).collect::>(),"cursor":rows.last().map(|(id,_)|id.as_str()).unwrap_or(cursor),"has_more":has_more}), + ) +} + +fn snapshot(request: &Value, check: Check<'_>) -> Result { + let mut db = open(request, false)?; + source_current(&db, request)?; + let path = required(request, "snapshot_path")?; + if !Path::new(path).is_absolute() || !path.ends_with(".tsv") { + return Err("Snapshot output must be an absolute TSV path".into()); + } + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| error("create incremental snapshot", e))?; + let result = (|| { + // A WAL read snapshot does not hold the persistent writer during file + // inspection/output. Backfill acquires its own short write transaction. + let transaction = db.transaction().map_err(|e| error("begin snapshot", e))?; + source_current(&transaction, request)?; + let mut statement = transaction.prepare("SELECT file_path,COALESCE(file_mod_time,0),COALESCE(audio_metadata_scan_version,0) FROM library WHERE source_id=? ORDER BY id").map_err(|e| error("prepare snapshot", e))?; + let mut rows = statement + .query([required(request, "source_id")?]) + .map_err(|e| error("query snapshot", e))?; + let mut output = BufWriter::new(file); + let mut count = 0; + let mut backfilled = Vec::new(); + while let Some(row) = rows.next().map_err(|e| error("read snapshot", e))? { + check()?; + let file_path: String = row.get(0).map_err(|e| error("snapshot file path", e))?; + let stored: i64 = row.get(1).map_err(|e| error("snapshot timestamp", e))?; + let version: i64 = row.get(2).map_err(|e| error("snapshot scan version", e))?; + let mut timestamp = if version < SCAN_VERSION { -1 } else { stored }; + if timestamp == 0 { + let supplied = + numeric(&request["backfilled_mod_times"][&file_path]).filter(|v| *v > 0); + let discovered = if !file_path.starts_with("content://") + && !file_path.starts_with("network://") + { + fs::metadata(&file_path) + .ok() + .filter(|m| m.is_file()) + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis() as i64) + } else { + None + }; + if let Some(time) = supplied.or(discovered) { + timestamp = time; + backfilled.push((file_path.clone(), time)); + } + } + if file_path.is_empty() { + continue; + } + if file_path.contains(['\n', '\r']) { + return Err("Snapshot file path contains a line break".into()); + } + writeln!(output, "{timestamp}\t{file_path}").map_err(|e| error("write snapshot", e))?; + count += 1; + } + drop(rows); + drop(statement); + transaction + .commit() + .map_err(|e| error("close snapshot read", e))?; + output.flush().map_err(|e| error("flush snapshot", e))?; + output + .get_ref() + .sync_all() + .map_err(|e| error("sync snapshot", e))?; + check()?; + let write = db + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|e| error("begin timestamp backfill", e))?; + source_current(&write, request)?; + for (file_path, timestamp) in &backfilled { + check()?; + write.execute("UPDATE library SET file_mod_time=?,sort_added=? WHERE source_id=? AND file_path=? AND COALESCE(file_mod_time,0)=0 AND audio_metadata_scan_version>=4",params![timestamp,timestamp,required(request,"source_id")?,file_path]).map_err(|e| error("backfill timestamp", e))?; + } + check()?; + write + .commit() + .map_err(|e| error("commit timestamp backfill", e))?; + Ok(json!({"committed":true,"path":path,"count":count,"backfilled_count":backfilled.len()})) + })(); + if result.is_err() { + let _ = fs::remove_file(path); + } + result +} + +#[cfg(test)] +mod tests; diff --git a/rust_backend/crates/mobile/src/native_library/dart_date.rs b/rust_backend/crates/mobile/src/native_library/dart_date.rs new file mode 100644 index 00000000..2b4a6f44 --- /dev/null +++ b/rust_backend/crates/mobile/src/native_library/dart_date.rs @@ -0,0 +1,99 @@ +//! DateTime.parse's accepted ISO subset, including compact dates, fractional +//! truncation and overflowing components used by old JSON backups. +use chrono::{DateTime, Duration, Local, NaiveDate, NaiveDateTime, SecondsFormat, TimeZone, Utc}; +use regex::Regex; +use std::sync::OnceLock; + +pub(crate) struct Parsed { + pub instant: DateTime, + pub local: Option, +} + +pub(crate) fn parse(value: &str) -> Option { + static FORMAT: OnceLock = OnceLock::new(); + let format=FORMAT.get_or_init(||Regex::new(r"^([+-]?[0-9]{4,6})-?([0-9]{2})-?([0-9]{2})(?:[ T]([0-9]{2})(?::?([0-9]{2})(?::?([0-9]{2})(?:[.,]([0-9]+))?)?)?( ?[zZ]| ?([-+])([0-9]{2})(?::?([0-9]{2}))?)?)?$").expect("Dart ISO pattern")); + let parts = format.captures(value)?; + let integer = |index| { + parts + .get(index) + .map_or(Some(0_i64), |v| v.as_str().parse::().ok()) + }; + let year = integer(1)?; + let month = integer(2)? - 1; + let normalized_year = year.checked_add(month.div_euclid(12))?; + let normalized_month = month.rem_euclid(12) + 1; + let day = NaiveDate::from_ymd_opt( + i32::try_from(normalized_year).ok()?, + normalized_month as u32, + 1, + )?; + let mut minutes = integer(5)?; + if parts.get(9).is_some() { + let sign = if parts.get(9)?.as_str() == "-" { -1 } else { 1 }; + minutes -= sign * (60 * integer(10)? + integer(11)?); + } + let fraction = parts + .get(7) + .map(|v| { + v.as_str() + .bytes() + .take(6) + .fold((0_i64, 0), |(n, len), digit| { + (n * 10 + i64::from(digit - b'0'), len + 1) + }) + }) + .unwrap_or((0, 6)); + let micros = fraction.0 * 10_i64.pow(6 - fraction.1); + let seconds = (integer(3)? - 1) + .checked_mul(86400)? + .checked_add(integer(4)?.checked_mul(3600)?)? + .checked_add(minutes.checked_mul(60)?)? + .checked_add(integer(6)?)?; + let naive = day + .and_hms_opt(0, 0, 0)? + .checked_add_signed(Duration::seconds(seconds))? + .checked_add_signed(Duration::microseconds(micros))?; + let utc = parts.get(8).is_some(); + let instant = if utc { + naive.and_utc() + } else { + Local + .from_local_datetime(&naive) + .earliest()? + .with_timezone(&Utc) + }; + Some(Parsed { + instant, + local: if utc { None } else { Some(naive) }, + }) +} + +pub(crate) fn iso_utc(value: &DateTime) -> String { + value.to_rfc3339_opts( + if value.timestamp_subsec_nanos().is_multiple_of(1_000_000) { + SecondsFormat::Millis + } else { + SecondsFormat::Micros + }, + true, + ) +} + +pub(crate) fn iso(value: &Parsed) -> String { + match value.local { + None => iso_utc(&value.instant), + Some(date) => date + .format( + if date + .and_utc() + .timestamp_subsec_nanos() + .is_multiple_of(1_000_000) + { + "%Y-%m-%dT%H:%M:%S%.3f" + } else { + "%Y-%m-%dT%H:%M:%S%.6f" + }, + ) + .to_string(), + } +} diff --git a/rust_backend/crates/mobile/src/native_library/path_keys.rs b/rust_backend/crates/mobile/src/native_library/path_keys.rs new file mode 100644 index 00000000..25adffed --- /dev/null +++ b/rust_backend/crates/mobile/src/native_library/path_keys.rs @@ -0,0 +1,316 @@ +//! Physical/converted path identities shared with Dart's path_match_keys.dart. +//! Opaque SAF document identities decode once and preserve case and extension. +use base64::{Engine, engine::general_purpose::URL_SAFE}; +use std::collections::{BTreeSet, VecDeque}; +use url::Url; + +const ALIASES: &[&str] = &[ + "/storage/emulated/0", + "/storage/emulated/legacy", + "/storage/self/primary", + "/sdcard", + "/mnt/sdcard", +]; +const EXTENSIONS: &[&str] = &[ + ".flac", ".m4a", ".mp3", ".opus", ".ogg", ".wav", ".aiff", ".aif", ".aac", +]; + +pub(crate) fn dart_trim(value: &str) -> &str { + value.trim_matches(|ch: char| ch.is_whitespace() || ch == '\u{feff}') +} + +/// Dart applies Unicode simple casing, without multi-character expansions or +/// the final-sigma context rules of Rust's full string lowercase operation. +pub(crate) fn dart_lower(value: &str) -> String { + value + .chars() + .map(|ch| ch.to_lowercase().next().unwrap_or(ch)) + .collect() +} + +fn decode(value: &str) -> Option { + let bytes = value.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' { + let high = (*bytes.get(index + 1)? as char).to_digit(16)?; + let low = (*bytes.get(index + 2)? as char).to_digit(16)?; + decoded.push((high * 16 + low) as u8); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + String::from_utf8(decoded).ok() +} + +fn dart_uri(value: &str) -> Result { + // Uri.parse escapes stray '%' before pathSegments decodes them. Keeping + // them unescaped would lose opaque document identities with literal '%'. + let mut sanitized = String::with_capacity(value.len()); + let bytes = value.as_bytes(); + for (index, ch) in value.char_indices() { + if ch == '%' + && !(bytes.get(index + 1).is_some_and(u8::is_ascii_hexdigit) + && bytes.get(index + 2).is_some_and(u8::is_ascii_hexdigit)) + { + sanitized.push_str("%25"); + } else { + sanitized.push(ch); + } + } + Url::parse(&sanitized) +} + +fn segments(uri: &Url) -> Option> { + uri.path_segments()?.map(decode).collect() +} + +fn document_key(uri: &Url) -> Option { + if uri.scheme() != "content" { + return None; + } + let authority = &uri[url::Position::BeforeUsername..url::Position::AfterPort]; + if authority.is_empty() { + return None; + } + let parts = segments(uri)?; + let direct = parts.len() == 2 && parts[0] == "document"; + let tree = + parts.len() == 4 && parts[0] == "tree" && !parts[1].is_empty() && parts[2] == "document"; + let id = parts.last()?; + if !(direct || tree) || id.is_empty() { + return None; + } + Some(format!( + "saf-document:{}:{id}", + URL_SAFE.encode(dart_lower(authority)) + )) +} + +fn storage_documents(uri: &Url) -> Vec { + if uri.scheme() != "content" + || !uri + .host_str() + .is_some_and(|host| host.eq_ignore_ascii_case("com.android.externalstorage.documents")) + { + return Vec::new(); + } + let Some(parts) = segments(uri) else { + return Vec::new(); + }; + let index = parts + .iter() + .rposition(|s| s == "document") + .or_else(|| parts.iter().rposition(|s| s == "tree")); + let Some(index) = index else { + return Vec::new(); + }; + if index + 1 >= parts.len() { + return Vec::new(); + } + let joined = parts[index + 1..].join("/"); + let id = decode(&joined).unwrap_or(joined); + let Some((volume, relative)) = id.split_once(':') else { + return Vec::new(); + }; + if volume.to_lowercase() != "primary" { + return Vec::new(); + } + let relative = relative.replace('\\', "/"); + let relative = relative.trim_start_matches('/'); + ALIASES + .iter() + .map(|prefix| { + if relative.is_empty() { + (*prefix).into() + } else { + format!("{prefix}/{relative}") + } + }) + .collect() +} + +fn android_aliases(value: &str) -> Vec { + let normalized = value.replace('\\', "/"); + let lower = dart_lower(&normalized); + for prefix in ALIASES { + if lower == *prefix || lower.starts_with(&format!("{prefix}/")) { + let suffix = &normalized[prefix.len()..]; + return ALIASES + .iter() + .map(|alias| format!("{alias}{suffix}")) + .collect(); + } + } + Vec::new() +} + +fn uri_text(uri: &Url) -> String { + // Dart canonicalizes percent escape hex and decodes unreserved ASCII in + // Uri.parse. url::Url keeps original percent escapes for custom schemes. + let mut uri = Url::parse(&uri.as_str().replace('\\', "/")).unwrap_or_else(|_| uri.clone()); + if let Some(host) = uri.host_str().map(dart_lower) { + let _ = uri.set_host(Some(&host)); + } + let bytes = uri.as_str().as_bytes(); + let mut output = String::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' + && index + 2 < bytes.len() + && let (Some(high), Some(low)) = ( + (bytes[index + 1] as char).to_digit(16), + (bytes[index + 2] as char).to_digit(16), + ) + { + let value = (high * 16 + low) as u8; + if value.is_ascii_alphanumeric() || b"-._~".contains(&value) { + output.push(value as char); + } else { + output.push_str(&format!("%{value:02X}")); + } + index += 3; + } else { + let tail = &uri.as_str()[index..]; + let ch = tail.chars().next().expect("URI character"); + output.push(ch); + index += ch.len_utf8(); + } + } + output +} + +pub(crate) fn build(path: &str, android: bool) -> Result, String> { + let path = dart_trim(path); + let cleaned = path.strip_prefix("EXISTS:").map(dart_trim).unwrap_or(path); + if cleaned.is_empty() { + return Ok(BTreeSet::new()); + } + let mut keys = BTreeSet::new(); + let mut visited = BTreeSet::new(); + let mut pending = VecDeque::from([cleaned.to_string()]); + if let Ok(uri) = dart_uri(cleaned) + && let Some(key) = document_key(&uri) + { + keys.insert(key); + } + while let Some(value) = pending.pop_front() { + let trimmed = dart_trim(&value); + if trimmed.is_empty() || !visited.insert(trimmed.to_string()) { + continue; + } + // A malicious string can recursively expose many percent-encoded + // variants. The existing cache is bounded; keep the native walk bounded. + if visited.len() > 1024 { + return Err("Too many Library path aliases".into()); + } + keys.insert(trimmed.to_string()); + keys.insert(dart_lower(trimmed)); + if trimmed.contains('\\') { + pending.push_back(trimmed.replace('\\', "/")); + } + if trimmed.contains('%') + && let Some(decoded) = decode(trimmed) + && decoded != trimmed + { + pending.push_back(decoded); + } + if let Ok(uri) = dart_uri(trimmed) { + // replace(query:null, fragment:null) in Dart retains existing fields. + // Preserve that behavior; clearing them would change path identity. + // WHATWG normalizes file://localhost to an empty host. Dart keeps + // that authority and refuses toFilePath on Unix, so do not invent + // a local filesystem alias for it. + let localhost = trimmed + .get(..16) + .is_some_and(|v| v.eq_ignore_ascii_case("file://localhost")) + && (trimmed.len() == 16 + || trimmed + .get(16..17) + .is_some_and(|v| ["/", "?", "#"].contains(&v))); + let text = if localhost { + let alternate = format!("nativefile{}", &trimmed[4..]); + let mut uri = dart_uri(&alternate).map_err(|e| e.to_string())?; + if uri.path().is_empty() { + uri.set_path("/"); + } + uri_text(&uri).replacen("nativefile:", "file:", 1) + } else { + uri_text(&uri) + }; + keys.insert(text.clone()); + keys.insert(dart_lower(&text)); + if uri.scheme() == "file" + && !localhost + && uri.query().is_none() + && uri.fragment().is_none() + && let Ok(file) = uri.to_file_path() + { + pending.push_back(file.to_string_lossy().into_owned()); + } + pending.extend(storage_documents(&uri)); + } else if trimmed.starts_with('/') + && let Ok(uri) = Url::from_file_path(trimmed) + { + let text = uri_text(&uri); + keys.insert(text.clone()); + keys.insert(dart_lower(&text)); + } + if android { + pending.extend(android_aliases(trimmed)); + } + } + let extensionless = keys + .iter() + .filter_map(|key| { + let lower = dart_lower(key); + EXTENSIONS + .iter() + .find(|extension| lower.ends_with(**extension)) + .map(|extension| key[..key.len() - extension.len()].to_string()) + .filter(|stripped| !stripped.is_empty()) + }) + .collect::>(); + keys.extend(extensionless); + Ok(keys) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn saf_document_aliases_keep_opaque_case_and_escape_identity() { + let direct = build("content://nas.provider/document/Ab%252FC.flac", false).unwrap(); + let tree = build( + "content://nas.provider/tree/root/document/Ab%252FC.flac", + false, + ) + .unwrap(); + let expected = format!( + "saf-document:{}:Ab%2FC.flac", + URL_SAFE.encode("nas.provider") + ); + assert!(direct.contains(&expected)); + assert!(tree.contains(&expected)); + assert!(!direct.contains(&expected.replace("Ab%2FC", "ab%2fc"))); + } + #[test] + fn android_storage_and_converted_paths_share_keys() { + let direct = build("/sdcard/Music/Café.flac", true).unwrap(); + let converted = build("/storage/emulated/0/Music/Café.opus", true).unwrap(); + let saf = build("content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9.flac",true).unwrap(); + assert!( + direct + .intersection(&converted) + .any(|key| key == "/storage/emulated/0/Music/Café") + ); + assert!( + direct + .intersection(&saf) + .any(|key| key == "/storage/emulated/0/Music/Café.flac") + ); + } +} diff --git a/rust_backend/crates/mobile/src/native_library/tests.rs b/rust_backend/crates/mobile/src/native_library/tests.rs new file mode 100644 index 00000000..ce042c30 --- /dev/null +++ b/rust_backend/crates/mobile/src/native_library/tests.rs @@ -0,0 +1,465 @@ +use super::*; +use std::cell::Cell; +use tempfile::TempDir; + +struct Fixture { + directory: TempDir, + db: Connection, + request: Value, +} + +impl Fixture { + fn new() -> Self { + let directory = tempfile::tempdir().unwrap(); + let library = directory.path().join("local_library.db"); + let history = directory.path().join("history.db"); + let db = Connection::open(&library).unwrap(); + let example = encode(&track("schema", "/music/schema.flac"), "source").unwrap(); + let definitions = example + .keys() + .map(|column| { + let kind = match column.as_str() { + "id" => "TEXT PRIMARY KEY", + "file_path" => "TEXT UNIQUE", + "file_mod_time" + | "sort_added" + | "audio_metadata_scan_version" + | "explicit" + | "has_lyrics" + | "has_replaygain" => "INTEGER", + _ => "TEXT", + }; + format!("{column} {kind}") + }) + .collect::>() + .join(","); + db.execute_batch(&format!("PRAGMA journal_mode=WAL; PRAGMA user_version=17; + CREATE TABLE library({definitions}); + CREATE TABLE library_sources(id TEXT PRIMARY KEY,path TEXT,enabled INTEGER); + INSERT INTO library_sources VALUES('source','/music',1),('other','/other',1); + CREATE TABLE library_path_keys(item_id TEXT,path_key TEXT,PRIMARY KEY(item_id,path_key)); + CREATE INDEX library_path_key_index ON library_path_keys(path_key); + CREATE VIRTUAL TABLE library_search_fts USING fts5(search_text,content='library',content_rowid='rowid',tokenize='trigram'); + CREATE TRIGGER library_fts_ai AFTER INSERT ON library BEGIN INSERT INTO library_search_fts(rowid,search_text) VALUES(new.rowid,new.search_text); END; + CREATE TRIGGER library_fts_ad AFTER DELETE ON library BEGIN INSERT INTO library_search_fts(library_search_fts,rowid,search_text) VALUES('delete',old.rowid,old.search_text); END;" )).unwrap(); + let history_db = Connection::open(&history).unwrap(); + history_db.execute_batch("CREATE TABLE history_path_keys(item_id TEXT,path_key TEXT); CREATE INDEX history_key_index ON history_path_keys(path_key);").unwrap(); + let request = json!({"operation":"library_full_import","library_path":library.to_str().unwrap(),"history_path":history.to_str().unwrap(),"source_id":"source","expected_source_path":"/music","platform":"ios","error_count":0}); + Self { + directory, + db, + request, + } + } + + fn add(&self, id: &str, path: &str, source: &str) { + let values = encode(&track(id, path), source).unwrap(); + let columns = values.keys().cloned().collect::>(); + self.db + .execute( + &format!( + "INSERT INTO library({}) VALUES({})", + columns.join(","), + vec!["?"; columns.len()].join(",") + ), + params_from_iter(columns.iter().map(|key| sql_value(&values[key]).unwrap())), + ) + .unwrap(); + for key in path_keys::build(path, false).unwrap() { + self.db + .execute( + "INSERT INTO library_path_keys VALUES(?,?)", + params![id, key], + ) + .unwrap(); + } + } + + fn scan( + &self, + rows: &[Value], + expected: u64, + errors: u64, + check: Check<'_>, + ) -> Result { + let path = self.directory.path().join("scan.ndjson"); + let mut output = File::create(&path).unwrap(); + for row in rows { + writeln!(output, "{row}").unwrap(); + } + let mut request = self.request.clone(); + request["ndjson_path"] = path.to_str().unwrap().into(); + request["expected_count"] = expected.into(); + request["error_count"] = errors.into(); + execute(&request, check) + } + + fn ids(&self) -> Vec { + self.db + .prepare("SELECT id FROM library ORDER BY id") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .map(Result::unwrap) + .collect() + } +} + +fn track(id: &str, path: &str) -> Value { + json!({"id":id,"trackName":format!("Track {id}"),"artistName":"Artist","albumName":"Album","albumArtist":null,"filePath":path,"scannedAt":"2026-10-06T12:00:00.000Z","fileModTime":1770000000000_i64,"isrc":"USAAA2600001","hasReplayGain":false,"replaygain_track_gain":"-4.5 dB"}) +} + +#[test] +fn production_dart_row_and_path_codec_parity() { + let fixture: Value = serde_json::from_str(include_str!( + "../../../../../test/fixtures/native_library_contract.json" + )) + .unwrap(); + for entry in fixture["rows"].as_array().unwrap() { + assert_eq!( + Value::Object(encode(&entry["input"], "source").unwrap()), + entry["expected"], + "row {:?}", + entry["input"] + ); + } + let mut mismatches = Vec::new(); + for entry in fixture["paths"].as_array().unwrap() { + let keys = path_keys::build(entry["input"].as_str().unwrap(), false).unwrap(); + let expected = entry["expected"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap().to_string()) + .collect::>(); + if keys != expected { + mismatches.push(format!( + "path {:?}: extra {:?}, missing {:?}", + entry["input"], + keys.difference(&expected).collect::>(), + expected.difference(&keys).collect::>() + )); + } + } + assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); +} + +#[test] +fn cancellation_during_atomic_swap_rolls_back_rows_keys_and_fts() { + let fixture = Fixture::new(); + fixture.add("old", "/music/old.flac", "source"); + let calls = Cell::new(0); + let check = || { + calls.set(calls.get() + 1); + if calls.get() == 8 { + Err("cancelled during swap".into()) + } else { + Ok(()) + } + }; + assert!( + fixture + .scan(&[track("new", "/music/new.flac")], 1, 0, &check) + .is_err() + ); + assert_eq!(calls.get(), 8); + assert_eq!(fixture.ids(), ["old"]); + assert!( + fixture + .db + .query_row( + "SELECT COUNT(*) FROM library_path_keys WHERE item_id='old'", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap() + > 0 + ); + assert_eq!( + fixture + .db + .query_row( + "SELECT COUNT(*) FROM library_search_fts WHERE library_search_fts MATCH ?", + ["old"], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); +} + +#[test] +fn external_wal_writer_can_publish_while_native_scan_stages_temp_rows() { + let fixture = Fixture::new(); + fixture.add("other", "/other/track.flac", "other"); + let path = fixture.request["library_path"].as_str().unwrap().to_owned(); + let calls = Cell::new(0); + let thread = std::cell::RefCell::new(None); + let check = || { + calls.set(calls.get() + 1); + if calls.get() == 305 { + let (sender, receiver) = std::sync::mpsc::channel(); + let path = path.clone(); + let handle = std::thread::spawn(move || { + let db = Connection::open(path).unwrap(); + db.busy_timeout(Duration::from_secs(2)).unwrap(); + db.execute_batch("BEGIN IMMEDIATE;UPDATE library_sources SET path='/writer-kept' WHERE id='other';").unwrap(); + sender.send(()).unwrap(); + std::thread::sleep(Duration::from_millis(50)); + db.execute_batch("COMMIT").unwrap(); + }); + *thread.borrow_mut() = Some(handle); + receiver.recv_timeout(Duration::from_secs(3)).unwrap(); + } + Ok(()) + }; + let rows = (0..300) + .map(|i| track(&format!("new-{i}"), &format!("/music/{i}.flac"))) + .collect::>(); + fixture.scan(&rows, 300, 0, &check).unwrap(); + thread.into_inner().unwrap().join().unwrap(); + assert_eq!(fixture.ids().len(), 301); + assert_eq!( + fixture + .db + .query_row( + "SELECT path FROM library_sources WHERE id='other'", + [], + |row| row.get::<_, String>(0) + ) + .unwrap(), + "/writer-kept" + ); +} + +#[test] +fn spilled_incremental_json_is_consumed_without_deleting_adapter_file() { + let fixture = Fixture::new(); + let path = fixture.directory.path().join("delta.json"); + fs::write(&path,json!({"files":[track("new","/music/new.flac")],"removedUris":[],"skippedCount":3,"totalFiles":4}).to_string()).unwrap(); + let mut request = fixture.request.clone(); + request["operation"] = "library_incremental_import".into(); + request["delta_path"] = path.to_str().unwrap().into(); + let result = execute(&request, &|| Ok(())).unwrap(); + assert_eq!(fixture.ids(), ["new"]); + assert_eq!(result["skippedCount"], 3); + assert!(path.exists()); +} + +#[test] +fn full_swap_excludes_downloads_and_preserves_other_source_and_fts() { + let fixture = Fixture::new(); + fixture.add("old", "/music/old.flac", "source"); + fixture.add("other", "/other/track.flac", "other"); + let history = Connection::open(fixture.request["history_path"].as_str().unwrap()).unwrap(); + history + .execute( + "INSERT INTO history_path_keys VALUES('downloaded','/music/downloaded')", + [], + ) + .unwrap(); + let result = fixture + .scan( + &[ + track("new", "/music/new.flac"), + track("downloaded", "/music/downloaded.opus"), + ], + 2, + 0, + &|| Ok(()), + ) + .unwrap(); + assert_eq!(result["inserted"], 1); + assert_eq!(result["skipped"], 1); + assert_eq!(fixture.ids(), ["new", "other"]); + assert_eq!( + fixture + .db + .query_row( + "SELECT COUNT(*) FROM library_search_fts WHERE library_search_fts MATCH ?", + ["new"], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); + assert_eq!( + fixture + .db + .query_row( + "SELECT COUNT(*) FROM library_search_fts WHERE library_search_fts MATCH ?", + ["old"], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 0 + ); +} + +#[test] +fn invalid_rows_count_mismatch_and_cancel_never_replace_existing_index() { + for failure in ["invalid", "count", "cancel"] { + let fixture = Fixture::new(); + fixture.add("old", "/music/old.flac", "source"); + let mut rows = (0..650) + .map(|index| track(&format!("row-{index}"), &format!("/music/{index}.flac"))) + .collect::>(); + if failure == "invalid" { + rows[640]["id"] = Value::Null; + } + let calls = Cell::new(0); + let check = || { + calls.set(calls.get() + 1); + if failure == "cancel" && calls.get() > 640 { + Err("cancelled".into()) + } else { + Ok(()) + } + }; + assert!( + fixture + .scan(&rows, if failure == "count" { 649 } else { 650 }, 0, &check) + .is_err() + ); + assert_eq!(fixture.ids(), ["old"]); + } +} + +#[test] +fn partial_scan_preserves_unreadable_rows_and_empty_success_clears_only_source() { + let fixture = Fixture::new(); + fixture.add("unreadable", "/music/unreadable.flac", "source"); + fixture.add("old-id", "/music/changed.flac", "source"); + fixture.add("other", "/other/track.flac", "other"); + fixture + .scan(&[track("new-id", "/music/changed.flac")], 1, 1, &|| Ok(())) + .unwrap(); + assert_eq!(fixture.ids(), ["new-id", "other", "unreadable"]); + fixture.scan(&[], 0, 1, &|| Ok(())).unwrap(); + assert_eq!(fixture.ids(), ["new-id", "other", "unreadable"]); + fixture.scan(&[], 0, 0, &|| Ok(())).unwrap(); + assert_eq!(fixture.ids(), ["other"]); +} + +#[test] +fn concurrent_source_edit_aborts_native_swap_without_overwriting_other_writer() { + let fixture = Fixture::new(); + fixture.add("old", "/music/old.flac", "source"); + let writer = Connection::open(fixture.request["library_path"].as_str().unwrap()).unwrap(); + let changed = Cell::new(false); + let check = || { + if !changed.replace(true) { + writer + .execute( + "UPDATE library_sources SET path='/changed' WHERE id='source'", + [], + ) + .unwrap(); + } + Ok(()) + }; + assert!( + fixture + .scan(&[track("new", "/music/new.flac")], 1, 0, &check) + .is_err() + ); + assert_eq!(fixture.ids(), ["old"]); + assert_eq!( + fixture + .db + .query_row( + "SELECT path FROM library_sources WHERE id='source'", + [], + |row| row.get::<_, String>(0) + ) + .unwrap(), + "/changed" + ); +} + +#[test] +fn incremental_deletes_only_its_source_and_reconciles_existing_downloads() { + let fixture = Fixture::new(); + fixture.add("old", "/music/old.flac", "source"); + fixture.add("download", "/music/download.flac", "source"); + fixture.add("other", "/other/track.flac", "other"); + let history = Connection::open(fixture.request["history_path"].as_str().unwrap()).unwrap(); + history + .execute( + "INSERT INTO history_path_keys VALUES('h','/music/download')", + [], + ) + .unwrap(); + let mut request = fixture.request.clone(); + request["operation"] = "library_incremental_import".into(); + request["delta"] = json!({"scanned":[track("new","/music/new.flac")],"deletedPaths":["/music/old.flac","/other/track.flac"],"skippedCount":10,"totalFiles":11}); + let result = execute(&request, &|| Ok(())).unwrap(); + assert_eq!(fixture.ids(), ["new", "other"]); + assert_eq!(result["skipped"], 1); + assert_eq!(result["deleted_count"], 1); +} + +#[test] +fn snapshot_keeps_legacy_sentinel_and_backfills_only_current_missing_timestamps() { + let fixture = Fixture::new(); + fixture.add("legacy", "/music/legacy.flac", "source"); + fixture.add("saf", "content://provider/document/song.flac", "source"); + fixture.add("current", "/music/current.flac", "source"); + fixture + .db + .execute( + "UPDATE library SET file_mod_time=0,audio_metadata_scan_version=3 WHERE id='legacy'", + [], + ) + .unwrap(); + fixture + .db + .execute("UPDATE library SET file_mod_time=0 WHERE id='saf'", []) + .unwrap(); + let mut request = fixture.request.clone(); + request["operation"] = "library_snapshot".into(); + request["snapshot_path"] = fixture + .directory + .path() + .join("snapshot.tsv") + .to_str() + .unwrap() + .into(); + request["backfilled_mod_times"] = json!({"content://provider/document/song.flac":1700000000000_i64,"/music/legacy.flac":1900000000000_i64}); + let result = execute(&request, &|| Ok(())).unwrap(); + assert_eq!(result["count"], 3); + assert_eq!(result["backfilled_count"], 1); + let snapshot = fs::read_to_string(request["snapshot_path"].as_str().unwrap()).unwrap(); + assert!(snapshot.contains("-1\t/music/legacy.flac\n")); + assert!(snapshot.contains("1700000000000\tcontent://provider/document/song.flac\n")); + assert_eq!( + fixture + .db + .query_row( + "SELECT file_mod_time FROM library WHERE id='legacy'", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 0 + ); + assert!( + execute(&request, &|| Ok(())).is_err(), + "snapshot must not overwrite an existing output" + ); +} + +#[test] +fn mapper_preserves_null_artist_flags_and_replaygain_contract() { + let mut row = track("full", "/music/full.flac"); + row["explicit"] = json!(1); + row["albumArtist"] = json!(""); + row["metadataFromFilename"] = json!(true); + let mapped = encode(&row, "nas").unwrap(); + assert_eq!(mapped.len(), 39); + assert_eq!(mapped["explicit"], 1); + assert_eq!(mapped["has_replaygain"], 1); + assert_eq!(mapped["audio_metadata_scan_version"], 0); + assert_eq!(mapped["album_artist_norm"], ""); + assert_eq!(mapped["search_text"], "track full artist album artist"); +} diff --git a/test/fixtures/native_library_contract.json b/test/fixtures/native_library_contract.json new file mode 100644 index 00000000..b843eaf9 --- /dev/null +++ b/test/fixtures/native_library_contract.json @@ -0,0 +1,991 @@ +{ + "rows": [ + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-06T12:00:00.000Z" + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-06T12:00:00.000Z", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 0, + "audio_metadata_scan_version": 4, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "artist", + "match_key": "café i 💫|artist", + "album_key": "album|artist", + "search_text": "café i 💫 artist album artist", + "sort_genre": "", + "sort_release": "", + "sort_added": 1791288000000 + } + }, + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-06T12:00:00.000Z", + "albumArtist": "" + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": "", + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-06T12:00:00.000Z", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 0, + "audio_metadata_scan_version": 4, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "", + "match_key": "café i 💫|artist", + "album_key": "album|", + "search_text": "café i 💫 artist album artist", + "sort_genre": "", + "sort_release": "", + "sort_added": 1791288000000 + } + }, + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-06T12:00:00.000Z", + "albumArtist": " Other ", + "explicit": 1.0, + "hasLyrics": true, + "hasReplayGain": 1 + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": " Other ", + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-06T12:00:00.000Z", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 1, + "has_lyrics": 1, + "has_replaygain": 1, + "audio_metadata_scan_version": 4, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "other", + "match_key": "café i 💫|artist", + "album_key": "album|other", + "search_text": "café i 💫 artist album other", + "sort_genre": "", + "sort_release": "", + "sort_added": 1791288000000 + } + }, + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-06T12:00:00.000Z", + "metadataFromFilename": true, + "fileModTime": 0, + "genre": " Rock ", + "releaseDate": " 2026-10 " + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-06T12:00:00.000Z", + "file_mod_time": 0, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": " 2026-10 ", + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": " Rock ", + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 0, + "audio_metadata_scan_version": 0, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "artist", + "match_key": "café i 💫|artist", + "album_key": "album|artist", + "search_text": "café i 💫 artist album artist", + "sort_genre": "rock", + "sort_release": "2026-10", + "sort_added": 0 + } + }, + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-06T12:00:00.000Z", + "audioMetadataScanVersion": 2.9, + "fileModTime": 1770000000000.9, + "replaygain_track_gain": " -4.5 DB " + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-06T12:00:00.000Z", + "file_mod_time": 1770000000000.9, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 1, + "audio_metadata_scan_version": 2, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "artist", + "match_key": "café i 💫|artist", + "album_key": "album|artist", + "search_text": "café i 💫 artist album artist", + "sort_genre": "", + "sort_release": "", + "sort_added": 1770000000000 + } + }, + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "invalid", + "replaygain_album_gain": "NaN" + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "invalid", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 0, + "audio_metadata_scan_version": 4, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "artist", + "match_key": "café i 💫|artist", + "album_key": "album|artist", + "search_text": "café i 💫 artist album artist", + "sort_genre": "", + "sort_release": "", + "sort_added": 0 + } + }, + { + "input": { + "id": "row", + "trackName": "", + "artistName": "", + "albumName": "", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-06T12:00:00.000Z", + "replaygain_album_gain": 2.5 + }, + "expected": { + "id": "row", + "track_name": "", + "artist_name": "", + "album_name": "", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-06T12:00:00.000Z", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 1, + "audio_metadata_scan_version": 4, + "track_name_norm": "", + "artist_name_norm": "", + "album_name_norm": "", + "album_artist_norm": "", + "match_key": "|", + "album_key": "|", + "search_text": "", + "sort_genre": "", + "sort_release": "", + "sort_added": 1791288000000 + } + }, + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-06T19:00:00.123456+07:00", + "explicit": false, + "hasLyrics": 0 + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-06T19:00:00.123456+07:00", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 0, + "audio_metadata_scan_version": 4, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "artist", + "match_key": "café i 💫|artist", + "album_key": "album|artist", + "search_text": "café i 💫 artist album artist", + "sort_genre": "", + "sort_release": "", + "sort_added": 1791288000123 + } + }, + { + "input": { + "id": "row", + "trackName": " CAFÉ İ 💫 ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "20261006T120000.123456789z" + }, + "expected": { + "id": "row", + "track_name": " CAFÉ İ 💫 ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "20261006T120000.123456789z", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 0, + "audio_metadata_scan_version": 4, + "track_name_norm": "café i 💫", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "artist", + "match_key": "café i 💫|artist", + "album_key": "album|artist", + "search_text": "café i 💫 artist album artist", + "sort_genre": "", + "sort_release": "", + "sort_added": 1791288000123 + } + }, + { + "input": { + "id": "row", + "trackName": "ΑΒΣ İ", + "artistName": "Artist", + "albumName": "Album", + "filePath": "/music/a.flac", + "scannedAt": "2026-10-32T25:61:00Z" + }, + "expected": { + "id": "row", + "track_name": "ΑΒΣ İ", + "artist_name": "Artist", + "album_name": "Album", + "album_artist": null, + "file_path": "/music/a.flac", + "cover_path": null, + "scanned_at": "2026-10-32T25:61:00Z", + "file_mod_time": null, + "isrc": null, + "track_number": null, + "total_tracks": null, + "disc_number": null, + "total_discs": null, + "duration": null, + "release_date": null, + "bit_depth": null, + "sample_rate": null, + "bitrate": null, + "genre": null, + "composer": null, + "label": null, + "copyright": null, + "format": null, + "source_id": "source", + "explicit": 0, + "has_lyrics": 0, + "has_replaygain": 0, + "audio_metadata_scan_version": 4, + "track_name_norm": "αβσ i", + "artist_name_norm": "artist", + "album_name_norm": "album", + "album_artist_norm": "artist", + "match_key": "αβσ i|artist", + "album_key": "album|artist", + "search_text": "αβσ i artist album artist", + "sort_genre": "", + "sort_release": "", + "sort_added": 1793584860000 + } + } + ], + "paths": [ + { + "input": "", + "expected": [] + }, + { + "input": " EXISTS: /Music/Café 01.FLAC ", + "expected": [ + "/Music/Café 01", + "/Music/Café 01.FLAC", + "/music/café 01", + "/music/café 01.flac", + "file:///Music/Caf%C3%A9%2001", + "file:///Music/Caf%C3%A9%2001.FLAC", + "file:///music/caf%c3%a9%2001", + "file:///music/caf%c3%a9%2001.flac" + ] + }, + { + "input": "/a/../b/song.flac", + "expected": [ + "/a/../b/song", + "/a/../b/song.flac", + "file:///b/song", + "file:///b/song.flac" + ] + }, + { + "input": "/Music/100%25/song.opus", + "expected": [ + "/Music/100%/song", + "/Music/100%/song.opus", + "/Music/100%25/song", + "/Music/100%25/song.opus", + "/music/100%/song", + "/music/100%/song.opus", + "/music/100%25/song", + "/music/100%25/song.opus", + "file:///Music/100%25/song", + "file:///Music/100%25/song.opus", + "file:///Music/100%2525/song", + "file:///Music/100%2525/song.opus", + "file:///music/100%25/song", + "file:///music/100%25/song.opus", + "file:///music/100%2525/song", + "file:///music/100%2525/song.opus" + ] + }, + { + "input": "/Music/100%zz.flac", + "expected": [ + "/Music/100%zz", + "/Music/100%zz.flac", + "/music/100%zz", + "/music/100%zz.flac", + "file:///Music/100%25zz", + "file:///Music/100%25zz.flac", + "file:///music/100%25zz", + "file:///music/100%25zz.flac" + ] + }, + { + "input": "/Music/💫 track.wav", + "expected": [ + "/Music/💫 track", + "/Music/💫 track.wav", + "/music/💫 track", + "/music/💫 track.wav", + "file:///Music/%F0%9F%92%AB%20track", + "file:///Music/%F0%9F%92%AB%20track.wav", + "file:///music/%f0%9f%92%ab%20track", + "file:///music/%f0%9f%92%ab%20track.wav" + ] + }, + { + "input": "/Music/a#b.flac", + "expected": [ + "/Music/a#b", + "/Music/a#b.flac", + "/music/a#b", + "/music/a#b.flac", + "file:///Music/a%23b", + "file:///Music/a%23b.flac", + "file:///music/a%23b", + "file:///music/a%23b.flac" + ] + }, + { + "input": "/Music/a?b.mp3", + "expected": [ + "/Music/a?b", + "/Music/a?b.mp3", + "/music/a?b", + "/music/a?b.mp3", + "file:///Music/a%3Fb", + "file:///Music/a%3Fb.mp3", + "file:///music/a%3fb", + "file:///music/a%3fb.mp3" + ] + }, + { + "input": "/Music/a\\b.flac", + "expected": [ + "/Music/a/b", + "/Music/a/b.flac", + "/Music/a\\b", + "/Music/a\\b.flac", + "/music/a/b", + "/music/a/b.flac", + "/music/a\\b", + "/music/a\\b.flac", + "file:///Music/a%5Cb", + "file:///Music/a%5Cb.flac", + "file:///Music/a/b", + "file:///Music/a/b.flac", + "file:///music/a%5cb", + "file:///music/a%5cb.flac", + "file:///music/a/b", + "file:///music/a/b.flac" + ] + }, + { + "input": "C:\\Music\\Hello.flac", + "expected": [ + "C:/Music/Hello", + "C:/Music/Hello.flac", + "C:\\Music\\Hello", + "C:\\Music\\Hello.flac", + "c:/Music/Hello", + "c:/Music/Hello.flac", + "c:/music/hello", + "c:/music/hello.flac", + "c:\\music\\hello", + "c:\\music\\hello.flac" + ] + }, + { + "input": "file:///music/a%20b.flac", + "expected": [ + "/music/a b", + "/music/a b.flac", + "file:///music/a b", + "file:///music/a b.flac", + "file:///music/a%20b", + "file:///music/a%20b.flac" + ] + }, + { + "input": "file://localhost/music/a.flac", + "expected": [ + "file://localhost/music/a", + "file://localhost/music/a.flac" + ] + }, + { + "input": "file://localhost", + "expected": [ + "file://localhost", + "file://localhost/" + ] + }, + { + "input": "file:///music/a.flac?x=1#frag", + "expected": [ + "file:///music/a.flac?x=1#frag" + ] + }, + { + "input": "file:///music/a.flac#frag", + "expected": [ + "file:///music/a.flac#frag" + ] + }, + { + "input": "file:///a/../b/song.flac", + "expected": [ + "/b/song", + "/b/song.flac", + "file:///a/../b/song", + "file:///a/../b/song.flac", + "file:///b/song", + "file:///b/song.flac" + ] + }, + { + "input": "file:///music/%7e.flac", + "expected": [ + "/music/~", + "/music/~.flac", + "file:///music/%7e", + "file:///music/%7e.flac", + "file:///music/~", + "file:///music/~.flac" + ] + }, + { + "input": "content://nas.provider/document/Ab%252FC.flac", + "expected": [ + "content://nas.provider/document/Ab%252FC", + "content://nas.provider/document/Ab%252FC.flac", + "content://nas.provider/document/Ab%2FC", + "content://nas.provider/document/Ab%2FC.flac", + "content://nas.provider/document/Ab/C", + "content://nas.provider/document/Ab/C.flac", + "content://nas.provider/document/ab%252fc", + "content://nas.provider/document/ab%252fc.flac", + "content://nas.provider/document/ab%2fc", + "content://nas.provider/document/ab%2fc.flac", + "content://nas.provider/document/ab/c", + "content://nas.provider/document/ab/c.flac", + "saf-document:bmFzLnByb3ZpZGVy:Ab%2FC", + "saf-document:bmFzLnByb3ZpZGVy:Ab%2FC.flac" + ] + }, + { + "input": "content://nas.provider/tree/root/document/Ab%252FC.flac", + "expected": [ + "content://nas.provider/tree/root/document/Ab%252FC", + "content://nas.provider/tree/root/document/Ab%252FC.flac", + "content://nas.provider/tree/root/document/Ab%2FC", + "content://nas.provider/tree/root/document/Ab%2FC.flac", + "content://nas.provider/tree/root/document/Ab/C", + "content://nas.provider/tree/root/document/Ab/C.flac", + "content://nas.provider/tree/root/document/ab%252fc", + "content://nas.provider/tree/root/document/ab%252fc.flac", + "content://nas.provider/tree/root/document/ab%2fc", + "content://nas.provider/tree/root/document/ab%2fc.flac", + "content://nas.provider/tree/root/document/ab/c", + "content://nas.provider/tree/root/document/ab/c.flac", + "saf-document:bmFzLnByb3ZpZGVy:Ab%2FC", + "saf-document:bmFzLnByb3ZpZGVy:Ab%2FC.flac" + ] + }, + { + "input": "content://NAS.PROVIDER/document/Ab%2FC.flac?x=Y#frag", + "expected": [ + "content://NAS.PROVIDER/document/Ab%2FC.flac?x=Y#frag", + "content://NAS.PROVIDER/document/Ab/C.flac?x=Y#frag", + "content://nas.provider/document/Ab%2FC.flac?x=Y#frag", + "content://nas.provider/document/Ab/C.flac?x=Y#frag", + "content://nas.provider/document/ab%2fc.flac?x=y#frag", + "content://nas.provider/document/ab/c.flac?x=y#frag", + "saf-document:bmFzLnByb3ZpZGVy:Ab/C", + "saf-document:bmFzLnByb3ZpZGVy:Ab/C.flac" + ] + }, + { + "input": "content://nas.provider/document/AB%20C.flac", + "expected": [ + "content://nas.provider/document/AB C", + "content://nas.provider/document/AB C.flac", + "content://nas.provider/document/AB%20C", + "content://nas.provider/document/AB%20C.flac", + "content://nas.provider/document/ab c", + "content://nas.provider/document/ab c.flac", + "content://nas.provider/document/ab%20c", + "content://nas.provider/document/ab%20c.flac", + "saf-document:bmFzLnByb3ZpZGVy:AB C", + "saf-document:bmFzLnByb3ZpZGVy:AB C.flac" + ] + }, + { + "input": "content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9.flac", + "expected": [ + "/mnt/sdcard/Music/Café", + "/mnt/sdcard/Music/Café.flac", + "/mnt/sdcard/music/café", + "/mnt/sdcard/music/café.flac", + "/sdcard/Music/Café", + "/sdcard/Music/Café.flac", + "/sdcard/music/café", + "/sdcard/music/café.flac", + "/storage/emulated/0/Music/Café", + "/storage/emulated/0/Music/Café.flac", + "/storage/emulated/0/music/café", + "/storage/emulated/0/music/café.flac", + "/storage/emulated/legacy/Music/Café", + "/storage/emulated/legacy/Music/Café.flac", + "/storage/emulated/legacy/music/café", + "/storage/emulated/legacy/music/café.flac", + "/storage/self/primary/Music/Café", + "/storage/self/primary/Music/Café.flac", + "/storage/self/primary/music/café", + "/storage/self/primary/music/café.flac", + "content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9", + "content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9.flac", + "content://com.android.externalstorage.documents/tree/primary%3amusic/document/primary%3amusic%2fcaf%c3%a9", + "content://com.android.externalstorage.documents/tree/primary%3amusic/document/primary%3amusic%2fcaf%c3%a9.flac", + "content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Caf%C3%A9", + "content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Caf%C3%A9.flac", + "content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Café", + "content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Café.flac", + "content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/caf%c3%a9", + "content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/caf%c3%a9.flac", + "content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/café", + "content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/café.flac", + "file:///mnt/sdcard/Music/Caf%C3%A9", + "file:///mnt/sdcard/Music/Caf%C3%A9.flac", + "file:///mnt/sdcard/music/caf%c3%a9", + "file:///mnt/sdcard/music/caf%c3%a9.flac", + "file:///sdcard/Music/Caf%C3%A9", + "file:///sdcard/Music/Caf%C3%A9.flac", + "file:///sdcard/music/caf%c3%a9", + "file:///sdcard/music/caf%c3%a9.flac", + "file:///storage/emulated/0/Music/Caf%C3%A9", + "file:///storage/emulated/0/Music/Caf%C3%A9.flac", + "file:///storage/emulated/0/music/caf%c3%a9", + "file:///storage/emulated/0/music/caf%c3%a9.flac", + "file:///storage/emulated/legacy/Music/Caf%C3%A9", + "file:///storage/emulated/legacy/Music/Caf%C3%A9.flac", + "file:///storage/emulated/legacy/music/caf%c3%a9", + "file:///storage/emulated/legacy/music/caf%c3%a9.flac", + "file:///storage/self/primary/Music/Caf%C3%A9", + "file:///storage/self/primary/Music/Caf%C3%A9.flac", + "file:///storage/self/primary/music/caf%c3%a9", + "file:///storage/self/primary/music/caf%c3%a9.flac", + "saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/Café", + "saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/Café.flac" + ] + }, + { + "input": "content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary%3AMusic%2Fsong.opus", + "expected": [ + "/mnt/sdcard/Music/song", + "/mnt/sdcard/Music/song.opus", + "/mnt/sdcard/music/song", + "/mnt/sdcard/music/song.opus", + "/sdcard/Music/song", + "/sdcard/Music/song.opus", + "/sdcard/music/song", + "/sdcard/music/song.opus", + "/storage/emulated/0/Music/song", + "/storage/emulated/0/Music/song.opus", + "/storage/emulated/0/music/song", + "/storage/emulated/0/music/song.opus", + "/storage/emulated/legacy/Music/song", + "/storage/emulated/legacy/Music/song.opus", + "/storage/emulated/legacy/music/song", + "/storage/emulated/legacy/music/song.opus", + "/storage/self/primary/Music/song", + "/storage/self/primary/Music/song.opus", + "/storage/self/primary/music/song", + "/storage/self/primary/music/song.opus", + "content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary%3AMusic%2Fsong", + "content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary%3AMusic%2Fsong.opus", + "content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary:Music/song", + "content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary:Music/song.opus", + "content://com.android.externalstorage.documents/document/primary%3AMusic%2Fsong", + "content://com.android.externalstorage.documents/document/primary%3AMusic%2Fsong.opus", + "content://com.android.externalstorage.documents/document/primary%3amusic%2fsong", + "content://com.android.externalstorage.documents/document/primary%3amusic%2fsong.opus", + "content://com.android.externalstorage.documents/document/primary:Music/song", + "content://com.android.externalstorage.documents/document/primary:Music/song.opus", + "content://com.android.externalstorage.documents/document/primary:music/song", + "content://com.android.externalstorage.documents/document/primary:music/song.opus", + "file:///mnt/sdcard/Music/song", + "file:///mnt/sdcard/Music/song.opus", + "file:///mnt/sdcard/music/song", + "file:///mnt/sdcard/music/song.opus", + "file:///sdcard/Music/song", + "file:///sdcard/Music/song.opus", + "file:///sdcard/music/song", + "file:///sdcard/music/song.opus", + "file:///storage/emulated/0/Music/song", + "file:///storage/emulated/0/Music/song.opus", + "file:///storage/emulated/0/music/song", + "file:///storage/emulated/0/music/song.opus", + "file:///storage/emulated/legacy/Music/song", + "file:///storage/emulated/legacy/Music/song.opus", + "file:///storage/emulated/legacy/music/song", + "file:///storage/emulated/legacy/music/song.opus", + "file:///storage/self/primary/Music/song", + "file:///storage/self/primary/Music/song.opus", + "file:///storage/self/primary/music/song", + "file:///storage/self/primary/music/song.opus", + "saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/song", + "saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/song.opus" + ] + }, + { + "input": "network://Connection/Music/Track%2520A.flac", + "expected": [ + "network://Connection/Music/Track A", + "network://Connection/Music/Track A.flac", + "network://Connection/Music/Track%20A", + "network://Connection/Music/Track%20A.flac", + "network://Connection/Music/Track%2520A", + "network://Connection/Music/Track%2520A.flac", + "network://connection/Music/Track%20A", + "network://connection/Music/Track%20A.flac", + "network://connection/Music/Track%2520A", + "network://connection/Music/Track%2520A.flac", + "network://connection/music/track a", + "network://connection/music/track a.flac", + "network://connection/music/track%20a", + "network://connection/music/track%20a.flac", + "network://connection/music/track%2520a", + "network://connection/music/track%2520a.flac" + ] + }, + { + "input": "https://EXAMPLE.com/Music/a%7eb.flac?q=x#Y", + "expected": [ + "https://EXAMPLE.com/Music/a%7eb.flac?q=x#Y", + "https://EXAMPLE.com/Music/a~b.flac?q=x#Y", + "https://example.com/Music/a~b.flac?q=x#Y", + "https://example.com/music/a%7eb.flac?q=x#y", + "https://example.com/music/a~b.flac?q=x#y" + ] + }, + { + "input": "content://provider/document/💫.flac", + "expected": [ + "content://provider/document/%F0%9F%92%AB", + "content://provider/document/%F0%9F%92%AB.flac", + "content://provider/document/%f0%9f%92%ab", + "content://provider/document/%f0%9f%92%ab.flac", + "content://provider/document/💫", + "content://provider/document/💫.flac", + "saf-document:cHJvdmlkZXI=:💫", + "saf-document:cHJvdmlkZXI=:💫.flac" + ] + }, + { + "input": "content://provider/document/a%zz.flac", + "expected": [ + "content://provider/document/a%25zz", + "content://provider/document/a%25zz.flac", + "content://provider/document/a%zz", + "content://provider/document/a%zz.flac", + "saf-document:cHJvdmlkZXI=:a%zz", + "saf-document:cHJvdmlkZXI=:a%zz.flac" + ] + }, + { + "input": "/sdcard/Music/song.flac", + "expected": [ + "/sdcard/Music/song", + "/sdcard/Music/song.flac", + "/sdcard/music/song", + "/sdcard/music/song.flac", + "file:///sdcard/Music/song", + "file:///sdcard/Music/song.flac", + "file:///sdcard/music/song", + "file:///sdcard/music/song.flac" + ] + }, + { + "input": "/storage/emulated/0/Music/song.opus", + "expected": [ + "/storage/emulated/0/Music/song", + "/storage/emulated/0/Music/song.opus", + "/storage/emulated/0/music/song", + "/storage/emulated/0/music/song.opus", + "file:///storage/emulated/0/Music/song", + "file:///storage/emulated/0/Music/song.opus", + "file:///storage/emulated/0/music/song", + "file:///storage/emulated/0/music/song.opus" + ] + } + ] +} diff --git a/test/library_scan_scheduling_test.dart b/test/library_scan_scheduling_test.dart index f2b827c3..c5bdf30b 100644 --- a/test/library_scan_scheduling_test.dart +++ b/test/library_scan_scheduling_test.dart @@ -10,6 +10,7 @@ import 'package:spotiflac_android/models/settings.dart'; import 'package:spotiflac_android/providers/local_library_provider.dart'; import 'package:spotiflac_android/providers/settings_provider.dart'; import 'package:spotiflac_android/services/library_database.dart'; +import 'package:spotiflac_android/services/platform_bridge.dart'; class _Settings extends SettingsNotifier { @override @@ -25,6 +26,8 @@ class _Database implements LibraryDatabase { List sources; Future> Function()? sourceRead; bool failLookup = false; + int? countOverride; + Future<({int inserted, int skipped})> Function()? fullReplacement; @override Future> getSources() async => @@ -38,8 +41,17 @@ class _Database implements LibraryDatabase { @override Future getCount() async => + countOverride ?? sources.where((source) => source.enabled && source.available).length; + @override + Future<({int inserted, int skipped})> replaceSourceScanFile( + String sourceId, + LibraryScanNDJSONFile scan, { + required String requestId, + required bool Function() isCancelled, + }) => fullReplacement!(); + @override Future getLookupIndex() async => const LocalLibraryLookupIndex(); @@ -328,4 +340,54 @@ void main() { await retry; expect(container.read(localLibraryProvider).isScanning, isFalse); }); + + test( + 'cancel after incremental native commit publishes the changed index', + () async { + final version = container.read(localLibraryProvider).loadedIndexVersion; + final active = library.startSourceScan('active'); + await _waitFor(() => scans.length == 1); + await library.cancelScan(); + database.countOverride = 7; + scans.single.$2.complete(jsonEncode({'committed': true, 'skipped': 2})); + await active; + final state = container.read(localLibraryProvider); + expect(state.totalCount, 7); + expect(state.loadedIndexVersion, greaterThan(version)); + expect(state.excludedDownloadedCount, 2); + expect(state.scanWasCancelled, isTrue); + expect(state.isScanning, isFalse); + }, + ); + + test('cancel after full replacement publishes the committed index', () async { + final entered = Completer(); + final release = Completer(); + database.fullReplacement = () async { + entered.complete(); + await release.future; + database.countOverride = 7; + return (inserted: 7, skipped: 2); + }; + messenger.setMockMethodCallHandler(backend, (call) async { + if (call.method == 'scanLibraryFolderToNDJSONFile') { + final path = (call.arguments as Map)['output_path'] as String; + await File(path).writeAsString(''); + return {'path': path, 'count': 0, 'error_count': 0}; + } + return null; + }); + final version = container.read(localLibraryProvider).loadedIndexVersion; + final active = library.startSourceScan('active', forceFullScan: true); + await entered.future; + await library.cancelScan(); + release.complete(); + await active; + final state = container.read(localLibraryProvider); + expect(state.totalCount, 7); + expect(state.loadedIndexVersion, greaterThan(version)); + expect(state.excludedDownloadedCount, 2); + expect(state.scanWasCancelled, isTrue); + expect(state.isScanning, isFalse); + }); } diff --git a/test/native_library_contract_test.dart b/test/native_library_contract_test.dart new file mode 100644 index 00000000..0a54d05c --- /dev/null +++ b/test/native_library_contract_test.dart @@ -0,0 +1,39 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:spotiflac_android/services/library_row_mapper.dart'; +import 'package:spotiflac_android/utils/path_match_keys.dart'; + +void main() { + test( + 'native library fixtures preserve production Dart row and path codecs', + () { + final fixture = + jsonDecode( + File( + 'test/fixtures/native_library_contract.json', + ).readAsStringSync(), + ) + as Map; + for (final entry in fixture['rows'] as List) { + final value = entry as Map; + expect( + LibraryRowMapper.encode( + Map.from(value['input'] as Map), + sourceId: 'source', + ), + value['expected'], + ); + } + for (final entry in fixture['paths'] as List) { + final value = entry as Map; + expect( + buildPathMatchKeys(value['input'] as String).toList()..sort(), + value['expected'], + reason: value['input'] as String, + ); + } + }, + ); +}