perf(library): index scans and snapshots with native SQLite workers

Move full and incremental ingestion, source snapshots, path keys, and download exclusion to native transactions. Preserve schema 17, FTS, WAL coordination, cancellation, and Android SAF timestamp backfill.
This commit is contained in:
zarzet committed 2026-10-06 23:29:33 +07:00
1 parent 56f91f4e6c
commit d1de833184
15 files changed
+2974 -59

No files matched your search

@@ -571,11 +571,11 @@ private const val SAF_SCAN_PAUSE_POLL_MS = 100L
* Holds a SAF scan at its checkpoint while paused, keeping its position in
* memory. Returns true when the scan should stop because it was cancelled.
*/
internal fun MainActivity.safScanStopRequested(): Boolean {
while (safScanPaused && !safScanCancel) {
internal fun MainActivity.safScanStopRequested(cancelled: () -> Boolean = { false }): Boolean {
while (safScanPaused && !safScanCancel && !cancelled()) {
Thread.sleep(SAF_SCAN_PAUSE_POLL_MS)
}
return safScanCancel
return safScanCancel || cancelled()
}
/**
@@ -1287,6 +1287,7 @@ internal fun MainActivity.scanSafTreeIncremental(treeUriStr: String, existingFil
internal fun MainActivity.scanSafTreeIncremental(
treeUriStr: String,
existingFiles: Map<String, Long>,
cancelled: () -> Boolean = { false },
): Any {
val (_, root, rootChildren) = resolveReadableSafTreeOrThrow(treeUriStr)
@@ -1329,7 +1330,7 @@ internal fun MainActivity.scanSafTreeIncremental(
val lister = SafTreeLister(this)
while (queue.isNotEmpty()) {
if (safScanStopRequested()) {
if (safScanStopRequested(cancelled)) {
updateSafScanProgress { it.isComplete = true }
val result = JSONObject()
result.put("files", JSONArray())
@@ -1361,7 +1362,7 @@ internal fun MainActivity.scanSafTreeIncremental(
rememberCueDirectoryListing(dir, children, safChildLookupCache)
for (child in children) {
if (safScanStopRequested()) {
if (safScanStopRequested(cancelled)) {
updateSafScanProgress { it.isComplete = true }
val result = JSONObject()
result.put("files", JSONArray())
@@ -1462,7 +1463,7 @@ internal fun MainActivity.scanSafTreeIncremental(
val cueReferencedAudioUris = mutableSetOf<String>()
for ((cueDoc, parentDir, cueName, cueLastModified) in cueFilesToScan) {
if (safScanStopRequested()) {
if (safScanStopRequested(cancelled)) {
updateSafScanProgress { it.isComplete = true }
spill.abandon()
val result = JSONObject()
@@ -1602,7 +1603,7 @@ internal fun MainActivity.scanSafTreeIncremental(
val pendingAudio = mutableListOf<ChangedAudio>()
for (audio in audioFiles) {
if (safScanStopRequested()) return cancelledIncrementalResult()
if (safScanStopRequested(cancelled)) return cancelledIncrementalResult()
if (cueReferencedAudioUris.contains(audio.doc.uri.toString())) {
scanned++
reportProcessed()
@@ -1613,7 +1614,7 @@ internal fun MainActivity.scanSafTreeIncremental(
val completed = runSafReadsInOrder(
pendingAudio,
cancelled = { safScanStopRequested() },
cancelled = { safScanStopRequested(cancelled) },
task = { audio ->
val ext = audio.name.substringAfterLast('.', "").lowercase(Locale.ROOT)
val fallbackExt = if (ext.isNotBlank()) ".${ext}" else null
@@ -3,6 +3,8 @@ package com.zarz.spotiflac
import com.spotiflac.backend.CancellationDomain
import com.spotiflac.backend.CancellationRegistry
import com.spotiflac.backend.RequestLease
import java.io.File
import org.json.JSONArray
import org.json.JSONObject
// Independent of extension initialization: palette/metadata jobs can start
@@ -45,5 +47,63 @@ internal fun MainActivity.runNativeDataJobPlatform(arguments: Any?, lease: Reque
val args = arguments as? Map<*, *> ?: error("Missing data job arguments")
val request = JSONObject(args["request_json"] as? String ?: error("Missing data job request"))
val bytes = args["bytes"] as? ByteArray ?: byteArrayOf()
return bridgeJsonResult(RustCoreBackend.runDataJob(request, bytes, lease))
var spilledDelta: File? = null
return run {
fun checkActive() { check(!lease.isCancelled()) { "Native data job cancelled" } }
try {
checkActive()
if (request.optString("operation") == "library_snapshot") {
// SAF timestamps require ContentResolver, but DB paging and all
// row mapping stay native. One lease spans every batch.
val backfilled = JSONObject()
var cursor = ""
do {
checkActive()
val pageRequest = JSONObject(request.toString())
.put("operation", "library_snapshot_candidates")
.put("cursor", cursor).put("limit", 500)
val page = JSONObject(RustCoreBackend.runDataJob(pageRequest, byteArrayOf(), lease))
val paths = page.getJSONArray("paths")
val uris = JSONArray()
for (index in 0 until paths.length()) uris.put(paths.getJSONObject(index).getString("file_path"))
val times = JSONObject(getSafFileModTimes(uris.toString()))
val keys = times.keys()
while (keys.hasNext()) {
val path = keys.next()
backfilled.put(path, times.getLong(path))
}
cursor = page.getString("cursor")
} while (page.getBoolean("has_more"))
request.put("backfilled_mod_times", backfilled)
}
if (request.optString("operation") == "library_scan_incremental" && !request.has("tree_uri")) {
safScanActive = false
}
if (request.optString("operation") == "library_scan_incremental" && request.has("tree_uri")) {
checkActive()
val delta = scanSafTreeIncremental(
request.getString("tree_uri"),
loadExistingFilesFromSnapshot(request.optString("snapshot_path")),
cancelled = { lease.isCancelled() },
)
val deltaPath = if (delta is String) null else {
(delta as? Map<*, *>)?.get("__json_file") as? String
?: error("Invalid SAF scan result")
}
// Own a spilled response before checking cancellation, so the
// finally block also removes a just-completed cancelled scan.
spilledDelta = deltaPath?.let(::File)
checkActive()
request.put("operation", "library_incremental_import")
if (delta is String) {
request.put("delta_json", delta)
} else {
request.put("delta_path", deltaPath)
}
}
bridgeJsonResult(RustCoreBackend.runDataJob(request, bytes, lease))
} finally {
spilledDelta?.delete()
}
}
}
@@ -137,7 +137,21 @@ internal object RustCoreBackend : CoreBackend {
owner() to requestRegistry().acquire(id)
internal fun runDataJob(request: JSONObject, bytes: ByteArray, lease: RequestLease): String {
return com.spotiflac.backend.runNativeDataJob(request.toString(), bytes, lease)
if (request.optString("operation") != "library_scan_incremental") {
return com.spotiflac.backend.runNativeDataJob(request.toString(), bytes, lease)
}
val folder = request.getString("folder_path")
return withLibraryDirectories(listOf(folder)) { current ->
val snapshot = request.optString("snapshot_path")
val staged = if (snapshot.isEmpty()) null else workspace.createTemporaryFile("library_snapshot_", ".tsv")
try {
if (staged != null) {
File(snapshot).copyTo(staged, overwrite = true)
request.put("snapshot_path", staged.canonicalPath)
}
current.runNativeDataJob(request.toString(), bytes, lease)
} finally { staged?.delete() }
}
}
@Synchronized
+17 -1
View File
@@ -302,10 +302,26 @@ final class RustCoreBackend: CoreBackend {
throw failure("Missing native data job lease")
}
defer { finishDataJob(id, lease) }
guard let raw = args["request_json"] as? String else {
guard let raw = args["request_json"] as? String,
var request = try JSONSerialization.jsonObject(with: Data(raw.utf8)) as? [String: Any] else {
throw failure("Missing native data job request")
}
let bytes = (args["bytes"] as? FlutterStandardTypedData)?.data ?? Data()
if request["operation"] as? String == "library_scan_incremental" {
guard let folder = request["folder_path"] as? String else { throw failure("Missing library folder") }
return try withLibraryDirectories([folder]) { current in
let staged = root.appendingPathComponent("files", isDirectory: true)
.appendingPathComponent("library_snapshot_\(UUID().uuidString).tsv")
defer { try? FileManager.default.removeItem(at: staged) }
if let snapshot = request["snapshot_path"] as? String, !snapshot.isEmpty {
try FileManager.default.copyItem(atPath: snapshot, toPath: staged.path)
request["snapshot_path"] = staged.path
}
request["folder_path"] = URL(fileURLWithPath: folder).resolvingSymlinksInPath().standardizedFileURL.path
let encoded = String(decoding: try JSONSerialization.data(withJSONObject: request), as: UTF8.self)
return try current.runNativeDataJob(requestJson: encoded, bytes: bytes, lease: lease)
}
}
return try SpotiFLACBackend.runNativeDataJob(requestJson: raw, bytes: bytes, lease: lease)
}
+94 -47
View File
@@ -170,6 +170,7 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
bool _scanPauseRequested = false;
bool _scanInProgress = false;
Future<void>? _scanCompletion;
String? _nativeLibraryRequestId;
final Set<String> _pendingReconnectScans = {};
bool _drainingReconnectScans = false;
StreamSubscription<void>? _storageEventsSubscription;
@@ -194,6 +195,16 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
_scanPauseRequested = false;
_pendingReconnectScans.clear();
if (_scanInProgress) {
final requestId = _nativeLibraryRequestId;
if (requestId != null) {
unawaited(
PlatformBridge.cancelNativeDataJob(requestId).catchError((
Object error,
) {
_log.w('Failed to cancel disposed library ingestion: $error');
}),
);
}
unawaited(
PlatformBridge.cancelLibraryScan().catchError((Object error) {
_log.w('Failed to cancel disposed library scan: $error');
@@ -783,27 +794,11 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
scannedFiles: scanFile.expectedCount,
scanErrorCount: scanFile.errorCount,
);
Stream<Map<String, dynamic>> validatedRows() async* {
var decodedRows = 0;
await for (final json in scanFile.rows()) {
if (_scanCancelRequested) {
throw StateError('Library scan cancelled during ingestion');
}
decodedRows++;
yield json;
}
if (decodedRows != scanFile.expectedCount) {
throw FormatException(
'Library scan row count mismatch: decoded $decodedRows, '
'expected ${scanFile.expectedCount}',
);
}
}
final result = await _db.replaceSourceStream(
final result = await _db.replaceSourceScanFile(
sourceId,
validatedRows(),
preserveMissing: scanFile.errorCount > 0,
scanFile,
requestId: _nativeLibraryRequestId ?? 'desktop-library-scan',
isCancelled: () => _scanCancelRequested || !ref.mounted,
);
_log.i(
'Stream-ingested ${result.inserted}/${scanFile.expectedCount} scan rows '
@@ -836,6 +831,9 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
if (sourceId != null) _pendingReconnectScans.remove(sourceId);
_scanCancelRequested = false;
_scanPauseRequested = false;
_nativeLibraryRequestId = PlatformBridge.supportsCoreBackend
? 'library-scan-${DateTime.now().microsecondsSinceEpoch}'
: null;
try {
await _runScan(
folderPath,
@@ -854,6 +852,7 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
}
_scanInProgress = false;
_scanCompletion = null;
_nativeLibraryRequestId = null;
completion.complete();
}
}
@@ -991,6 +990,12 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
);
if (!ref.mounted) return;
if (scanResult == null || _scanCancelRequested) {
if (scanResult != null) {
await _refreshSummaryFromStorage(
excludedDownloadedCount: scanResult.skipped,
);
}
if (!ref.mounted) return;
state = state.copyWith(
isScanning: false,
scanIsFinalizing: false,
@@ -1060,26 +1065,34 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
errorCount: state.scanErrorCount,
);
} else {
final existingFiles = await _db.getFileModTimes(
sourceId: activeSourceId,
);
_log.i(
'Incremental scan: ${existingFiles.length} existing files in database',
);
final backfilledModTimes = await _backfillLegacyFileModTimes(
isSaf: isSaf,
existingFiles: existingFiles,
);
if (backfilledModTimes.isNotEmpty) {
await _db.updateFileModTimes(backfilledModTimes);
existingFiles.addAll(backfilledModTimes);
_log.i('Backfilled ${backfilledModTimes.length} legacy mod times');
final nativeIngestion = PlatformBridge.supportsCoreBackend;
final existingFiles = nativeIngestion
? const <String, int>{}
: await _db.getFileModTimes(sourceId: activeSourceId);
if (!nativeIngestion) {
_log.i(
'Incremental scan: ${existingFiles.length} existing files in database',
);
final backfilledModTimes = await _backfillLegacyFileModTimes(
isSaf: isSaf,
existingFiles: existingFiles,
);
if (backfilledModTimes.isNotEmpty) {
await _db.updateFileModTimes(backfilledModTimes);
existingFiles.addAll(backfilledModTimes);
_log.i('Backfilled ${backfilledModTimes.length} legacy mod times');
}
}
final useSnapshotBridge =
Platform.isAndroid && existingFiles.isNotEmpty;
final snapshotPath = useSnapshotBridge
final snapshotPath = nativeIngestion
? await _db.writeNativeFileModTimesSnapshot(
activeSourceId,
requestId: _nativeLibraryRequestId!,
isCancelled: () => _scanCancelRequested || !ref.mounted,
)
: useSnapshotBridge
? await _db.writeFileModTimesSnapshot(existingFiles)
: null;
@@ -1090,7 +1103,16 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
!ref.mounted) {
return;
}
if (isSaf) {
if (nativeIngestion) {
result = await _db.scanIncrementalNative(
activeSourceId,
effectiveFolderPath,
snapshotPath!,
isSaf: isSaf,
requestId: _nativeLibraryRequestId!,
isCancelled: () => _scanCancelRequested || !ref.mounted,
);
} else if (isSaf) {
result = useSnapshotBridge && snapshotPath != null
? await PlatformBridge.scanSafTreeIncrementalFromSnapshot(
effectiveFolderPath,
@@ -1121,6 +1143,12 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
if (!ref.mounted) return;
if (_scanCancelRequested || result['cancelled'] == true) {
if (result['committed'] == true) {
await _refreshSummaryFromStorage(
excludedDownloadedCount: (result['skipped'] as num?)?.toInt(),
);
}
if (!ref.mounted) return;
state = state.copyWith(
isScanning: false,
scanIsFinalizing: false,
@@ -1151,15 +1179,24 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
[];
final skippedCount = result['skippedCount'] as int? ?? 0;
final totalFiles = result['totalFiles'] as int? ?? 0;
final scannedCount = nativeIngestion
? (result['scanned_count'] as num).toInt()
: scannedList.length;
final deletedCount = nativeIngestion
? (result['deleted_count'] as num).toInt()
: deletedPaths.length;
if (nativeIngestion && result['committed'] != true) {
throw StateError('Native Library scan did not commit its index');
}
_log.i(
'Incremental result: ${scannedList.length} scanned, '
'$skippedCount skipped, ${deletedPaths.length} deleted, $totalFiles total',
'Incremental result: $scannedCount scanned, '
'$skippedCount skipped, $deletedCount deleted, $totalFiles total',
);
final removedDownloaded = await _db.deleteDownloadedRowsForSource(
activeSourceId,
);
final removedDownloaded = nativeIngestion
? 0
: await _db.deleteDownloadedRowsForSource(activeSourceId);
if (removedDownloaded > 0) {
_log.i(
'Removed $removedDownloaded downloaded tracks already present in '
@@ -1168,8 +1205,10 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
}
final updatedItems = <LocalLibraryItem>[];
var skippedDownloads = removedDownloaded;
if (scannedList.isNotEmpty) {
var skippedDownloads = nativeIngestion
? (result['skipped'] as num).toInt()
: removedDownloaded;
if (!nativeIngestion && scannedList.isNotEmpty) {
for (final json in scannedList) {
final map = json as Map<String, dynamic>;
final item = LocalLibraryItem.fromJson(map);
@@ -1190,7 +1229,7 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
}
}
if (deletedPaths.isNotEmpty) {
if (!nativeIngestion && deletedPaths.isNotEmpty) {
final deleteCount = await _db.deleteByPaths(deletedPaths);
_log.i('Deleted $deleteCount items from database');
}
@@ -1229,8 +1268,8 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
final summary =
'Incremental scan complete: ${state.totalCount} total tracks '
'(${scannedList.length} new/updated, $skippedCount unchanged, '
'${deletedPaths.length} removed, $skippedDownloads already in downloads, '
'($scannedCount new/updated, $skippedCount unchanged, '
'$deletedCount removed, $skippedDownloads already in downloads, '
'${state.scanErrorCount} errors)';
if (state.scanErrorCount > 0) {
_log.e(summary);
@@ -1408,6 +1447,14 @@ class LocalLibraryNotifier extends Notifier<LocalLibraryState> {
_log.i('Cancelling library scan');
_scanCancelRequested = true;
final requestId = _nativeLibraryRequestId;
if (requestId != null) {
try {
await PlatformBridge.cancelNativeDataJob(requestId);
} catch (error) {
_log.w('Failed to cancel library data job: $error');
}
}
_pendingReconnectScans.clear();
_availabilityRequestVersion++;
_scanReconnectedOnRefresh = false;
+113
View File
@@ -13,6 +13,7 @@ import 'package:spotiflac_android/services/library_search.dart';
import 'package:spotiflac_android/services/library_database_models.dart';
import 'package:spotiflac_android/services/library_row_mapper.dart';
import 'package:spotiflac_android/services/library_schema.dart';
import 'package:spotiflac_android/services/platform_bridge.dart';
import 'package:spotiflac_android/services/sqlite_helpers.dart' as sqlite;
export 'library_database_models.dart';
@@ -76,6 +77,118 @@ class LibraryDatabase {
bool get searchFtsAvailable => _searchFtsAvailable ?? false;
Future<Map<String, dynamic>> _nativeScanRequest(String sourceId) async {
final db = await database;
final history = await HistoryDatabase.instance.database;
final sources = await db.query(
'library_sources',
columns: ['path', 'enabled'],
where: 'id = ?',
whereArgs: [sourceId],
limit: 1,
);
if (sources.isEmpty || sources.single['enabled'] != 1) {
throw StateError('Library source changed or was removed during scan');
}
return {
'library_path': db.path,
'history_path': history.path,
'source_id': sourceId,
'expected_source_path': sources.single['path'],
'platform': Platform.isAndroid ? 'android' : 'ios',
};
}
/// Keeps native scan output native through parsing, indexing and the atomic
/// source swap. The stream implementation remains the desktop/test adapter.
Future<({int inserted, int skipped})> replaceSourceScanFile(
String sourceId,
LibraryScanNDJSONFile scan, {
required String requestId,
required bool Function() isCancelled,
}) async {
if (isCancelled()) throw StateError('Library scan cancelled');
if (PlatformBridge.supportsCoreBackend) {
final request = await _nativeScanRequest(sourceId);
if (isCancelled()) throw StateError('Library scan cancelled');
final result = await PlatformBridge.runNativeDataJob({
...request,
'operation': 'library_full_import',
'ndjson_path': scan.file.path,
'expected_count': scan.expectedCount,
'error_count': scan.errorCount,
}, requestId: requestId);
return (
inserted: (result['inserted'] as num).toInt(),
skipped: (result['skipped'] as num).toInt(),
);
}
Stream<Map<String, dynamic>> validatedRows() async* {
var count = 0;
await for (final row in scan.rows()) {
if (isCancelled()) {
throw StateError('Library scan cancelled during ingestion');
}
count++;
yield row;
}
if (count != scan.expectedCount) {
throw FormatException(
'Library scan row count mismatch: decoded $count, '
'expected ${scan.expectedCount}',
);
}
}
return replaceSourceStream(
sourceId,
validatedRows(),
preserveMissing: scan.errorCount > 0,
);
}
/// Exports timestamps directly from the DB on the worker. Android's adapter
/// backfills zero SAF timestamps in bounded ContentResolver batches first.
Future<String> writeNativeFileModTimesSnapshot(
String sourceId, {
required String requestId,
required bool Function() isCancelled,
}) async {
final request = await _nativeScanRequest(sourceId);
if (isCancelled()) throw StateError('Library scan cancelled');
final directory = await getTemporaryDirectory();
if (isCancelled()) throw StateError('Library scan cancelled');
final result = await PlatformBridge.runNativeDataJob({
...request,
'operation': 'library_snapshot',
'snapshot_path': join(
directory.path,
'library_file_mod_times_${DateTime.now().microsecondsSinceEpoch}.tsv',
),
}, requestId: requestId);
return result['path'] as String;
}
/// Native folder scanning and delta ingestion share one operation, so delta
/// track maps never return to Flutter. SAF traversal stays in its OS adapter.
Future<Map<String, dynamic>> scanIncrementalNative(
String sourceId,
String folderPath,
String snapshotPath, {
required bool isSaf,
required String requestId,
required bool Function() isCancelled,
}) async {
final request = await _nativeScanRequest(sourceId);
if (isCancelled()) throw StateError('Library scan cancelled');
return PlatformBridge.runNativeDataJob({
...request,
'operation': 'library_scan_incremental',
if (isSaf) 'tree_uri': folderPath else 'folder_path': folderPath,
'snapshot_path': snapshotPath,
}, requestId: requestId);
}
Future<List<LibrarySearchHit>> searchLibrary({
required String query,
required LibrarySearchKind kind,
+9 -1
View File
@@ -55,7 +55,7 @@ fn execute(
request: &Value,
bytes: &[u8],
check: &(dyn Fn() -> Result<(), String> + Sync),
_manager: Option<&ExtensionManager>,
manager: Option<&ExtensionManager>,
) -> Result<Value, String> {
let operation = string(request, "operation")?;
match operation {
@@ -120,6 +120,14 @@ fn execute(
)?;
Ok(json!({"cutoff": cutoff}))
}
"library_scan_incremental" => crate::native_library::scan_incremental(
manager.ok_or("Library scan requires the initialized backend")?,
request,
check,
),
operation if operation.starts_with("library_") => {
crate::native_library::execute(request, check)
}
_ => Err(format!("Unknown native data operation: {operation}")),
}
}
+1
View File
@@ -12,6 +12,7 @@ mod lyrics;
mod manager;
mod metadata;
mod native_id3;
mod native_library;
mod native_listing;
mod native_lyrics;
mod native_playlists;
@@ -0,0 +1,683 @@
//! App-owned Library persistence. The schema remains owned by Dart; scans enter
//! through one worker call and never transfer a library-sized row list to UI.
pub(crate) mod dart_date;
pub(crate) mod path_keys;
use crate::manager::ExtensionManager;
use rusqlite::{Connection, OpenFlags, TransactionBehavior, params, params_from_iter};
use serde_json::{Map, Value, json};
use std::{
fs::{self, File, OpenOptions},
io::{BufRead, BufReader, BufWriter, Read, Write},
path::Path,
time::Duration,
};
const SCHEMA_VERSION: i64 = 17;
const SCAN_VERSION: i64 = 4;
const BATCH_SIZE: usize = 300;
const MAX_ROW_BYTES: usize = 16 * 1024 * 1024;
const COLUMNS: &[(&str, &str)] = &[
("id", "id"),
("trackName", "track_name"),
("artistName", "artist_name"),
("albumName", "album_name"),
("albumArtist", "album_artist"),
("filePath", "file_path"),
("coverPath", "cover_path"),
("scannedAt", "scanned_at"),
("fileModTime", "file_mod_time"),
("isrc", "isrc"),
("trackNumber", "track_number"),
("totalTracks", "total_tracks"),
("discNumber", "disc_number"),
("totalDiscs", "total_discs"),
("duration", "duration"),
("releaseDate", "release_date"),
("bitDepth", "bit_depth"),
("sampleRate", "sample_rate"),
("bitrate", "bitrate"),
("genre", "genre"),
("composer", "composer"),
("label", "label"),
("copyright", "copyright"),
("format", "format"),
];
type Check<'a> = &'a dyn Fn() -> Result<(), String>;
fn error(context: &str, error: impl std::fmt::Display) -> String {
format!("{context}: {error}")
}
fn required<'a>(request: &'a Value, key: &str) -> Result<&'a str, String> {
request[key]
.as_str()
.filter(|value| !value.is_empty())
.ok_or_else(|| format!("Library request requires {key}"))
}
fn open(request: &Value, with_history: bool) -> Result<Connection, String> {
let path = required(request, "library_path")?;
if !Path::new(path).is_absolute()
|| Path::new(path).file_name().and_then(|v| v.to_str()) != Some("local_library.db")
{
return Err("Library database path must reference local_library.db".into());
}
// Never create a missing DB or migrate it from a worker. Dart initializes
// the current schema and optional FTS/triggers before handing us this path.
let db = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)
.map_err(|e| error("open Library", e))?;
db.busy_timeout(Duration::from_secs(5))
.map_err(|e| error("Library busy timeout", e))?;
let version: i64 = db
.pragma_query_value(None, "user_version", |row| row.get(0))
.map_err(|e| error("Library schema version", e))?;
if version != SCHEMA_VERSION {
return Err(format!(
"Library schema v{version} is incompatible with native v{SCHEMA_VERSION}"
));
}
db.execute_batch("PRAGMA recursive_triggers = ON; PRAGMA synchronous = NORMAL;")
.map_err(|e| error("configure Library", e))?;
if with_history {
let history = required(request, "history_path")?;
if !Path::new(history).is_absolute()
|| Path::new(history).file_name().and_then(|v| v.to_str()) != Some("history.db")
|| !Path::new(history).is_file()
{
return Err("History database must already exist".into());
}
db.execute("ATTACH DATABASE ? AS history_db", [history])
.map_err(|e| error("attach History", e))?;
}
Ok(db)
}
fn source_current(db: &Connection, request: &Value) -> Result<(), String> {
let source = required(request, "source_id")?;
let path = required(request, "expected_source_path")?;
let current: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM library_sources WHERE id = ? AND path = ? AND enabled = 1)",
params![source, path], |row| row.get(0),
).map_err(|e| error("check Library source", e))?;
if !current {
return Err("Library source changed or was removed during scan".into());
}
Ok(())
}
/// Unified native-data dispatcher entry point. All filesystem/SQLite work must
/// execute on the platform worker, never on its method-channel/main thread.
pub(crate) fn execute(request: &Value, check: Check<'_>) -> Result<Value, String> {
check()?;
match required(request, "operation")? {
"library_full_import" => import(request, None, check),
"library_incremental_import" => {
let owned;
let delta = if let Some(path) = request["delta_path"].as_str() {
let file = File::open(path).map_err(|e| error("open scan delta", e))?;
owned = serde_json::from_reader(BufReader::new(file))
.map_err(|e| error("decode scan delta file", e))?;
&owned
} else if let Some(text) = request["delta_json"].as_str() {
owned = serde_json::from_str(text).map_err(|e| error("decode scan delta", e))?;
&owned
} else {
&request["delta"]
};
if !delta.is_object() {
return Err("Library delta must be an object".into());
}
import(request, Some(delta), check)
}
"library_snapshot" => snapshot(request, check),
"library_snapshot_candidates" => snapshot_candidates(request, check),
_ => Err("Unknown native Library operation".into()),
}
}
/// The existing native scanner's delta stays native through ingestion. The
/// platform SAF adapter uses library_incremental_import with its native delta.
pub(crate) fn scan_incremental(
manager: &ExtensionManager,
request: &Value,
check: &(dyn Fn() -> Result<(), String> + Sync),
) -> Result<Value, String> {
let delta = manager
.inner
.scan_library_folder_incremental_from_snapshot(
required(request, "folder_path")?,
required(request, "snapshot_path")?,
check,
)?;
check()?;
if delta["cancelled"] == true {
return Err("Library scan cancelled".into());
}
import(request, Some(&delta), check)
}
fn normalized(value: &str) -> String {
path_keys::dart_lower(path_keys::dart_trim(value))
}
fn text<'a>(row: &'a Value, key: &str) -> Result<&'a str, String> {
match &row[key] {
Value::Null => Ok(""),
Value::String(value) => Ok(value),
_ => Err(format!("Library row {key} must be a string")),
}
}
fn flag(value: &Value) -> i64 {
i64::from(value == &Value::Bool(true) || value.as_f64() == Some(1.0))
}
fn has_replaygain(row: &Value) -> bool {
if flag(&row["hasReplayGain"]) != 0 {
return true;
}
for key in ["replaygain_track_gain", "replaygain_album_gain"] {
let value = match &row[key] {
Value::String(s) => s.clone(),
Value::Number(n) => n.to_string(),
_ => continue,
};
let trimmed = path_keys::dart_trim(&value);
let lowered = trimmed.to_lowercase();
let number = if lowered.ends_with("db") {
&trimmed[..trimmed.len() - 2]
} else {
trimmed
};
if number.trim().parse::<f64>().is_ok_and(f64::is_finite) {
return true;
}
}
false
}
fn numeric(value: &Value) -> Option<i64> {
value
.as_i64()
.or_else(|| value.as_f64().filter(|n| n.is_finite()).map(|n| n as i64))
}
fn scan_timestamp(value: &str) -> i64 {
dart_date::parse(value)
.map(|date| date.instant.timestamp_millis())
.unwrap_or(0)
}
fn incremental_row(row: &Value) -> Result<Value, String> {
let mut canonical = row.clone();
let timestamp = text(row, "scannedAt")?;
let parsed =
dart_date::parse(timestamp).ok_or("Invalid incremental Library scannedAt timestamp")?;
let formatted = dart_date::iso(&parsed);
canonical["scannedAt"] = formatted.into();
// Preserve the existing LocalLibraryItem.fromJson(...).toJson() contract,
// including its numeric truncation and scan-version defaults. Full-scan
// imports retain the raw scan flags through LibraryRowMapper instead.
let object = canonical
.as_object_mut()
.ok_or("Library row must be an object")?;
object.remove("audioMetadataScanVersion");
object.remove("metadataFromFilename");
for key in [
"fileModTime",
"trackNumber",
"totalTracks",
"discNumber",
"totalDiscs",
"duration",
"bitDepth",
"sampleRate",
"bitrate",
] {
if let Some(value) = object.get(key)
&& !value.is_null()
{
let value = numeric(value)
.ok_or_else(|| format!("Incremental Library {key} must be numeric"))?;
object.insert(key.into(), value.into());
}
}
Ok(canonical)
}
fn encode(row: &Value, source: &str) -> Result<Map<String, Value>, String> {
if !row.is_object() || path_keys::dart_trim(text(row, "id")?).is_empty() {
return Err("Library scan row has no valid id".into());
}
let mut values = Map::new();
for key in ["audioMetadataScanVersion", "fileModTime"] {
if !row[key].is_null() && !row[key].is_number() {
return Err(format!("Library {key} must be numeric"));
}
}
for (json_key, column) in COLUMNS {
values.insert((*column).into(), row[*json_key].clone());
}
// Staging CTAS does not retain NOT NULL constraints. Validate required
// storage fields before staging so corrupt input cannot survive to swap.
for key in [
"trackName",
"artistName",
"albumName",
"filePath",
"scannedAt",
] {
if !row[key].is_string() {
return Err(format!("Library scan row requires string {key}"));
}
}
values.insert("source_id".into(), source.into());
values.insert("explicit".into(), flag(&row["explicit"]).into());
values.insert("has_lyrics".into(), flag(&row["hasLyrics"]).into());
values.insert(
"has_replaygain".into(),
i64::from(has_replaygain(row)).into(),
);
values.insert(
"audio_metadata_scan_version".into(),
numeric(&row["audioMetadataScanVersion"])
.unwrap_or(if row["metadataFromFilename"] == true {
0
} else {
SCAN_VERSION
})
.into(),
);
let track = normalized(text(row, "trackName")?);
let artist = normalized(text(row, "artistName")?);
let album = normalized(text(row, "albumName")?);
let album_artist = if row["albumArtist"].is_null() {
artist.clone()
} else {
normalized(text(row, "albumArtist")?)
};
for (key, value) in [
("track_name_norm", track.clone()),
("artist_name_norm", artist.clone()),
("album_name_norm", album.clone()),
("album_artist_norm", album_artist.clone()),
("match_key", format!("{track}|{artist}")),
("album_key", format!("{album}|{album_artist}")),
] {
values.insert(key.into(), value.into());
}
let search_album_artist = if path_keys::dart_trim(text(row, "albumArtist")?).is_empty() {
artist.clone()
} else {
album_artist
};
values.insert(
"search_text".into(),
[track, artist, album, search_album_artist]
.into_iter()
.filter(|value| !value.is_empty())
.collect::<Vec<_>>()
.join(" ")
.into(),
);
values.insert("sort_genre".into(), normalized(text(row, "genre")?).into());
values.insert(
"sort_release".into(),
path_keys::dart_trim(text(row, "releaseDate")?).into(),
);
values.insert(
"sort_added".into(),
numeric(&row["fileModTime"])
.unwrap_or_else(|| scan_timestamp(text(row, "scannedAt").unwrap_or("")))
.into(),
);
Ok(values)
}
fn sql_value(value: &Value) -> Result<rusqlite::types::Value, String> {
use rusqlite::types::Value as SqlValue;
match value {
Value::Null => Ok(SqlValue::Null),
Value::String(s) => Ok(SqlValue::Text(s.clone())),
Value::Bool(v) => Ok(SqlValue::Integer(i64::from(*v))),
Value::Number(n) => n
.as_i64()
.map(SqlValue::Integer)
.or_else(|| n.as_f64().map(SqlValue::Real))
.ok_or("Invalid Library number".into()),
_ => Err("Library storage field cannot contain an object or array".into()),
}
}
fn import(request: &Value, delta: Option<&Value>, check: Check<'_>) -> Result<Value, String> {
let mut db = open(request, true)?;
source_current(&db, request)?;
let source = required(request, "source_id")?;
let columns = db
.prepare("PRAGMA table_info(library)")
.map_err(|e| error("Library columns", e))?
.query_map([], |row| row.get::<_, String>(1))
.map_err(|e| error("Library columns", e))?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| error("Library columns", e))?;
let column_list = columns
.iter()
.map(|name| format!("\"{}\"", name.replace('"', "\"\"")))
.collect::<Vec<_>>()
.join(", ");
db.execute_batch("CREATE TEMP TABLE native_library_stage AS SELECT * FROM library WHERE 0;
CREATE UNIQUE INDEX native_library_stage_id ON native_library_stage(id);
CREATE UNIQUE INDEX native_library_stage_path ON native_library_stage(file_path);
CREATE TEMP TABLE native_library_keys(item_id TEXT NOT NULL, path_key TEXT NOT NULL, PRIMARY KEY(item_id,path_key));
CREATE INDEX native_library_keys_key ON native_library_keys(path_key);")
.map_err(|e| error("stage Library scan", e))?;
let placeholders = vec!["?"; columns.len()].join(", ");
let insert = format!(
"INSERT OR REPLACE INTO native_library_stage ({column_list}) VALUES ({placeholders})"
);
let android = request["platform"] == "android";
let mut streamed = 0_usize;
let mut pending = Vec::with_capacity(BATCH_SIZE);
let mut flush = |rows: &mut Vec<Value>| -> Result<(), String> {
if rows.is_empty() {
return Ok(());
}
check()?;
let transaction = db
.transaction()
.map_err(|e| error("stage transaction", e))?;
for row in rows.drain(..) {
check()?;
let row = if delta.is_some() {
incremental_row(&row)?
} else {
row
};
let mapped = encode(&row, source)?;
let fields = columns
.iter()
.map(|column| sql_value(mapped.get(column).unwrap_or(&Value::Null)))
.collect::<Result<Vec<_>, _>>()?;
transaction
.prepare_cached(&insert)
.map_err(|e| error("prepare staged row", e))?
.execute(params_from_iter(fields))
.map_err(|e| error("stage Library row", e))?;
let id = text(&row, "id")?;
transaction
.execute("DELETE FROM native_library_keys WHERE item_id = ?", [id])
.map_err(|e| error("replace staged keys", e))?;
for key in path_keys::build(text(&row, "filePath")?, android)? {
transaction
.prepare_cached("INSERT OR IGNORE INTO native_library_keys VALUES (?, ?)")
.map_err(|e| error("prepare staged keys", e))?
.execute(params![id, key])
.map_err(|e| error("stage path key", e))?;
}
}
transaction
.commit()
.map_err(|e| error("commit staged batch", e))?;
Ok(())
};
if let Some(delta) = delta {
let rows = delta["files"]
.as_array()
.or_else(|| delta["scanned"].as_array())
.ok_or("Native incremental scan requires a rows array")?;
for row in rows {
pending.push(row.clone());
streamed += 1;
if pending.len() == BATCH_SIZE {
flush(&mut pending)?;
}
}
} else {
let expected = request["expected_count"]
.as_u64()
.ok_or("Full scan requires expected_count")?;
let mut reader = BufReader::new(
File::open(required(request, "ndjson_path")?)
.map_err(|e| error("open scan NDJSON", e))?,
);
let mut line = String::new();
loop {
check()?;
line.clear();
let bytes = reader
.by_ref()
.take((MAX_ROW_BYTES + 1) as u64)
.read_line(&mut line)
.map_err(|e| error("read scan NDJSON", e))?;
if bytes == 0 {
break;
}
if bytes > MAX_ROW_BYTES {
return Err("Library scan NDJSON row is too large".into());
}
if path_keys::dart_trim(&line).is_empty() {
continue;
}
pending.push(serde_json::from_str(&line).map_err(|e| error("decode scan NDJSON", e))?);
streamed += 1;
if pending.len() == BATCH_SIZE {
flush(&mut pending)?;
}
}
if streamed as u64 != expected {
return Err(format!(
"Library scan row count mismatch: decoded {streamed}, expected {expected}"
));
}
}
flush(&mut pending)?;
check()?;
let transaction = db
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(|e| error("begin Library swap", e))?;
source_current(&transaction, request)?;
let history_match = "EXISTS(SELECT 1 FROM native_library_keys sk JOIN history_db.history_path_keys hk ON hk.path_key = sk.path_key WHERE sk.item_id = s.id)";
let staged: i64 = transaction
.query_row("SELECT COUNT(*) FROM native_library_stage", [], |row| {
row.get(0)
})
.map_err(|e| error("count staged rows", e))?;
let skipped: i64 = transaction
.query_row(
&format!("SELECT COUNT(*) FROM native_library_stage s WHERE {history_match}"),
[],
|row| row.get(0),
)
.map_err(|e| error("count excluded downloads", e))?;
let mut deleted = 0;
let mut excluded_existing = 0;
if let Some(delta) = delta {
// Remove files which became downloads, then apply this source's delta
// atomically. All deletes remain source-scoped even if aliases overlap.
let excluded = "source_id = ? AND EXISTS(SELECT 1 FROM library_path_keys lk JOIN history_db.history_path_keys hk ON hk.path_key = lk.path_key WHERE lk.item_id = library.id)";
transaction.execute(&format!("CREATE TEMP TABLE native_library_excluded AS SELECT id FROM library WHERE {excluded}"), [source]).map_err(|e| error("select downloaded rows", e))?;
transaction.execute("DELETE FROM library_path_keys WHERE item_id IN(SELECT id FROM native_library_excluded)", []).map_err(|e| error("remove downloaded path keys", e))?;
excluded_existing += transaction
.execute(
"DELETE FROM library WHERE id IN(SELECT id FROM native_library_excluded)",
[],
)
.map_err(|e| error("remove downloaded rows", e))?;
let paths = delta["removedUris"]
.as_array()
.or_else(|| delta["deletedPaths"].as_array());
if let Some(paths) = paths {
for path in paths {
check()?;
let path = path
.as_str()
.ok_or("Deleted Library path must be a string")?;
transaction.execute("DELETE FROM library_path_keys WHERE item_id IN(SELECT id FROM library WHERE source_id = ? AND file_path = ?)", params![source,path]).map_err(|e| error("delete removed path keys", e))?;
deleted += transaction
.execute(
"DELETE FROM library WHERE source_id = ? AND file_path = ?",
params![source, path],
)
.map_err(|e| error("delete removed track", e))?;
}
}
// staged/download overlap is the same anti-join used by full scans.
let ids = format!(
"SELECT id FROM library WHERE source_id = ? AND file_path IN(SELECT file_path FROM native_library_stage s WHERE {history_match})"
);
transaction
.execute(
&format!("DELETE FROM library_path_keys WHERE item_id IN({ids})"),
[source],
)
.map_err(|e| error("remove excluded keys", e))?;
excluded_existing += transaction
.execute(&format!("DELETE FROM library WHERE id IN({ids})"), [source])
.map_err(|e| error("remove excluded rows", e))?;
} else {
let errors = request["error_count"]
.as_u64()
.ok_or("Full scan requires nonnegative error_count")?;
let where_clause = if errors > 0 {
"source_id = ? AND (id IN(SELECT id FROM native_library_stage) OR file_path IN(SELECT file_path FROM native_library_stage))"
} else {
"source_id = ?"
};
transaction.execute(&format!("DELETE FROM library_path_keys WHERE item_id IN(SELECT id FROM library WHERE {where_clause})"), [source]).map_err(|e| error("replace source keys", e))?;
transaction
.execute(
&format!("DELETE FROM library WHERE {where_clause}"),
[source],
)
.map_err(|e| error("replace source rows", e))?;
}
check()?;
transaction.execute(&format!("DELETE FROM library_path_keys WHERE item_id IN(SELECT s.id FROM native_library_stage s WHERE NOT {history_match})"), []).map_err(|e| error("replace imported keys", e))?;
transaction.execute(&format!("INSERT OR REPLACE INTO library({column_list}) SELECT {} FROM native_library_stage s WHERE NOT {history_match}", columns.iter().map(|c| format!("s.\"{c}\"")).collect::<Vec<_>>().join(", ")), []).map_err(|e| error("swap imported rows", e))?;
transaction.execute(&format!("INSERT OR IGNORE INTO library_path_keys SELECT sk.item_id,sk.path_key FROM native_library_keys sk JOIN native_library_stage s ON s.id=sk.item_id WHERE NOT {history_match}"), []).map_err(|e| error("swap imported keys", e))?;
check()?;
transaction
.commit()
.map_err(|e| error("commit Library swap", e))?;
Ok(
json!({"committed":true,"inserted":staged-skipped,"upserted":staged-skipped,"skipped":skipped+excluded_existing as i64,"scanned_count":streamed,"deleted_count":deleted,"skippedCount":delta.map(|d|d["skippedCount"].clone()).unwrap_or(json!(0)),"totalFiles":delta.map(|d|d["totalFiles"].clone()).unwrap_or(json!(streamed))}),
)
}
fn snapshot_candidates(request: &Value, check: Check<'_>) -> Result<Value, String> {
let db = open(request, false)?;
source_current(&db, request)?;
let cursor = request["cursor"].as_str().unwrap_or("");
let limit = request["limit"].as_i64().unwrap_or(500).clamp(1, 500);
let mut statement = db.prepare("SELECT id,file_path FROM library WHERE source_id=? AND id>? AND COALESCE(file_mod_time,0)=0 AND audio_metadata_scan_version>=4 AND file_path LIKE 'content://%' ORDER BY id LIMIT ?").map_err(|e| error("prepare snapshot backfill", e))?;
let rows = statement
.query_map(
params![required(request, "source_id")?, cursor, limit + 1],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
)
.map_err(|e| error("query snapshot backfill", e))?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| error("read snapshot backfill", e))?;
check()?;
let has_more = rows.len() > limit as usize;
let rows = rows.into_iter().take(limit as usize).collect::<Vec<_>>();
Ok(
json!({"paths":rows.iter().map(|(id,path)|json!({"id":id,"file_path":path})).collect::<Vec<_>>(),"cursor":rows.last().map(|(id,_)|id.as_str()).unwrap_or(cursor),"has_more":has_more}),
)
}
fn snapshot(request: &Value, check: Check<'_>) -> Result<Value, String> {
let mut db = open(request, false)?;
source_current(&db, request)?;
let path = required(request, "snapshot_path")?;
if !Path::new(path).is_absolute() || !path.ends_with(".tsv") {
return Err("Snapshot output must be an absolute TSV path".into());
}
let file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|e| error("create incremental snapshot", e))?;
let result = (|| {
// A WAL read snapshot does not hold the persistent writer during file
// inspection/output. Backfill acquires its own short write transaction.
let transaction = db.transaction().map_err(|e| error("begin snapshot", e))?;
source_current(&transaction, request)?;
let mut statement = transaction.prepare("SELECT file_path,COALESCE(file_mod_time,0),COALESCE(audio_metadata_scan_version,0) FROM library WHERE source_id=? ORDER BY id").map_err(|e| error("prepare snapshot", e))?;
let mut rows = statement
.query([required(request, "source_id")?])
.map_err(|e| error("query snapshot", e))?;
let mut output = BufWriter::new(file);
let mut count = 0;
let mut backfilled = Vec::new();
while let Some(row) = rows.next().map_err(|e| error("read snapshot", e))? {
check()?;
let file_path: String = row.get(0).map_err(|e| error("snapshot file path", e))?;
let stored: i64 = row.get(1).map_err(|e| error("snapshot timestamp", e))?;
let version: i64 = row.get(2).map_err(|e| error("snapshot scan version", e))?;
let mut timestamp = if version < SCAN_VERSION { -1 } else { stored };
if timestamp == 0 {
let supplied =
numeric(&request["backfilled_mod_times"][&file_path]).filter(|v| *v > 0);
let discovered = if !file_path.starts_with("content://")
&& !file_path.starts_with("network://")
{
fs::metadata(&file_path)
.ok()
.filter(|m| m.is_file())
.and_then(|m| m.modified().ok())
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_millis() as i64)
} else {
None
};
if let Some(time) = supplied.or(discovered) {
timestamp = time;
backfilled.push((file_path.clone(), time));
}
}
if file_path.is_empty() {
continue;
}
if file_path.contains(['\n', '\r']) {
return Err("Snapshot file path contains a line break".into());
}
writeln!(output, "{timestamp}\t{file_path}").map_err(|e| error("write snapshot", e))?;
count += 1;
}
drop(rows);
drop(statement);
transaction
.commit()
.map_err(|e| error("close snapshot read", e))?;
output.flush().map_err(|e| error("flush snapshot", e))?;
output
.get_ref()
.sync_all()
.map_err(|e| error("sync snapshot", e))?;
check()?;
let write = db
.transaction_with_behavior(TransactionBehavior::Immediate)
.map_err(|e| error("begin timestamp backfill", e))?;
source_current(&write, request)?;
for (file_path, timestamp) in &backfilled {
check()?;
write.execute("UPDATE library SET file_mod_time=?,sort_added=? WHERE source_id=? AND file_path=? AND COALESCE(file_mod_time,0)=0 AND audio_metadata_scan_version>=4",params![timestamp,timestamp,required(request,"source_id")?,file_path]).map_err(|e| error("backfill timestamp", e))?;
}
check()?;
write
.commit()
.map_err(|e| error("commit timestamp backfill", e))?;
Ok(json!({"committed":true,"path":path,"count":count,"backfilled_count":backfilled.len()}))
})();
if result.is_err() {
let _ = fs::remove_file(path);
}
result
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,99 @@
//! DateTime.parse's accepted ISO subset, including compact dates, fractional
//! truncation and overflowing components used by old JSON backups.
use chrono::{DateTime, Duration, Local, NaiveDate, NaiveDateTime, SecondsFormat, TimeZone, Utc};
use regex::Regex;
use std::sync::OnceLock;
pub(crate) struct Parsed {
pub instant: DateTime<Utc>,
pub local: Option<NaiveDateTime>,
}
pub(crate) fn parse(value: &str) -> Option<Parsed> {
static FORMAT: OnceLock<Regex> = OnceLock::new();
let format=FORMAT.get_or_init(||Regex::new(r"^([+-]?[0-9]{4,6})-?([0-9]{2})-?([0-9]{2})(?:[ T]([0-9]{2})(?::?([0-9]{2})(?::?([0-9]{2})(?:[.,]([0-9]+))?)?)?( ?[zZ]| ?([-+])([0-9]{2})(?::?([0-9]{2}))?)?)?$").expect("Dart ISO pattern"));
let parts = format.captures(value)?;
let integer = |index| {
parts
.get(index)
.map_or(Some(0_i64), |v| v.as_str().parse::<i64>().ok())
};
let year = integer(1)?;
let month = integer(2)? - 1;
let normalized_year = year.checked_add(month.div_euclid(12))?;
let normalized_month = month.rem_euclid(12) + 1;
let day = NaiveDate::from_ymd_opt(
i32::try_from(normalized_year).ok()?,
normalized_month as u32,
1,
)?;
let mut minutes = integer(5)?;
if parts.get(9).is_some() {
let sign = if parts.get(9)?.as_str() == "-" { -1 } else { 1 };
minutes -= sign * (60 * integer(10)? + integer(11)?);
}
let fraction = parts
.get(7)
.map(|v| {
v.as_str()
.bytes()
.take(6)
.fold((0_i64, 0), |(n, len), digit| {
(n * 10 + i64::from(digit - b'0'), len + 1)
})
})
.unwrap_or((0, 6));
let micros = fraction.0 * 10_i64.pow(6 - fraction.1);
let seconds = (integer(3)? - 1)
.checked_mul(86400)?
.checked_add(integer(4)?.checked_mul(3600)?)?
.checked_add(minutes.checked_mul(60)?)?
.checked_add(integer(6)?)?;
let naive = day
.and_hms_opt(0, 0, 0)?
.checked_add_signed(Duration::seconds(seconds))?
.checked_add_signed(Duration::microseconds(micros))?;
let utc = parts.get(8).is_some();
let instant = if utc {
naive.and_utc()
} else {
Local
.from_local_datetime(&naive)
.earliest()?
.with_timezone(&Utc)
};
Some(Parsed {
instant,
local: if utc { None } else { Some(naive) },
})
}
pub(crate) fn iso_utc(value: &DateTime<Utc>) -> String {
value.to_rfc3339_opts(
if value.timestamp_subsec_nanos().is_multiple_of(1_000_000) {
SecondsFormat::Millis
} else {
SecondsFormat::Micros
},
true,
)
}
pub(crate) fn iso(value: &Parsed) -> String {
match value.local {
None => iso_utc(&value.instant),
Some(date) => date
.format(
if date
.and_utc()
.timestamp_subsec_nanos()
.is_multiple_of(1_000_000)
{
"%Y-%m-%dT%H:%M:%S%.3f"
} else {
"%Y-%m-%dT%H:%M:%S%.6f"
},
)
.to_string(),
}
}
@@ -0,0 +1,316 @@
//! Physical/converted path identities shared with Dart's path_match_keys.dart.
//! Opaque SAF document identities decode once and preserve case and extension.
use base64::{Engine, engine::general_purpose::URL_SAFE};
use std::collections::{BTreeSet, VecDeque};
use url::Url;
const ALIASES: &[&str] = &[
"/storage/emulated/0",
"/storage/emulated/legacy",
"/storage/self/primary",
"/sdcard",
"/mnt/sdcard",
];
const EXTENSIONS: &[&str] = &[
".flac", ".m4a", ".mp3", ".opus", ".ogg", ".wav", ".aiff", ".aif", ".aac",
];
pub(crate) fn dart_trim(value: &str) -> &str {
value.trim_matches(|ch: char| ch.is_whitespace() || ch == '\u{feff}')
}
/// Dart applies Unicode simple casing, without multi-character expansions or
/// the final-sigma context rules of Rust's full string lowercase operation.
pub(crate) fn dart_lower(value: &str) -> String {
value
.chars()
.map(|ch| ch.to_lowercase().next().unwrap_or(ch))
.collect()
}
fn decode(value: &str) -> Option<String> {
let bytes = value.as_bytes();
let mut decoded = Vec::with_capacity(bytes.len());
let mut index = 0;
while index < bytes.len() {
if bytes[index] == b'%' {
let high = (*bytes.get(index + 1)? as char).to_digit(16)?;
let low = (*bytes.get(index + 2)? as char).to_digit(16)?;
decoded.push((high * 16 + low) as u8);
index += 3;
} else {
decoded.push(bytes[index]);
index += 1;
}
}
String::from_utf8(decoded).ok()
}
fn dart_uri(value: &str) -> Result<Url, url::ParseError> {
// Uri.parse escapes stray '%' before pathSegments decodes them. Keeping
// them unescaped would lose opaque document identities with literal '%'.
let mut sanitized = String::with_capacity(value.len());
let bytes = value.as_bytes();
for (index, ch) in value.char_indices() {
if ch == '%'
&& !(bytes.get(index + 1).is_some_and(u8::is_ascii_hexdigit)
&& bytes.get(index + 2).is_some_and(u8::is_ascii_hexdigit))
{
sanitized.push_str("%25");
} else {
sanitized.push(ch);
}
}
Url::parse(&sanitized)
}
fn segments(uri: &Url) -> Option<Vec<String>> {
uri.path_segments()?.map(decode).collect()
}
fn document_key(uri: &Url) -> Option<String> {
if uri.scheme() != "content" {
return None;
}
let authority = &uri[url::Position::BeforeUsername..url::Position::AfterPort];
if authority.is_empty() {
return None;
}
let parts = segments(uri)?;
let direct = parts.len() == 2 && parts[0] == "document";
let tree =
parts.len() == 4 && parts[0] == "tree" && !parts[1].is_empty() && parts[2] == "document";
let id = parts.last()?;
if !(direct || tree) || id.is_empty() {
return None;
}
Some(format!(
"saf-document:{}:{id}",
URL_SAFE.encode(dart_lower(authority))
))
}
fn storage_documents(uri: &Url) -> Vec<String> {
if uri.scheme() != "content"
|| !uri
.host_str()
.is_some_and(|host| host.eq_ignore_ascii_case("com.android.externalstorage.documents"))
{
return Vec::new();
}
let Some(parts) = segments(uri) else {
return Vec::new();
};
let index = parts
.iter()
.rposition(|s| s == "document")
.or_else(|| parts.iter().rposition(|s| s == "tree"));
let Some(index) = index else {
return Vec::new();
};
if index + 1 >= parts.len() {
return Vec::new();
}
let joined = parts[index + 1..].join("/");
let id = decode(&joined).unwrap_or(joined);
let Some((volume, relative)) = id.split_once(':') else {
return Vec::new();
};
if volume.to_lowercase() != "primary" {
return Vec::new();
}
let relative = relative.replace('\\', "/");
let relative = relative.trim_start_matches('/');
ALIASES
.iter()
.map(|prefix| {
if relative.is_empty() {
(*prefix).into()
} else {
format!("{prefix}/{relative}")
}
})
.collect()
}
fn android_aliases(value: &str) -> Vec<String> {
let normalized = value.replace('\\', "/");
let lower = dart_lower(&normalized);
for prefix in ALIASES {
if lower == *prefix || lower.starts_with(&format!("{prefix}/")) {
let suffix = &normalized[prefix.len()..];
return ALIASES
.iter()
.map(|alias| format!("{alias}{suffix}"))
.collect();
}
}
Vec::new()
}
fn uri_text(uri: &Url) -> String {
// Dart canonicalizes percent escape hex and decodes unreserved ASCII in
// Uri.parse. url::Url keeps original percent escapes for custom schemes.
let mut uri = Url::parse(&uri.as_str().replace('\\', "/")).unwrap_or_else(|_| uri.clone());
if let Some(host) = uri.host_str().map(dart_lower) {
let _ = uri.set_host(Some(&host));
}
let bytes = uri.as_str().as_bytes();
let mut output = String::with_capacity(bytes.len());
let mut index = 0;
while index < bytes.len() {
if bytes[index] == b'%'
&& index + 2 < bytes.len()
&& let (Some(high), Some(low)) = (
(bytes[index + 1] as char).to_digit(16),
(bytes[index + 2] as char).to_digit(16),
)
{
let value = (high * 16 + low) as u8;
if value.is_ascii_alphanumeric() || b"-._~".contains(&value) {
output.push(value as char);
} else {
output.push_str(&format!("%{value:02X}"));
}
index += 3;
} else {
let tail = &uri.as_str()[index..];
let ch = tail.chars().next().expect("URI character");
output.push(ch);
index += ch.len_utf8();
}
}
output
}
pub(crate) fn build(path: &str, android: bool) -> Result<BTreeSet<String>, String> {
let path = dart_trim(path);
let cleaned = path.strip_prefix("EXISTS:").map(dart_trim).unwrap_or(path);
if cleaned.is_empty() {
return Ok(BTreeSet::new());
}
let mut keys = BTreeSet::new();
let mut visited = BTreeSet::new();
let mut pending = VecDeque::from([cleaned.to_string()]);
if let Ok(uri) = dart_uri(cleaned)
&& let Some(key) = document_key(&uri)
{
keys.insert(key);
}
while let Some(value) = pending.pop_front() {
let trimmed = dart_trim(&value);
if trimmed.is_empty() || !visited.insert(trimmed.to_string()) {
continue;
}
// A malicious string can recursively expose many percent-encoded
// variants. The existing cache is bounded; keep the native walk bounded.
if visited.len() > 1024 {
return Err("Too many Library path aliases".into());
}
keys.insert(trimmed.to_string());
keys.insert(dart_lower(trimmed));
if trimmed.contains('\\') {
pending.push_back(trimmed.replace('\\', "/"));
}
if trimmed.contains('%')
&& let Some(decoded) = decode(trimmed)
&& decoded != trimmed
{
pending.push_back(decoded);
}
if let Ok(uri) = dart_uri(trimmed) {
// replace(query:null, fragment:null) in Dart retains existing fields.
// Preserve that behavior; clearing them would change path identity.
// WHATWG normalizes file://localhost to an empty host. Dart keeps
// that authority and refuses toFilePath on Unix, so do not invent
// a local filesystem alias for it.
let localhost = trimmed
.get(..16)
.is_some_and(|v| v.eq_ignore_ascii_case("file://localhost"))
&& (trimmed.len() == 16
|| trimmed
.get(16..17)
.is_some_and(|v| ["/", "?", "#"].contains(&v)));
let text = if localhost {
let alternate = format!("nativefile{}", &trimmed[4..]);
let mut uri = dart_uri(&alternate).map_err(|e| e.to_string())?;
if uri.path().is_empty() {
uri.set_path("/");
}
uri_text(&uri).replacen("nativefile:", "file:", 1)
} else {
uri_text(&uri)
};
keys.insert(text.clone());
keys.insert(dart_lower(&text));
if uri.scheme() == "file"
&& !localhost
&& uri.query().is_none()
&& uri.fragment().is_none()
&& let Ok(file) = uri.to_file_path()
{
pending.push_back(file.to_string_lossy().into_owned());
}
pending.extend(storage_documents(&uri));
} else if trimmed.starts_with('/')
&& let Ok(uri) = Url::from_file_path(trimmed)
{
let text = uri_text(&uri);
keys.insert(text.clone());
keys.insert(dart_lower(&text));
}
if android {
pending.extend(android_aliases(trimmed));
}
}
let extensionless = keys
.iter()
.filter_map(|key| {
let lower = dart_lower(key);
EXTENSIONS
.iter()
.find(|extension| lower.ends_with(**extension))
.map(|extension| key[..key.len() - extension.len()].to_string())
.filter(|stripped| !stripped.is_empty())
})
.collect::<Vec<_>>();
keys.extend(extensionless);
Ok(keys)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn saf_document_aliases_keep_opaque_case_and_escape_identity() {
let direct = build("content://nas.provider/document/Ab%252FC.flac", false).unwrap();
let tree = build(
"content://nas.provider/tree/root/document/Ab%252FC.flac",
false,
)
.unwrap();
let expected = format!(
"saf-document:{}:Ab%2FC.flac",
URL_SAFE.encode("nas.provider")
);
assert!(direct.contains(&expected));
assert!(tree.contains(&expected));
assert!(!direct.contains(&expected.replace("Ab%2FC", "ab%2fc")));
}
#[test]
fn android_storage_and_converted_paths_share_keys() {
let direct = build("/sdcard/Music/Café.flac", true).unwrap();
let converted = build("/storage/emulated/0/Music/Café.opus", true).unwrap();
let saf = build("content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9.flac",true).unwrap();
assert!(
direct
.intersection(&converted)
.any(|key| key == "/storage/emulated/0/Music/Café")
);
assert!(
direct
.intersection(&saf)
.any(|key| key == "/storage/emulated/0/Music/Café.flac")
);
}
}
@@ -0,0 +1,465 @@
use super::*;
use std::cell::Cell;
use tempfile::TempDir;
struct Fixture {
directory: TempDir,
db: Connection,
request: Value,
}
impl Fixture {
fn new() -> Self {
let directory = tempfile::tempdir().unwrap();
let library = directory.path().join("local_library.db");
let history = directory.path().join("history.db");
let db = Connection::open(&library).unwrap();
let example = encode(&track("schema", "/music/schema.flac"), "source").unwrap();
let definitions = example
.keys()
.map(|column| {
let kind = match column.as_str() {
"id" => "TEXT PRIMARY KEY",
"file_path" => "TEXT UNIQUE",
"file_mod_time"
| "sort_added"
| "audio_metadata_scan_version"
| "explicit"
| "has_lyrics"
| "has_replaygain" => "INTEGER",
_ => "TEXT",
};
format!("{column} {kind}")
})
.collect::<Vec<_>>()
.join(",");
db.execute_batch(&format!("PRAGMA journal_mode=WAL; PRAGMA user_version=17;
CREATE TABLE library({definitions});
CREATE TABLE library_sources(id TEXT PRIMARY KEY,path TEXT,enabled INTEGER);
INSERT INTO library_sources VALUES('source','/music',1),('other','/other',1);
CREATE TABLE library_path_keys(item_id TEXT,path_key TEXT,PRIMARY KEY(item_id,path_key));
CREATE INDEX library_path_key_index ON library_path_keys(path_key);
CREATE VIRTUAL TABLE library_search_fts USING fts5(search_text,content='library',content_rowid='rowid',tokenize='trigram');
CREATE TRIGGER library_fts_ai AFTER INSERT ON library BEGIN INSERT INTO library_search_fts(rowid,search_text) VALUES(new.rowid,new.search_text); END;
CREATE TRIGGER library_fts_ad AFTER DELETE ON library BEGIN INSERT INTO library_search_fts(library_search_fts,rowid,search_text) VALUES('delete',old.rowid,old.search_text); END;" )).unwrap();
let history_db = Connection::open(&history).unwrap();
history_db.execute_batch("CREATE TABLE history_path_keys(item_id TEXT,path_key TEXT); CREATE INDEX history_key_index ON history_path_keys(path_key);").unwrap();
let request = json!({"operation":"library_full_import","library_path":library.to_str().unwrap(),"history_path":history.to_str().unwrap(),"source_id":"source","expected_source_path":"/music","platform":"ios","error_count":0});
Self {
directory,
db,
request,
}
}
fn add(&self, id: &str, path: &str, source: &str) {
let values = encode(&track(id, path), source).unwrap();
let columns = values.keys().cloned().collect::<Vec<_>>();
self.db
.execute(
&format!(
"INSERT INTO library({}) VALUES({})",
columns.join(","),
vec!["?"; columns.len()].join(",")
),
params_from_iter(columns.iter().map(|key| sql_value(&values[key]).unwrap())),
)
.unwrap();
for key in path_keys::build(path, false).unwrap() {
self.db
.execute(
"INSERT INTO library_path_keys VALUES(?,?)",
params![id, key],
)
.unwrap();
}
}
fn scan(
&self,
rows: &[Value],
expected: u64,
errors: u64,
check: Check<'_>,
) -> Result<Value, String> {
let path = self.directory.path().join("scan.ndjson");
let mut output = File::create(&path).unwrap();
for row in rows {
writeln!(output, "{row}").unwrap();
}
let mut request = self.request.clone();
request["ndjson_path"] = path.to_str().unwrap().into();
request["expected_count"] = expected.into();
request["error_count"] = errors.into();
execute(&request, check)
}
fn ids(&self) -> Vec<String> {
self.db
.prepare("SELECT id FROM library ORDER BY id")
.unwrap()
.query_map([], |row| row.get(0))
.unwrap()
.map(Result::unwrap)
.collect()
}
}
fn track(id: &str, path: &str) -> Value {
json!({"id":id,"trackName":format!("Track {id}"),"artistName":"Artist","albumName":"Album","albumArtist":null,"filePath":path,"scannedAt":"2026-10-06T12:00:00.000Z","fileModTime":1770000000000_i64,"isrc":"USAAA2600001","hasReplayGain":false,"replaygain_track_gain":"-4.5 dB"})
}
#[test]
fn production_dart_row_and_path_codec_parity() {
let fixture: Value = serde_json::from_str(include_str!(
"../../../../../test/fixtures/native_library_contract.json"
))
.unwrap();
for entry in fixture["rows"].as_array().unwrap() {
assert_eq!(
Value::Object(encode(&entry["input"], "source").unwrap()),
entry["expected"],
"row {:?}",
entry["input"]
);
}
let mut mismatches = Vec::new();
for entry in fixture["paths"].as_array().unwrap() {
let keys = path_keys::build(entry["input"].as_str().unwrap(), false).unwrap();
let expected = entry["expected"]
.as_array()
.unwrap()
.iter()
.map(|v| v.as_str().unwrap().to_string())
.collect::<std::collections::BTreeSet<_>>();
if keys != expected {
mismatches.push(format!(
"path {:?}: extra {:?}, missing {:?}",
entry["input"],
keys.difference(&expected).collect::<Vec<_>>(),
expected.difference(&keys).collect::<Vec<_>>()
));
}
}
assert!(mismatches.is_empty(), "{}", mismatches.join("\n"));
}
#[test]
fn cancellation_during_atomic_swap_rolls_back_rows_keys_and_fts() {
let fixture = Fixture::new();
fixture.add("old", "/music/old.flac", "source");
let calls = Cell::new(0);
let check = || {
calls.set(calls.get() + 1);
if calls.get() == 8 {
Err("cancelled during swap".into())
} else {
Ok(())
}
};
assert!(
fixture
.scan(&[track("new", "/music/new.flac")], 1, 0, &check)
.is_err()
);
assert_eq!(calls.get(), 8);
assert_eq!(fixture.ids(), ["old"]);
assert!(
fixture
.db
.query_row(
"SELECT COUNT(*) FROM library_path_keys WHERE item_id='old'",
[],
|row| row.get::<_, i64>(0)
)
.unwrap()
> 0
);
assert_eq!(
fixture
.db
.query_row(
"SELECT COUNT(*) FROM library_search_fts WHERE library_search_fts MATCH ?",
["old"],
|row| row.get::<_, i64>(0)
)
.unwrap(),
1
);
}
#[test]
fn external_wal_writer_can_publish_while_native_scan_stages_temp_rows() {
let fixture = Fixture::new();
fixture.add("other", "/other/track.flac", "other");
let path = fixture.request["library_path"].as_str().unwrap().to_owned();
let calls = Cell::new(0);
let thread = std::cell::RefCell::new(None);
let check = || {
calls.set(calls.get() + 1);
if calls.get() == 305 {
let (sender, receiver) = std::sync::mpsc::channel();
let path = path.clone();
let handle = std::thread::spawn(move || {
let db = Connection::open(path).unwrap();
db.busy_timeout(Duration::from_secs(2)).unwrap();
db.execute_batch("BEGIN IMMEDIATE;UPDATE library_sources SET path='/writer-kept' WHERE id='other';").unwrap();
sender.send(()).unwrap();
std::thread::sleep(Duration::from_millis(50));
db.execute_batch("COMMIT").unwrap();
});
*thread.borrow_mut() = Some(handle);
receiver.recv_timeout(Duration::from_secs(3)).unwrap();
}
Ok(())
};
let rows = (0..300)
.map(|i| track(&format!("new-{i}"), &format!("/music/{i}.flac")))
.collect::<Vec<_>>();
fixture.scan(&rows, 300, 0, &check).unwrap();
thread.into_inner().unwrap().join().unwrap();
assert_eq!(fixture.ids().len(), 301);
assert_eq!(
fixture
.db
.query_row(
"SELECT path FROM library_sources WHERE id='other'",
[],
|row| row.get::<_, String>(0)
)
.unwrap(),
"/writer-kept"
);
}
#[test]
fn spilled_incremental_json_is_consumed_without_deleting_adapter_file() {
let fixture = Fixture::new();
let path = fixture.directory.path().join("delta.json");
fs::write(&path,json!({"files":[track("new","/music/new.flac")],"removedUris":[],"skippedCount":3,"totalFiles":4}).to_string()).unwrap();
let mut request = fixture.request.clone();
request["operation"] = "library_incremental_import".into();
request["delta_path"] = path.to_str().unwrap().into();
let result = execute(&request, &|| Ok(())).unwrap();
assert_eq!(fixture.ids(), ["new"]);
assert_eq!(result["skippedCount"], 3);
assert!(path.exists());
}
#[test]
fn full_swap_excludes_downloads_and_preserves_other_source_and_fts() {
let fixture = Fixture::new();
fixture.add("old", "/music/old.flac", "source");
fixture.add("other", "/other/track.flac", "other");
let history = Connection::open(fixture.request["history_path"].as_str().unwrap()).unwrap();
history
.execute(
"INSERT INTO history_path_keys VALUES('downloaded','/music/downloaded')",
[],
)
.unwrap();
let result = fixture
.scan(
&[
track("new", "/music/new.flac"),
track("downloaded", "/music/downloaded.opus"),
],
2,
0,
&|| Ok(()),
)
.unwrap();
assert_eq!(result["inserted"], 1);
assert_eq!(result["skipped"], 1);
assert_eq!(fixture.ids(), ["new", "other"]);
assert_eq!(
fixture
.db
.query_row(
"SELECT COUNT(*) FROM library_search_fts WHERE library_search_fts MATCH ?",
["new"],
|row| row.get::<_, i64>(0)
)
.unwrap(),
1
);
assert_eq!(
fixture
.db
.query_row(
"SELECT COUNT(*) FROM library_search_fts WHERE library_search_fts MATCH ?",
["old"],
|row| row.get::<_, i64>(0)
)
.unwrap(),
0
);
}
#[test]
fn invalid_rows_count_mismatch_and_cancel_never_replace_existing_index() {
for failure in ["invalid", "count", "cancel"] {
let fixture = Fixture::new();
fixture.add("old", "/music/old.flac", "source");
let mut rows = (0..650)
.map(|index| track(&format!("row-{index}"), &format!("/music/{index}.flac")))
.collect::<Vec<_>>();
if failure == "invalid" {
rows[640]["id"] = Value::Null;
}
let calls = Cell::new(0);
let check = || {
calls.set(calls.get() + 1);
if failure == "cancel" && calls.get() > 640 {
Err("cancelled".into())
} else {
Ok(())
}
};
assert!(
fixture
.scan(&rows, if failure == "count" { 649 } else { 650 }, 0, &check)
.is_err()
);
assert_eq!(fixture.ids(), ["old"]);
}
}
#[test]
fn partial_scan_preserves_unreadable_rows_and_empty_success_clears_only_source() {
let fixture = Fixture::new();
fixture.add("unreadable", "/music/unreadable.flac", "source");
fixture.add("old-id", "/music/changed.flac", "source");
fixture.add("other", "/other/track.flac", "other");
fixture
.scan(&[track("new-id", "/music/changed.flac")], 1, 1, &|| Ok(()))
.unwrap();
assert_eq!(fixture.ids(), ["new-id", "other", "unreadable"]);
fixture.scan(&[], 0, 1, &|| Ok(())).unwrap();
assert_eq!(fixture.ids(), ["new-id", "other", "unreadable"]);
fixture.scan(&[], 0, 0, &|| Ok(())).unwrap();
assert_eq!(fixture.ids(), ["other"]);
}
#[test]
fn concurrent_source_edit_aborts_native_swap_without_overwriting_other_writer() {
let fixture = Fixture::new();
fixture.add("old", "/music/old.flac", "source");
let writer = Connection::open(fixture.request["library_path"].as_str().unwrap()).unwrap();
let changed = Cell::new(false);
let check = || {
if !changed.replace(true) {
writer
.execute(
"UPDATE library_sources SET path='/changed' WHERE id='source'",
[],
)
.unwrap();
}
Ok(())
};
assert!(
fixture
.scan(&[track("new", "/music/new.flac")], 1, 0, &check)
.is_err()
);
assert_eq!(fixture.ids(), ["old"]);
assert_eq!(
fixture
.db
.query_row(
"SELECT path FROM library_sources WHERE id='source'",
[],
|row| row.get::<_, String>(0)
)
.unwrap(),
"/changed"
);
}
#[test]
fn incremental_deletes_only_its_source_and_reconciles_existing_downloads() {
let fixture = Fixture::new();
fixture.add("old", "/music/old.flac", "source");
fixture.add("download", "/music/download.flac", "source");
fixture.add("other", "/other/track.flac", "other");
let history = Connection::open(fixture.request["history_path"].as_str().unwrap()).unwrap();
history
.execute(
"INSERT INTO history_path_keys VALUES('h','/music/download')",
[],
)
.unwrap();
let mut request = fixture.request.clone();
request["operation"] = "library_incremental_import".into();
request["delta"] = json!({"scanned":[track("new","/music/new.flac")],"deletedPaths":["/music/old.flac","/other/track.flac"],"skippedCount":10,"totalFiles":11});
let result = execute(&request, &|| Ok(())).unwrap();
assert_eq!(fixture.ids(), ["new", "other"]);
assert_eq!(result["skipped"], 1);
assert_eq!(result["deleted_count"], 1);
}
#[test]
fn snapshot_keeps_legacy_sentinel_and_backfills_only_current_missing_timestamps() {
let fixture = Fixture::new();
fixture.add("legacy", "/music/legacy.flac", "source");
fixture.add("saf", "content://provider/document/song.flac", "source");
fixture.add("current", "/music/current.flac", "source");
fixture
.db
.execute(
"UPDATE library SET file_mod_time=0,audio_metadata_scan_version=3 WHERE id='legacy'",
[],
)
.unwrap();
fixture
.db
.execute("UPDATE library SET file_mod_time=0 WHERE id='saf'", [])
.unwrap();
let mut request = fixture.request.clone();
request["operation"] = "library_snapshot".into();
request["snapshot_path"] = fixture
.directory
.path()
.join("snapshot.tsv")
.to_str()
.unwrap()
.into();
request["backfilled_mod_times"] = json!({"content://provider/document/song.flac":1700000000000_i64,"/music/legacy.flac":1900000000000_i64});
let result = execute(&request, &|| Ok(())).unwrap();
assert_eq!(result["count"], 3);
assert_eq!(result["backfilled_count"], 1);
let snapshot = fs::read_to_string(request["snapshot_path"].as_str().unwrap()).unwrap();
assert!(snapshot.contains("-1\t/music/legacy.flac\n"));
assert!(snapshot.contains("1700000000000\tcontent://provider/document/song.flac\n"));
assert_eq!(
fixture
.db
.query_row(
"SELECT file_mod_time FROM library WHERE id='legacy'",
[],
|row| row.get::<_, i64>(0)
)
.unwrap(),
0
);
assert!(
execute(&request, &|| Ok(())).is_err(),
"snapshot must not overwrite an existing output"
);
}
#[test]
fn mapper_preserves_null_artist_flags_and_replaygain_contract() {
let mut row = track("full", "/music/full.flac");
row["explicit"] = json!(1);
row["albumArtist"] = json!("");
row["metadataFromFilename"] = json!(true);
let mapped = encode(&row, "nas").unwrap();
assert_eq!(mapped.len(), 39);
assert_eq!(mapped["explicit"], 1);
assert_eq!(mapped["has_replaygain"], 1);
assert_eq!(mapped["audio_metadata_scan_version"], 0);
assert_eq!(mapped["album_artist_norm"], "");
assert_eq!(mapped["search_text"], "track full artist album artist");
}
+991
View File
@@ -0,0 +1,991 @@
{
"rows": [
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-06T12:00:00.000Z"
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-06T12:00:00.000Z",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 0,
"audio_metadata_scan_version": 4,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "artist",
"match_key": "café i 💫|artist",
"album_key": "album|artist",
"search_text": "café i 💫 artist album artist",
"sort_genre": "",
"sort_release": "",
"sort_added": 1791288000000
}
},
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-06T12:00:00.000Z",
"albumArtist": ""
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": "",
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-06T12:00:00.000Z",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 0,
"audio_metadata_scan_version": 4,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "",
"match_key": "café i 💫|artist",
"album_key": "album|",
"search_text": "café i 💫 artist album artist",
"sort_genre": "",
"sort_release": "",
"sort_added": 1791288000000
}
},
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-06T12:00:00.000Z",
"albumArtist": " Other ",
"explicit": 1.0,
"hasLyrics": true,
"hasReplayGain": 1
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": " Other ",
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-06T12:00:00.000Z",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 1,
"has_lyrics": 1,
"has_replaygain": 1,
"audio_metadata_scan_version": 4,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "other",
"match_key": "café i 💫|artist",
"album_key": "album|other",
"search_text": "café i 💫 artist album other",
"sort_genre": "",
"sort_release": "",
"sort_added": 1791288000000
}
},
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-06T12:00:00.000Z",
"metadataFromFilename": true,
"fileModTime": 0,
"genre": " Rock ",
"releaseDate": " 2026-10 "
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-06T12:00:00.000Z",
"file_mod_time": 0,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": " 2026-10 ",
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": " Rock ",
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 0,
"audio_metadata_scan_version": 0,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "artist",
"match_key": "café i 💫|artist",
"album_key": "album|artist",
"search_text": "café i 💫 artist album artist",
"sort_genre": "rock",
"sort_release": "2026-10",
"sort_added": 0
}
},
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-06T12:00:00.000Z",
"audioMetadataScanVersion": 2.9,
"fileModTime": 1770000000000.9,
"replaygain_track_gain": " -4.5 DB "
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-06T12:00:00.000Z",
"file_mod_time": 1770000000000.9,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 1,
"audio_metadata_scan_version": 2,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "artist",
"match_key": "café i 💫|artist",
"album_key": "album|artist",
"search_text": "café i 💫 artist album artist",
"sort_genre": "",
"sort_release": "",
"sort_added": 1770000000000
}
},
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "invalid",
"replaygain_album_gain": "NaN"
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "invalid",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 0,
"audio_metadata_scan_version": 4,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "artist",
"match_key": "café i 💫|artist",
"album_key": "album|artist",
"search_text": "café i 💫 artist album artist",
"sort_genre": "",
"sort_release": "",
"sort_added": 0
}
},
{
"input": {
"id": "row",
"trackName": "",
"artistName": "",
"albumName": "",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-06T12:00:00.000Z",
"replaygain_album_gain": 2.5
},
"expected": {
"id": "row",
"track_name": "",
"artist_name": "",
"album_name": "",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-06T12:00:00.000Z",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 1,
"audio_metadata_scan_version": 4,
"track_name_norm": "",
"artist_name_norm": "",
"album_name_norm": "",
"album_artist_norm": "",
"match_key": "|",
"album_key": "|",
"search_text": "",
"sort_genre": "",
"sort_release": "",
"sort_added": 1791288000000
}
},
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-06T19:00:00.123456+07:00",
"explicit": false,
"hasLyrics": 0
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-06T19:00:00.123456+07:00",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 0,
"audio_metadata_scan_version": 4,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "artist",
"match_key": "café i 💫|artist",
"album_key": "album|artist",
"search_text": "café i 💫 artist album artist",
"sort_genre": "",
"sort_release": "",
"sort_added": 1791288000123
}
},
{
"input": {
"id": "row",
"trackName": " CAFÉ İ 💫 ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "20261006T120000.123456789z"
},
"expected": {
"id": "row",
"track_name": " CAFÉ İ 💫 ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "20261006T120000.123456789z",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 0,
"audio_metadata_scan_version": 4,
"track_name_norm": "café i 💫",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "artist",
"match_key": "café i 💫|artist",
"album_key": "album|artist",
"search_text": "café i 💫 artist album artist",
"sort_genre": "",
"sort_release": "",
"sort_added": 1791288000123
}
},
{
"input": {
"id": "row",
"trackName": "ΑΒΣ İ",
"artistName": "Artist",
"albumName": "Album",
"filePath": "/music/a.flac",
"scannedAt": "2026-10-32T25:61:00Z"
},
"expected": {
"id": "row",
"track_name": "ΑΒΣ İ",
"artist_name": "Artist",
"album_name": "Album",
"album_artist": null,
"file_path": "/music/a.flac",
"cover_path": null,
"scanned_at": "2026-10-32T25:61:00Z",
"file_mod_time": null,
"isrc": null,
"track_number": null,
"total_tracks": null,
"disc_number": null,
"total_discs": null,
"duration": null,
"release_date": null,
"bit_depth": null,
"sample_rate": null,
"bitrate": null,
"genre": null,
"composer": null,
"label": null,
"copyright": null,
"format": null,
"source_id": "source",
"explicit": 0,
"has_lyrics": 0,
"has_replaygain": 0,
"audio_metadata_scan_version": 4,
"track_name_norm": "αβσ i",
"artist_name_norm": "artist",
"album_name_norm": "album",
"album_artist_norm": "artist",
"match_key": "αβσ i|artist",
"album_key": "album|artist",
"search_text": "αβσ i artist album artist",
"sort_genre": "",
"sort_release": "",
"sort_added": 1793584860000
}
}
],
"paths": [
{
"input": "",
"expected": []
},
{
"input": " EXISTS: /Music/Café 01.FLAC ",
"expected": [
"/Music/Café 01",
"/Music/Café 01.FLAC",
"/music/café 01",
"/music/café 01.flac",
"file:///Music/Caf%C3%A9%2001",
"file:///Music/Caf%C3%A9%2001.FLAC",
"file:///music/caf%c3%a9%2001",
"file:///music/caf%c3%a9%2001.flac"
]
},
{
"input": "/a/../b/song.flac",
"expected": [
"/a/../b/song",
"/a/../b/song.flac",
"file:///b/song",
"file:///b/song.flac"
]
},
{
"input": "/Music/100%25/song.opus",
"expected": [
"/Music/100%/song",
"/Music/100%/song.opus",
"/Music/100%25/song",
"/Music/100%25/song.opus",
"/music/100%/song",
"/music/100%/song.opus",
"/music/100%25/song",
"/music/100%25/song.opus",
"file:///Music/100%25/song",
"file:///Music/100%25/song.opus",
"file:///Music/100%2525/song",
"file:///Music/100%2525/song.opus",
"file:///music/100%25/song",
"file:///music/100%25/song.opus",
"file:///music/100%2525/song",
"file:///music/100%2525/song.opus"
]
},
{
"input": "/Music/100%zz.flac",
"expected": [
"/Music/100%zz",
"/Music/100%zz.flac",
"/music/100%zz",
"/music/100%zz.flac",
"file:///Music/100%25zz",
"file:///Music/100%25zz.flac",
"file:///music/100%25zz",
"file:///music/100%25zz.flac"
]
},
{
"input": "/Music/💫 track.wav",
"expected": [
"/Music/💫 track",
"/Music/💫 track.wav",
"/music/💫 track",
"/music/💫 track.wav",
"file:///Music/%F0%9F%92%AB%20track",
"file:///Music/%F0%9F%92%AB%20track.wav",
"file:///music/%f0%9f%92%ab%20track",
"file:///music/%f0%9f%92%ab%20track.wav"
]
},
{
"input": "/Music/a#b.flac",
"expected": [
"/Music/a#b",
"/Music/a#b.flac",
"/music/a#b",
"/music/a#b.flac",
"file:///Music/a%23b",
"file:///Music/a%23b.flac",
"file:///music/a%23b",
"file:///music/a%23b.flac"
]
},
{
"input": "/Music/a?b.mp3",
"expected": [
"/Music/a?b",
"/Music/a?b.mp3",
"/music/a?b",
"/music/a?b.mp3",
"file:///Music/a%3Fb",
"file:///Music/a%3Fb.mp3",
"file:///music/a%3fb",
"file:///music/a%3fb.mp3"
]
},
{
"input": "/Music/a\\b.flac",
"expected": [
"/Music/a/b",
"/Music/a/b.flac",
"/Music/a\\b",
"/Music/a\\b.flac",
"/music/a/b",
"/music/a/b.flac",
"/music/a\\b",
"/music/a\\b.flac",
"file:///Music/a%5Cb",
"file:///Music/a%5Cb.flac",
"file:///Music/a/b",
"file:///Music/a/b.flac",
"file:///music/a%5cb",
"file:///music/a%5cb.flac",
"file:///music/a/b",
"file:///music/a/b.flac"
]
},
{
"input": "C:\\Music\\Hello.flac",
"expected": [
"C:/Music/Hello",
"C:/Music/Hello.flac",
"C:\\Music\\Hello",
"C:\\Music\\Hello.flac",
"c:/Music/Hello",
"c:/Music/Hello.flac",
"c:/music/hello",
"c:/music/hello.flac",
"c:\\music\\hello",
"c:\\music\\hello.flac"
]
},
{
"input": "file:///music/a%20b.flac",
"expected": [
"/music/a b",
"/music/a b.flac",
"file:///music/a b",
"file:///music/a b.flac",
"file:///music/a%20b",
"file:///music/a%20b.flac"
]
},
{
"input": "file://localhost/music/a.flac",
"expected": [
"file://localhost/music/a",
"file://localhost/music/a.flac"
]
},
{
"input": "file://localhost",
"expected": [
"file://localhost",
"file://localhost/"
]
},
{
"input": "file:///music/a.flac?x=1#frag",
"expected": [
"file:///music/a.flac?x=1#frag"
]
},
{
"input": "file:///music/a.flac#frag",
"expected": [
"file:///music/a.flac#frag"
]
},
{
"input": "file:///a/../b/song.flac",
"expected": [
"/b/song",
"/b/song.flac",
"file:///a/../b/song",
"file:///a/../b/song.flac",
"file:///b/song",
"file:///b/song.flac"
]
},
{
"input": "file:///music/%7e.flac",
"expected": [
"/music/~",
"/music/~.flac",
"file:///music/%7e",
"file:///music/%7e.flac",
"file:///music/~",
"file:///music/~.flac"
]
},
{
"input": "content://nas.provider/document/Ab%252FC.flac",
"expected": [
"content://nas.provider/document/Ab%252FC",
"content://nas.provider/document/Ab%252FC.flac",
"content://nas.provider/document/Ab%2FC",
"content://nas.provider/document/Ab%2FC.flac",
"content://nas.provider/document/Ab/C",
"content://nas.provider/document/Ab/C.flac",
"content://nas.provider/document/ab%252fc",
"content://nas.provider/document/ab%252fc.flac",
"content://nas.provider/document/ab%2fc",
"content://nas.provider/document/ab%2fc.flac",
"content://nas.provider/document/ab/c",
"content://nas.provider/document/ab/c.flac",
"saf-document:bmFzLnByb3ZpZGVy:Ab%2FC",
"saf-document:bmFzLnByb3ZpZGVy:Ab%2FC.flac"
]
},
{
"input": "content://nas.provider/tree/root/document/Ab%252FC.flac",
"expected": [
"content://nas.provider/tree/root/document/Ab%252FC",
"content://nas.provider/tree/root/document/Ab%252FC.flac",
"content://nas.provider/tree/root/document/Ab%2FC",
"content://nas.provider/tree/root/document/Ab%2FC.flac",
"content://nas.provider/tree/root/document/Ab/C",
"content://nas.provider/tree/root/document/Ab/C.flac",
"content://nas.provider/tree/root/document/ab%252fc",
"content://nas.provider/tree/root/document/ab%252fc.flac",
"content://nas.provider/tree/root/document/ab%2fc",
"content://nas.provider/tree/root/document/ab%2fc.flac",
"content://nas.provider/tree/root/document/ab/c",
"content://nas.provider/tree/root/document/ab/c.flac",
"saf-document:bmFzLnByb3ZpZGVy:Ab%2FC",
"saf-document:bmFzLnByb3ZpZGVy:Ab%2FC.flac"
]
},
{
"input": "content://NAS.PROVIDER/document/Ab%2FC.flac?x=Y#frag",
"expected": [
"content://NAS.PROVIDER/document/Ab%2FC.flac?x=Y#frag",
"content://NAS.PROVIDER/document/Ab/C.flac?x=Y#frag",
"content://nas.provider/document/Ab%2FC.flac?x=Y#frag",
"content://nas.provider/document/Ab/C.flac?x=Y#frag",
"content://nas.provider/document/ab%2fc.flac?x=y#frag",
"content://nas.provider/document/ab/c.flac?x=y#frag",
"saf-document:bmFzLnByb3ZpZGVy:Ab/C",
"saf-document:bmFzLnByb3ZpZGVy:Ab/C.flac"
]
},
{
"input": "content://nas.provider/document/AB%20C.flac",
"expected": [
"content://nas.provider/document/AB C",
"content://nas.provider/document/AB C.flac",
"content://nas.provider/document/AB%20C",
"content://nas.provider/document/AB%20C.flac",
"content://nas.provider/document/ab c",
"content://nas.provider/document/ab c.flac",
"content://nas.provider/document/ab%20c",
"content://nas.provider/document/ab%20c.flac",
"saf-document:bmFzLnByb3ZpZGVy:AB C",
"saf-document:bmFzLnByb3ZpZGVy:AB C.flac"
]
},
{
"input": "content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9.flac",
"expected": [
"/mnt/sdcard/Music/Café",
"/mnt/sdcard/Music/Café.flac",
"/mnt/sdcard/music/café",
"/mnt/sdcard/music/café.flac",
"/sdcard/Music/Café",
"/sdcard/Music/Café.flac",
"/sdcard/music/café",
"/sdcard/music/café.flac",
"/storage/emulated/0/Music/Café",
"/storage/emulated/0/Music/Café.flac",
"/storage/emulated/0/music/café",
"/storage/emulated/0/music/café.flac",
"/storage/emulated/legacy/Music/Café",
"/storage/emulated/legacy/Music/Café.flac",
"/storage/emulated/legacy/music/café",
"/storage/emulated/legacy/music/café.flac",
"/storage/self/primary/Music/Café",
"/storage/self/primary/Music/Café.flac",
"/storage/self/primary/music/café",
"/storage/self/primary/music/café.flac",
"content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9",
"content://com.android.externalstorage.documents/tree/primary%3AMusic/document/primary%3AMusic%2FCaf%C3%A9.flac",
"content://com.android.externalstorage.documents/tree/primary%3amusic/document/primary%3amusic%2fcaf%c3%a9",
"content://com.android.externalstorage.documents/tree/primary%3amusic/document/primary%3amusic%2fcaf%c3%a9.flac",
"content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Caf%C3%A9",
"content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Caf%C3%A9.flac",
"content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Café",
"content://com.android.externalstorage.documents/tree/primary:Music/document/primary:Music/Café.flac",
"content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/caf%c3%a9",
"content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/caf%c3%a9.flac",
"content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/café",
"content://com.android.externalstorage.documents/tree/primary:music/document/primary:music/café.flac",
"file:///mnt/sdcard/Music/Caf%C3%A9",
"file:///mnt/sdcard/Music/Caf%C3%A9.flac",
"file:///mnt/sdcard/music/caf%c3%a9",
"file:///mnt/sdcard/music/caf%c3%a9.flac",
"file:///sdcard/Music/Caf%C3%A9",
"file:///sdcard/Music/Caf%C3%A9.flac",
"file:///sdcard/music/caf%c3%a9",
"file:///sdcard/music/caf%c3%a9.flac",
"file:///storage/emulated/0/Music/Caf%C3%A9",
"file:///storage/emulated/0/Music/Caf%C3%A9.flac",
"file:///storage/emulated/0/music/caf%c3%a9",
"file:///storage/emulated/0/music/caf%c3%a9.flac",
"file:///storage/emulated/legacy/Music/Caf%C3%A9",
"file:///storage/emulated/legacy/Music/Caf%C3%A9.flac",
"file:///storage/emulated/legacy/music/caf%c3%a9",
"file:///storage/emulated/legacy/music/caf%c3%a9.flac",
"file:///storage/self/primary/Music/Caf%C3%A9",
"file:///storage/self/primary/Music/Caf%C3%A9.flac",
"file:///storage/self/primary/music/caf%c3%a9",
"file:///storage/self/primary/music/caf%c3%a9.flac",
"saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/Café",
"saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/Café.flac"
]
},
{
"input": "content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary%3AMusic%2Fsong.opus",
"expected": [
"/mnt/sdcard/Music/song",
"/mnt/sdcard/Music/song.opus",
"/mnt/sdcard/music/song",
"/mnt/sdcard/music/song.opus",
"/sdcard/Music/song",
"/sdcard/Music/song.opus",
"/sdcard/music/song",
"/sdcard/music/song.opus",
"/storage/emulated/0/Music/song",
"/storage/emulated/0/Music/song.opus",
"/storage/emulated/0/music/song",
"/storage/emulated/0/music/song.opus",
"/storage/emulated/legacy/Music/song",
"/storage/emulated/legacy/Music/song.opus",
"/storage/emulated/legacy/music/song",
"/storage/emulated/legacy/music/song.opus",
"/storage/self/primary/Music/song",
"/storage/self/primary/Music/song.opus",
"/storage/self/primary/music/song",
"/storage/self/primary/music/song.opus",
"content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary%3AMusic%2Fsong",
"content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary%3AMusic%2Fsong.opus",
"content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary:Music/song",
"content://COM.ANDROID.EXTERNALSTORAGE.DOCUMENTS/document/primary:Music/song.opus",
"content://com.android.externalstorage.documents/document/primary%3AMusic%2Fsong",
"content://com.android.externalstorage.documents/document/primary%3AMusic%2Fsong.opus",
"content://com.android.externalstorage.documents/document/primary%3amusic%2fsong",
"content://com.android.externalstorage.documents/document/primary%3amusic%2fsong.opus",
"content://com.android.externalstorage.documents/document/primary:Music/song",
"content://com.android.externalstorage.documents/document/primary:Music/song.opus",
"content://com.android.externalstorage.documents/document/primary:music/song",
"content://com.android.externalstorage.documents/document/primary:music/song.opus",
"file:///mnt/sdcard/Music/song",
"file:///mnt/sdcard/Music/song.opus",
"file:///mnt/sdcard/music/song",
"file:///mnt/sdcard/music/song.opus",
"file:///sdcard/Music/song",
"file:///sdcard/Music/song.opus",
"file:///sdcard/music/song",
"file:///sdcard/music/song.opus",
"file:///storage/emulated/0/Music/song",
"file:///storage/emulated/0/Music/song.opus",
"file:///storage/emulated/0/music/song",
"file:///storage/emulated/0/music/song.opus",
"file:///storage/emulated/legacy/Music/song",
"file:///storage/emulated/legacy/Music/song.opus",
"file:///storage/emulated/legacy/music/song",
"file:///storage/emulated/legacy/music/song.opus",
"file:///storage/self/primary/Music/song",
"file:///storage/self/primary/Music/song.opus",
"file:///storage/self/primary/music/song",
"file:///storage/self/primary/music/song.opus",
"saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/song",
"saf-document:Y29tLmFuZHJvaWQuZXh0ZXJuYWxzdG9yYWdlLmRvY3VtZW50cw==:primary:Music/song.opus"
]
},
{
"input": "network://Connection/Music/Track%2520A.flac",
"expected": [
"network://Connection/Music/Track A",
"network://Connection/Music/Track A.flac",
"network://Connection/Music/Track%20A",
"network://Connection/Music/Track%20A.flac",
"network://Connection/Music/Track%2520A",
"network://Connection/Music/Track%2520A.flac",
"network://connection/Music/Track%20A",
"network://connection/Music/Track%20A.flac",
"network://connection/Music/Track%2520A",
"network://connection/Music/Track%2520A.flac",
"network://connection/music/track a",
"network://connection/music/track a.flac",
"network://connection/music/track%20a",
"network://connection/music/track%20a.flac",
"network://connection/music/track%2520a",
"network://connection/music/track%2520a.flac"
]
},
{
"input": "https://EXAMPLE.com/Music/a%7eb.flac?q=x#Y",
"expected": [
"https://EXAMPLE.com/Music/a%7eb.flac?q=x#Y",
"https://EXAMPLE.com/Music/a~b.flac?q=x#Y",
"https://example.com/Music/a~b.flac?q=x#Y",
"https://example.com/music/a%7eb.flac?q=x#y",
"https://example.com/music/a~b.flac?q=x#y"
]
},
{
"input": "content://provider/document/💫.flac",
"expected": [
"content://provider/document/%F0%9F%92%AB",
"content://provider/document/%F0%9F%92%AB.flac",
"content://provider/document/%f0%9f%92%ab",
"content://provider/document/%f0%9f%92%ab.flac",
"content://provider/document/💫",
"content://provider/document/💫.flac",
"saf-document:cHJvdmlkZXI=:💫",
"saf-document:cHJvdmlkZXI=:💫.flac"
]
},
{
"input": "content://provider/document/a%zz.flac",
"expected": [
"content://provider/document/a%25zz",
"content://provider/document/a%25zz.flac",
"content://provider/document/a%zz",
"content://provider/document/a%zz.flac",
"saf-document:cHJvdmlkZXI=:a%zz",
"saf-document:cHJvdmlkZXI=:a%zz.flac"
]
},
{
"input": "/sdcard/Music/song.flac",
"expected": [
"/sdcard/Music/song",
"/sdcard/Music/song.flac",
"/sdcard/music/song",
"/sdcard/music/song.flac",
"file:///sdcard/Music/song",
"file:///sdcard/Music/song.flac",
"file:///sdcard/music/song",
"file:///sdcard/music/song.flac"
]
},
{
"input": "/storage/emulated/0/Music/song.opus",
"expected": [
"/storage/emulated/0/Music/song",
"/storage/emulated/0/Music/song.opus",
"/storage/emulated/0/music/song",
"/storage/emulated/0/music/song.opus",
"file:///storage/emulated/0/Music/song",
"file:///storage/emulated/0/Music/song.opus",
"file:///storage/emulated/0/music/song",
"file:///storage/emulated/0/music/song.opus"
]
}
]
}
+62
View File
@@ -10,6 +10,7 @@ import 'package:spotiflac_android/models/settings.dart';
import 'package:spotiflac_android/providers/local_library_provider.dart';
import 'package:spotiflac_android/providers/settings_provider.dart';
import 'package:spotiflac_android/services/library_database.dart';
import 'package:spotiflac_android/services/platform_bridge.dart';
class _Settings extends SettingsNotifier {
@override
@@ -25,6 +26,8 @@ class _Database implements LibraryDatabase {
List<LocalLibrarySource> sources;
Future<List<LocalLibrarySource>> Function()? sourceRead;
bool failLookup = false;
int? countOverride;
Future<({int inserted, int skipped})> Function()? fullReplacement;
@override
Future<List<LocalLibrarySource>> getSources() async =>
@@ -38,8 +41,17 @@ class _Database implements LibraryDatabase {
@override
Future<int> getCount() async =>
countOverride ??
sources.where((source) => source.enabled && source.available).length;
@override
Future<({int inserted, int skipped})> replaceSourceScanFile(
String sourceId,
LibraryScanNDJSONFile scan, {
required String requestId,
required bool Function() isCancelled,
}) => fullReplacement!();
@override
Future<LocalLibraryLookupIndex> getLookupIndex() async =>
const LocalLibraryLookupIndex();
@@ -328,4 +340,54 @@ void main() {
await retry;
expect(container.read(localLibraryProvider).isScanning, isFalse);
});
test(
'cancel after incremental native commit publishes the changed index',
() async {
final version = container.read(localLibraryProvider).loadedIndexVersion;
final active = library.startSourceScan('active');
await _waitFor(() => scans.length == 1);
await library.cancelScan();
database.countOverride = 7;
scans.single.$2.complete(jsonEncode({'committed': true, 'skipped': 2}));
await active;
final state = container.read(localLibraryProvider);
expect(state.totalCount, 7);
expect(state.loadedIndexVersion, greaterThan(version));
expect(state.excludedDownloadedCount, 2);
expect(state.scanWasCancelled, isTrue);
expect(state.isScanning, isFalse);
},
);
test('cancel after full replacement publishes the committed index', () async {
final entered = Completer<void>();
final release = Completer<void>();
database.fullReplacement = () async {
entered.complete();
await release.future;
database.countOverride = 7;
return (inserted: 7, skipped: 2);
};
messenger.setMockMethodCallHandler(backend, (call) async {
if (call.method == 'scanLibraryFolderToNDJSONFile') {
final path = (call.arguments as Map)['output_path'] as String;
await File(path).writeAsString('');
return {'path': path, 'count': 0, 'error_count': 0};
}
return null;
});
final version = container.read(localLibraryProvider).loadedIndexVersion;
final active = library.startSourceScan('active', forceFullScan: true);
await entered.future;
await library.cancelScan();
release.complete();
await active;
final state = container.read(localLibraryProvider);
expect(state.totalCount, 7);
expect(state.loadedIndexVersion, greaterThan(version));
expect(state.excludedDownloadedCount, 2);
expect(state.scanWasCancelled, isTrue);
expect(state.isScanning, isFalse);
});
}
+39
View File
@@ -0,0 +1,39 @@
import 'dart:convert';
import 'dart:io';
import 'package:flutter_test/flutter_test.dart';
import 'package:spotiflac_android/services/library_row_mapper.dart';
import 'package:spotiflac_android/utils/path_match_keys.dart';
void main() {
test(
'native library fixtures preserve production Dart row and path codecs',
() {
final fixture =
jsonDecode(
File(
'test/fixtures/native_library_contract.json',
).readAsStringSync(),
)
as Map<String, dynamic>;
for (final entry in fixture['rows'] as List) {
final value = entry as Map<String, dynamic>;
expect(
LibraryRowMapper.encode(
Map<String, dynamic>.from(value['input'] as Map),
sourceId: 'source',
),
value['expected'],
);
}
for (final entry in fixture['paths'] as List) {
final value = entry as Map<String, dynamic>;
expect(
buildPathMatchKeys(value['input'] as String).toList()..sort(),
value['expected'],
reason: value['input'] as String,
);
}
},
);
}