From 7fc27e8314f5d6c1d2a7f1fc34c19335178ac054 Mon Sep 17 00:00:00 2001 From: zarzet Date: Tue, 18 Aug 2026 11:03:05 +0700 Subject: [PATCH] perf(library): stream scans and optimize queue queries --- .../kotlin/com/zarz/spotiflac/MainActivity.kt | 25 ++ .../com/zarz/spotiflac/MainActivitySafScan.kt | 178 ++++++--- .../zarz/spotiflac/NativeDownloadFinalizer.kt | 102 ++++- go_backend/exports_library.go | 4 + go_backend/httputil_utls.go | 102 ++++- go_backend/httputil_utls_pool_test.go | 187 +++++++++ go_backend/library_scan.go | 198 ++++++++-- go_backend/library_scan_supplement_test.go | 59 +++ ios/Runner/AppDelegate.swift | 76 +++- lib/providers/download_history_models.dart | 4 +- lib/providers/download_queue_provider.dart | 27 +- .../library_collections_provider.dart | 115 +++++- lib/providers/local_library_provider.dart | 134 +++++-- lib/screens/queue_tab.dart | 112 ++++-- lib/screens/queue_tab_helpers.dart | 22 +- lib/screens/track_metadata_edit_sheet.dart | 13 + lib/services/conversion_library_service.dart | 2 +- lib/services/cover_download_service.dart | 10 +- lib/services/history_database.dart | 148 +++++++- lib/services/library_database.dart | 303 ++++++++++++++- lib/services/library_database_models.dart | 36 ++ lib/services/library_database_queue_sql.dart | 359 +++++++++++++++--- lib/services/platform_bridge.dart | 212 ++++++++++- lib/widgets/cross_extension_share_sheet.dart | 7 +- lib/widgets/extension_row.dart | 6 + lib/widgets/playlist_picker_sheet.dart | 11 +- test/download_history_logic_test.dart | 10 + test/platform_bridge_metadata_test.dart | 21 + 28 files changed, 2145 insertions(+), 338 deletions(-) create mode 100644 go_backend/httputil_utls_pool_test.go diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt index 199eec09..5ace7826 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt @@ -2165,6 +2165,23 @@ class MainActivity: FlutterFragmentActivity() { } result.success(response) } + "scanLibraryFolderToNDJSONFile" -> { + val folderPath = call.argument("folder_path") ?: "" + val outputPath = call.argument("output_path") ?: "" + val count = withContext(Dispatchers.IO) { + safScanActive = false + Gobackend.scanLibraryFolderToNDJSONFileJSON( + folderPath, + outputPath, + ) + } + result.success( + mapOf( + "path" to outputPath, + "count" to count, + ) + ) + } "scanLibraryFolderIncremental" -> { val folderPath = call.argument("folder_path") ?: "" val existingFiles = call.argument("existing_files") ?: "{}" @@ -2197,6 +2214,14 @@ class MainActivity: FlutterFragmentActivity() { } result.success(response) } + "scanSafTreeToNDJSONFile" -> { + val treeUri = call.argument("tree_uri") ?: "" + val outputPath = call.argument("output_path") ?: "" + val response = withContext(Dispatchers.IO) { + scanSafTree(treeUri, outputPath) + } + result.success(response) + } "scanSafTreeIncremental" -> { val treeUri = call.argument("tree_uri") ?: "" val existingFiles = call.argument("existing_files") ?: "{}" diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt index cb8def30..0178bc0e 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt @@ -3,9 +3,11 @@ package com.zarz.spotiflac import android.app.Activity import android.content.Context import android.content.Intent +import android.content.pm.PackageManager import android.net.Uri import android.os.Build import android.os.Bundle +import android.provider.DocumentsContract import androidx.activity.OnBackPressedCallback import androidx.activity.result.contract.ActivityResultContracts import androidx.documentfile.provider.DocumentFile @@ -34,6 +36,7 @@ import org.json.JSONTokener import java.io.File import java.io.FileInputStream import java.io.FileOutputStream +import java.io.IOException import java.security.MessageDigest import java.util.Locale @@ -460,21 +463,13 @@ internal fun MainActivity.getSafChildFileLookup( ): Map { val dirKey = dir.uri.toString() return cache.getOrPut(dirKey) { - try { - buildMap { - for (child in dir.listFiles()) { - if (!child.isFile) continue - val childName = child.name?.trim().orEmpty() - if (childName.isBlank()) continue - put(childName.lowercase(Locale.ROOT), child) - } + buildMap { + for (child in listSafChildrenOrThrow(dir)) { + if (!child.isFile) continue + val childName = child.name?.trim().orEmpty() + if (childName.isBlank()) continue + put(childName.lowercase(Locale.ROOT), child) } - } catch (e: Exception) { - android.util.Log.w( - "SpotiFLAC", - "Failed to build SAF child lookup for $dirKey: ${e.message}", - ) - emptyMap() } } } @@ -510,11 +505,80 @@ internal fun MainActivity.resolveCueAudioSibling( return null } -internal fun MainActivity.scanSafTree(treeUriStr: String): Any { - if (treeUriStr.isBlank()) return "[]" +internal fun MainActivity.listSafChildrenOrThrow(dir: DocumentFile): List { + val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree( + dir.uri, + DocumentsContract.getDocumentId(dir.uri), + ) + val cursor = contentResolver.query( + childrenUri, + arrayOf(DocumentsContract.Document.COLUMN_DOCUMENT_ID), + null, + null, + null, + ) ?: throw IOException("SAF provider returned no cursor for ${dir.uri}") + return cursor.use { + val documentIdIndex = it.getColumnIndexOrThrow( + DocumentsContract.Document.COLUMN_DOCUMENT_ID, + ) + buildList { + while (it.moveToNext()) { + val childUri = DocumentsContract.buildDocumentUriUsingTree( + dir.uri, + it.getString(documentIdIndex), + ) + val child = DocumentFile.fromSingleUri(this@listSafChildrenOrThrow, childUri) + ?: throw IOException("Invalid SAF child URI: $childUri") + add(child) + } + } + } + } +internal fun MainActivity.resolveReadableSafTreeOrThrow( + treeUriStr: String, + ): Pair { + if (treeUriStr.isBlank()) { + throw IllegalArgumentException("SAF tree URI is empty") + } val treeUri = Uri.parse(treeUriStr) - val root = DocumentFile.fromTreeUri(this, treeUri) ?: return "[]" + val hasReadPermission = contentResolver.persistedUriPermissions.any { + it.uri == treeUri && it.isReadPermission + } || checkUriPermission( + treeUri, + android.os.Process.myPid(), + android.os.Process.myUid(), + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) == PackageManager.PERMISSION_GRANTED + if (!hasReadPermission) { + throw SecurityException("Read access to the SAF tree has been revoked") + } + val root = DocumentFile.fromTreeUri(this, treeUri) + ?: throw IOException("Unable to resolve SAF tree") + if (!root.exists() || !root.canRead()) { + throw IOException("SAF tree is unavailable or unreadable") + } + return treeUri to root + } + +internal fun MainActivity.scanSafTree( + treeUriStr: String, + ndjsonOutputPath: String? = null, + ): Any { + fun emptyResult(): Any { + if (ndjsonOutputPath == null) return "[]" + File(ndjsonOutputPath).writeText("", Charsets.UTF_8) + return mapOf("path" to ndjsonOutputPath, "count" to 0) + } + + fun cancelledResult(): Any { + updateSafScanProgress { it.isComplete = true } + if (ndjsonOutputPath == null) return "[]" + try { File(ndjsonOutputPath).delete() } catch (_: Exception) {} + throw java.util.concurrent.CancellationException("SAF library scan cancelled") + } + + val (_, root) = resolveReadableSafTreeOrThrow(treeUriStr) resetSafScanProgress() safScanCancel = false @@ -535,8 +599,7 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { while (queue.isNotEmpty()) { if (safScanCancel) { - updateSafScanProgress { it.isComplete = true } - return "[]" + return cancelledResult() } val (dir, path) = queue.removeFirst() @@ -546,7 +609,7 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { } val children = try { - dir.listFiles() + listSafChildrenOrThrow(dir) } catch (e: Exception) { traversalErrors++ updateSafScanProgress { it.errorCount = traversalErrors } @@ -559,8 +622,7 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { for (child in children) { if (safScanCancel) { - updateSafScanProgress { it.isComplete = true } - return "[]" + return cancelledResult() } try { @@ -592,6 +654,10 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { } } + if (traversalErrors > 0) { + throw IOException("SAF traversal failed for $traversalErrors entries") + } + val totalItems = audioFiles.size + cueFiles.size updateSafScanProgress { it.totalFiles = totalItems @@ -602,19 +668,28 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { it.isComplete = true it.progressPct = 100.0 } - return "[]" + return emptyResult() } // Stream results to a spill file: a full-library scan's JSONArray plus // its serialized string would otherwise hold the whole payload on the // Java heap several times over. - val spill = this.SpillJsonWriter() + val spill = if (ndjsonOutputPath == null) this.SpillJsonWriter() else null + val ndjsonWriter = ndjsonOutputPath?.let { + File(it).bufferedWriter(Charsets.UTF_8, 64 * 1024) + } var resultCount = 0 fun putResult(obj: JSONObject) { - spill.raw(if (resultCount == 0) "[" else ",") - spill.raw(obj.toString()) + if (ndjsonWriter != null) { + ndjsonWriter.write(obj.toString()) + ndjsonWriter.newLine() + } else { + spill!!.raw(if (resultCount == 0) "[" else ",") + spill.raw(obj.toString()) + } resultCount++ } + try { var scanned = 0 var errors = traversalErrors @@ -622,9 +697,9 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { for ((cueDoc, parentDir) in cueFiles) { if (safScanCancel) { - updateSafScanProgress { it.isComplete = true } - spill.abandon() - return "[]" + ndjsonWriter?.close() + spill?.abandon() + return cancelledResult() } val cueName = try { cueDoc.name ?: "" } catch (_: Exception) { "" } @@ -722,9 +797,9 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { for ((doc, _) in audioFiles) { if (safScanCancel) { - updateSafScanProgress { it.isComplete = true } - spill.abandon() - return "[]" + ndjsonWriter?.close() + spill?.abandon() + return cancelledResult() } if (cueReferencedAudioUris.contains(doc.uri.toString())) { @@ -780,8 +855,20 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any { it.progressPct = 100.0 } - spill.raw(if (resultCount == 0) "[]" else "]") + if (ndjsonWriter != null) { + ndjsonWriter.close() + return mapOf("path" to ndjsonOutputPath, "count" to resultCount) + } + spill!!.raw(if (resultCount == 0) "[]" else "]") return spill.result() + } catch (e: Exception) { + try { ndjsonWriter?.close() } catch (_: Exception) {} + spill?.abandon() + if (ndjsonOutputPath != null) { + try { File(ndjsonOutputPath).delete() } catch (_: Exception) {} + } + throw e + } } /** @@ -809,24 +896,7 @@ internal fun MainActivity.scanSafTreeIncremental( treeUriStr: String, existingFiles: Map, ): Any { - if (treeUriStr.isBlank()) { - val result = JSONObject() - result.put("files", JSONArray()) - result.put("removedUris", JSONArray()) - result.put("skippedCount", 0) - result.put("totalFiles", 0) - return result.toString() - } - - val treeUri = Uri.parse(treeUriStr) - val root = DocumentFile.fromTreeUri(this, treeUri) ?: run { - val result = JSONObject() - result.put("files", JSONArray()) - result.put("removedUris", JSONArray()) - result.put("skippedCount", 0) - result.put("totalFiles", 0) - return result.toString() - } + val (_, root) = resolveReadableSafTreeOrThrow(treeUriStr) resetSafScanProgress() safScanCancel = false @@ -875,7 +945,7 @@ internal fun MainActivity.scanSafTreeIncremental( } val children = try { - dir.listFiles() + listSafChildrenOrThrow(dir) } catch (e: Exception) { traversalErrors++ updateSafScanProgress { it.errorCount = traversalErrors } @@ -954,6 +1024,10 @@ internal fun MainActivity.scanSafTreeIncremental( } } + if (traversalErrors > 0) { + throw IOException("SAF traversal failed for $traversalErrors entries") + } + val removedUris = existingFiles.keys.filter { !currentUris.contains(it) } val totalFiles = currentUris.size val filesToProcess = audioFiles.size + cueFilesToScan.size diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt index d5ff58ac..6125d6c5 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt @@ -39,7 +39,7 @@ object NativeDownloadFinalizer { const val NATIVE_WORKER_CONTRACT_VERSION = 1 // Native finalizer owns background-safe history writes while Flutter may be suspended. // Keep this schema contract in sync with Dart HistoryDatabase before bumping either side. - const val HISTORY_SCHEMA_VERSION = 10 + const val HISTORY_SCHEMA_VERSION = 11 internal val activeFFmpegSessionIds = mutableSetOf() internal val nativeFFmpegSessionIds = mutableSetOf() internal val activeFFmpegSessionLock = Any() @@ -94,6 +94,13 @@ object NativeDownloadFinalizer { "match_key", "album_key", "search_text", + "sort_track", + "sort_artist", + "sort_album", + "sort_album_artist", + "sort_genre", + "sort_release", + "sort_added", ) private val androidStoragePathAliases = listOf( "/storage/emulated/0", @@ -1363,7 +1370,14 @@ object NativeDownloadFinalizer { isrc_norm TEXT, match_key TEXT, album_key TEXT, - search_text TEXT + search_text TEXT, + sort_track TEXT, + sort_artist TEXT, + sort_album TEXT, + sort_album_artist TEXT, + sort_genre TEXT, + sort_release TEXT, + sort_added INTEGER ) """.trimIndent() ) @@ -1382,6 +1396,13 @@ object NativeDownloadFinalizer { ensureHistoryColumn(db, "match_key", "ALTER TABLE history ADD COLUMN match_key TEXT") ensureHistoryColumn(db, "album_key", "ALTER TABLE history ADD COLUMN album_key TEXT") ensureHistoryColumn(db, "search_text", "ALTER TABLE history ADD COLUMN search_text TEXT") + ensureHistoryColumn(db, "sort_track", "ALTER TABLE history ADD COLUMN sort_track TEXT") + ensureHistoryColumn(db, "sort_artist", "ALTER TABLE history ADD COLUMN sort_artist TEXT") + ensureHistoryColumn(db, "sort_album", "ALTER TABLE history ADD COLUMN sort_album TEXT") + ensureHistoryColumn(db, "sort_album_artist", "ALTER TABLE history ADD COLUMN sort_album_artist TEXT") + ensureHistoryColumn(db, "sort_genre", "ALTER TABLE history ADD COLUMN sort_genre TEXT") + ensureHistoryColumn(db, "sort_release", "ALTER TABLE history ADD COLUMN sort_release TEXT") + ensureHistoryColumn(db, "sort_added", "ALTER TABLE history ADD COLUMN sort_added INTEGER") ensureHistoryPathKeyTable(db) if (needsBackfill) { backfillNormalizedHistoryColumns(db) @@ -1397,6 +1418,12 @@ object NativeDownloadFinalizer { db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_isrc_norm ON history(isrc_norm)") db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_match_key ON history(match_key)") db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_album_key ON history(album_key)") + db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_added ON history(sort_added DESC, sort_track, id)") + db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_track ON history(sort_track, sort_artist, id)") + db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_artist ON history(sort_artist, sort_track, id)") + db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_album ON history(sort_album, sort_track, id)") + db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_genre ON history(sort_genre, sort_track, id)") + db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_release ON history(sort_release, sort_track, id)") if (db.version < HISTORY_SCHEMA_VERSION) db.version = HISTORY_SCHEMA_VERSION if (deduplicateTrack) deleteDuplicateHistoryRows(db, values) db.insertWithOnConflict("history", null, values, SQLiteDatabase.CONFLICT_REPLACE) @@ -1523,8 +1550,8 @@ object NativeDownloadFinalizer { private fun backfillNormalizedHistoryColumns(db: SQLiteDatabase) { db.query( "history", - arrayOf("id", "spotify_id", "isrc", "track_name", "artist_name", "album_name", "album_artist"), - "spotify_id_norm IS NULL OR isrc_norm IS NULL OR match_key IS NULL OR album_key IS NULL OR search_text IS NULL", + arrayOf("id", "spotify_id", "isrc", "track_name", "artist_name", "album_name", "album_artist", "genre", "release_date", "downloaded_at"), + "spotify_id_norm IS NULL OR isrc_norm IS NULL OR match_key IS NULL OR album_key IS NULL OR search_text IS NULL OR sort_track IS NULL OR sort_release IS NULL OR sort_added IS NULL", null, null, null, @@ -1537,6 +1564,9 @@ object NativeDownloadFinalizer { val artistIndex = cursor.getColumnIndex("artist_name") val albumIndex = cursor.getColumnIndex("album_name") val albumArtistIndex = cursor.getColumnIndex("album_artist") + val genreIndex = cursor.getColumnIndex("genre") + val releaseDateIndex = cursor.getColumnIndex("release_date") + val downloadedAtIndex = cursor.getColumnIndex("downloaded_at") while (cursor.moveToNext()) { if (idIndex < 0) continue val values = ContentValues() @@ -1556,6 +1586,16 @@ object NativeDownloadFinalizer { albumName = albumName, albumArtist = albumArtist, ) + putQueueSortHistoryColumns( + values, + trackName = trackName, + artistName = artistName, + albumName = albumName, + albumArtist = albumArtist, + genre = cursor.getNullableString(genreIndex), + releaseDate = cursor.getNullableString(releaseDateIndex), + downloadedAt = cursor.getNullableString(downloadedAtIndex), + ) db.update("history", values, "id = ?", arrayOf(cursor.getString(idIndex))) } } @@ -1599,6 +1639,60 @@ object NativeDownloadFinalizer { albumName = values.getAsString("album_name"), albumArtist = values.getAsString("album_artist"), ) + putQueueSortHistoryColumns( + values, + trackName = values.getAsString("track_name"), + artistName = values.getAsString("artist_name"), + albumName = values.getAsString("album_name"), + albumArtist = values.getAsString("album_artist"), + genre = values.getAsString("genre"), + releaseDate = values.getAsString("release_date"), + downloadedAt = values.getAsString("downloaded_at"), + ) + } + + private fun putQueueSortHistoryColumns( + values: ContentValues, + trackName: String?, + artistName: String?, + albumName: String?, + albumArtist: String?, + genre: String?, + releaseDate: String?, + downloadedAt: String?, + ) { + values.put("sort_track", normalizeLookupText(trackName)) + values.put("sort_artist", normalizeLookupText(artistName)) + values.put("sort_album", normalizeLookupText(albumName)) + values.put( + "sort_album_artist", + normalizeLookupText(albumArtist?.takeIf { it.trim().isNotEmpty() } ?: artistName), + ) + values.put("sort_genre", normalizeLookupText(genre)) + values.put("sort_release", releaseDate?.trim().orEmpty()) + val sortAdded = parseHistoryTimestampMillis(downloadedAt) + values.put("sort_added", sortAdded) + } + + private fun parseHistoryTimestampMillis(value: String?): Long { + val timestamp = value?.trim().orEmpty() + if (timestamp.isEmpty()) return 0L + return try { + java.time.Instant.parse(timestamp).toEpochMilli() + } catch (_: Exception) { + try { + java.time.OffsetDateTime.parse(timestamp).toInstant().toEpochMilli() + } catch (_: Exception) { + try { + java.time.LocalDateTime.parse(timestamp) + .atZone(java.time.ZoneId.systemDefault()) + .toInstant() + .toEpochMilli() + } catch (_: Exception) { + 0L + } + } + } } private fun putAlbumSearchHistoryColumns( diff --git a/go_backend/exports_library.go b/go_backend/exports_library.go index b342989e..edc93128 100644 --- a/go_backend/exports_library.go +++ b/go_backend/exports_library.go @@ -8,6 +8,10 @@ func ScanLibraryFolderJSON(folderPath string) (string, error) { return ScanLibraryFolder(folderPath) } +func ScanLibraryFolderToNDJSONFileJSON(folderPath, outputPath string) (int, error) { + return ScanLibraryFolderToNDJSONFile(folderPath, outputPath) +} + func ScanLibraryFolderIncrementalJSON(folderPath, existingFilesJSON string) (string, error) { return ScanLibraryFolderIncremental(folderPath, existingFilesJSON) } diff --git a/go_backend/httputil_utls.go b/go_backend/httputil_utls.go index db6fd3f7..39112924 100644 --- a/go_backend/httputil_utls.go +++ b/go_backend/httputil_utls.go @@ -10,6 +10,7 @@ import ( "net/url" "strings" "sync" + "time" utls "github.com/refraction-networking/utls" "golang.org/x/net/http2" @@ -25,7 +26,15 @@ type utlsTransport struct { dialer *net.Dialer h2 *http2.Transport mu sync.Mutex - conns map[string]*http2.ClientConn + conns map[string]pooledHTTP2ClientConn +} + +type pooledHTTP2ClientConn interface { + RoundTrip(*http.Request) (*http.Response, error) + ReserveNewRequest() bool + State() http2.ClientConnState + Close() error + Shutdown(context.Context) error } func newUTLSTransport() *utlsTransport { @@ -35,7 +44,7 @@ func newUTLSTransport() *utlsTransport { KeepAlive: 30 * Second, }, h2: &http2.Transport{}, - conns: make(map[string]*http2.ClientConn), + conns: make(map[string]pooledHTTP2ClientConn), } } @@ -52,6 +61,9 @@ func (t *utlsTransport) RoundTrip(req *http.Request) (*http.Response, error) { if err == nil { return resp, nil } + if req.Context().Err() != nil { + return nil, err + } // A pooled conn can be silently dead after a network switch. Drop it // and, when the request is safely repeatable, fall through to a fresh // dial instead of failing where the old dial-per-request code would @@ -85,8 +97,8 @@ func (t *utlsTransport) RoundTrip(req *http.Request) (*http.Response, error) { tlsConn.Close() return nil, err } - cc = t.storeConn(addr, cc) - return cc.RoundTrip(req) + pooled := t.storeConn(addr, cc) + return pooled.RoundTrip(req) } // rewindRequestBody returns a request whose body can be sent again after a @@ -132,37 +144,99 @@ func (t *utlsTransport) dial(ctx context.Context, host, addr string) (*utls.UCon return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol, nil } -func (t *utlsTransport) cachedConn(addr string) *http2.ClientConn { +func (t *utlsTransport) cachedConn(addr string) pooledHTTP2ClientConn { t.mu.Lock() - defer t.mu.Unlock() - if cc := t.conns[addr]; cc != nil && cc.CanTakeNewRequest() { + cc := t.conns[addr] + if cc != nil && cc.ReserveNewRequest() { + t.mu.Unlock() return cc } + if cc != nil { + delete(t.conns, addr) + } + t.mu.Unlock() + if cc != nil { + retirePooledHTTP2Conn(cc) + } return nil } -func (t *utlsTransport) invalidate(addr string, cc *http2.ClientConn) { +func (t *utlsTransport) invalidate(addr string, cc pooledHTTP2ClientConn) { t.mu.Lock() + removed := false if t.conns[addr] == cc { delete(t.conns, addr) + removed = true } t.mu.Unlock() + if removed { + retirePooledHTTP2Conn(cc) + } } // storeConn caches cc, but if a concurrent dial already cached a healthy conn for // addr it discards the freshly built cc (no in-flight requests) and returns the // existing one, avoiding a leaked connection. -func (t *utlsTransport) storeConn(addr string, cc *http2.ClientConn) *http2.ClientConn { +func (t *utlsTransport) storeConn(addr string, cc pooledHTTP2ClientConn) pooledHTTP2ClientConn { t.mu.Lock() - defer t.mu.Unlock() - if existing := t.conns[addr]; existing != nil && existing.CanTakeNewRequest() { - cc.Close() + if existing := t.conns[addr]; existing != nil && existing.ReserveNewRequest() { + t.mu.Unlock() + _ = cc.Close() return existing } + stale := t.conns[addr] t.conns[addr] = cc + _ = cc.ReserveNewRequest() + t.mu.Unlock() + if stale != nil { + retirePooledHTTP2Conn(stale) + } return cc } +// retirePooledHTTP2Conn prevents new streams while allowing existing streams +// to finish. The independent watchdog also bounds Shutdown implementations +// that block before observing their context. +func pooledHTTP2RetirementTimeout(state http2.ClientConnState) time.Duration { + if state.StreamsActive > 0 || state.StreamsPending > 0 || state.StreamsReserved > 0 { + return 0 + } + return 5 * Second +} + +func retirePooledHTTP2Conn(conn pooledHTTP2ClientConn) { + go func() { + retirePooledHTTP2ConnWithTimeout(conn, pooledHTTP2RetirementTimeout(conn.State())) + }() +} + +func retirePooledHTTP2ConnWithTimeout(conn pooledHTTP2ClientConn, timeout time.Duration) { + go func() { + var closeOnce sync.Once + forceClose := func() { + closeOnce.Do(func() { _ = conn.Close() }) + } + if timeout <= 0 { + if err := conn.Shutdown(context.Background()); err != nil { + forceClose() + } + return + } + + watchdogDone := make(chan struct{}) + watchdog := time.AfterFunc(timeout, func() { + forceClose() + close(watchdogDone) + }) + if err := conn.Shutdown(context.Background()); err != nil { + forceClose() + } + if !watchdog.Stop() { + <-watchdogDone + } + }() +} + // closeIdleConnections drops every pooled conn so the next request re-dials — // needed after a network switch, where pooled conns are silently dead and the // first request would otherwise hang on one until its timeout. Conns are shut @@ -170,10 +244,10 @@ func (t *utlsTransport) storeConn(addr string, cc *http2.ClientConn) *http2.Clie func (t *utlsTransport) closeIdleConnections() { t.mu.Lock() conns := t.conns - t.conns = make(map[string]*http2.ClientConn) + t.conns = make(map[string]pooledHTTP2ClientConn) t.mu.Unlock() for _, cc := range conns { - go cc.Shutdown(context.Background()) + retirePooledHTTP2Conn(cc) } } diff --git a/go_backend/httputil_utls_pool_test.go b/go_backend/httputil_utls_pool_test.go new file mode 100644 index 00000000..f33a78f6 --- /dev/null +++ b/go_backend/httputil_utls_pool_test.go @@ -0,0 +1,187 @@ +//go:build !ios + +package gobackend + +import ( + "context" + "net/http" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/http2" +) + +type fakePooledHTTP2Conn struct { + healthy bool + streamsActive int + blockShutdown bool + ignoreShutdownCtx bool + closeCount atomic.Int32 + shutdownCount atomic.Int32 + shutdownOnce sync.Once + shutdownDone chan struct{} + forceCloseUnblocked chan struct{} + forceCloseOnce sync.Once +} + +func newFakePooledHTTP2Conn(healthy bool) *fakePooledHTTP2Conn { + return &fakePooledHTTP2Conn{ + healthy: healthy, + shutdownDone: make(chan struct{}), + forceCloseUnblocked: make(chan struct{}), + } +} + +func (c *fakePooledHTTP2Conn) RoundTrip(*http.Request) (*http.Response, error) { + return nil, nil +} + +func (c *fakePooledHTTP2Conn) ReserveNewRequest() bool { return c.healthy } + +func (c *fakePooledHTTP2Conn) State() http2.ClientConnState { + if c.healthy { + return http2.ClientConnState{ + StreamsActive: c.streamsActive, + MaxConcurrentStreams: 100, + } + } + return http2.ClientConnState{Closing: true, StreamsActive: c.streamsActive} +} + +func (c *fakePooledHTTP2Conn) Close() error { + c.closeCount.Add(1) + c.forceCloseOnce.Do(func() { close(c.forceCloseUnblocked) }) + return nil +} + +func (c *fakePooledHTTP2Conn) Shutdown(ctx context.Context) error { + c.shutdownCount.Add(1) + c.shutdownOnce.Do(func() { close(c.shutdownDone) }) + if c.ignoreShutdownCtx { + <-c.forceCloseUnblocked + return context.DeadlineExceeded + } + if c.blockShutdown { + <-ctx.Done() + return ctx.Err() + } + return nil +} + +func waitForShutdown(t *testing.T, conn *fakePooledHTTP2Conn) { + t.Helper() + select { + case <-conn.shutdownDone: + case <-time.After(time.Second): + t.Fatal("connection did not begin graceful shutdown") + } +} + +func TestUTLSPoolRetiresStaleCachedConnection(t *testing.T) { + transport := newUTLSTransport() + stale := newFakePooledHTTP2Conn(false) + transport.conns["example:443"] = stale + + if got := transport.cachedConn("example:443"); got != nil { + t.Fatalf("cachedConn returned stale connection: %#v", got) + } + waitForShutdown(t, stale) + if stale.closeCount.Load() != 0 { + t.Fatalf("gracefully retired connection was force closed") + } + if _, exists := transport.conns["example:443"]; exists { + t.Fatal("stale connection was not removed") + } +} + +func TestUTLSPoolStoreClosesDiscardedAndRetiresReplacedConnection(t *testing.T) { + transport := newUTLSTransport() + healthy := newFakePooledHTTP2Conn(true) + transport.conns["example:443"] = healthy + fresh := newFakePooledHTTP2Conn(true) + + if got := transport.storeConn("example:443", fresh); got != healthy { + t.Fatal("healthy pooled connection was not reused") + } + if fresh.closeCount.Load() != 1 { + t.Fatalf("discarded fresh close count = %d", fresh.closeCount.Load()) + } + + stale := newFakePooledHTTP2Conn(false) + transport.conns["example:443"] = stale + replacement := newFakePooledHTTP2Conn(true) + if got := transport.storeConn("example:443", replacement); got != replacement { + t.Fatal("stale connection was not replaced") + } + waitForShutdown(t, stale) + if stale.closeCount.Load() != 0 { + t.Fatalf("replaced connection was force closed") + } +} + +func TestUTLSPoolInvalidateRetiresOnlyRequestedConnection(t *testing.T) { + transport := newUTLSTransport() + current := newFakePooledHTTP2Conn(true) + old := newFakePooledHTTP2Conn(false) + transport.conns["example:443"] = current + + transport.invalidate("example:443", old) + if transport.conns["example:443"] != current { + t.Fatal("invalidating an old connection removed the replacement") + } + if old.shutdownCount.Load() != 0 { + t.Fatal("connection already removed from the pool was retired again") + } + + transport.invalidate("example:443", current) + waitForShutdown(t, current) + if _, exists := transport.conns["example:443"]; exists { + t.Fatal("invalidated current connection remained in the pool") + } +} + +func TestUTLSPoolCloseIdleUsesBoundedShutdown(t *testing.T) { + transport := newUTLSTransport() + conn := newFakePooledHTTP2Conn(true) + transport.conns["example:443"] = conn + + transport.closeIdleConnections() + if len(transport.conns) != 0 { + t.Fatal("pool was not cleared synchronously") + } + select { + case <-conn.shutdownDone: + case <-time.After(time.Second): + t.Fatal("pooled connection was not shut down") + } + if conn.shutdownCount.Load() != 1 { + t.Fatalf("shutdown count = %d", conn.shutdownCount.Load()) + } +} + +func TestUTLSPoolForcesCloseWhenGracefulShutdownTimesOut(t *testing.T) { + conn := newFakePooledHTTP2Conn(true) + conn.ignoreShutdownCtx = true + + retirePooledHTTP2ConnWithTimeout(conn, 20*time.Millisecond) + waitForShutdown(t, conn) + + deadline := time.After(time.Second) + for conn.closeCount.Load() == 0 { + select { + case <-deadline: + t.Fatal("timed-out graceful shutdown did not force close") + case <-time.After(10 * time.Millisecond): + } + } +} + +func TestUTLSPoolDoesNotPutADeadlineOnActiveStreams(t *testing.T) { + conn := newFakePooledHTTP2Conn(false) + conn.streamsActive = 1 + if timeout := pooledHTTP2RetirementTimeout(conn.State()); timeout != 0 { + t.Fatalf("active connection retirement timeout = %v", timeout) + } +} diff --git a/go_backend/library_scan.go b/go_backend/library_scan.go index 41089fa0..619d6929 100644 --- a/go_backend/library_scan.go +++ b/go_backend/library_scan.go @@ -1,6 +1,7 @@ package gobackend import ( + "bufio" "encoding/json" "fmt" "os" @@ -122,7 +123,7 @@ func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]li err := filepath.WalkDir(folderPath, func(path string, entry os.DirEntry, err error) error { if err != nil { - return nil + return fmt.Errorf("walk library path %s: %w", path, err) } select { @@ -145,7 +146,7 @@ func collectLibraryAudioFiles(folderPath string, cancelCh <-chan struct{}) ([]li info, err := entry.Info() if err != nil { - return nil + return fmt.Errorf("read library file info %s: %w", path, err) } files = append(files, libraryAudioFileInfo{ @@ -195,7 +196,28 @@ func updateLibraryScanProgress(scannedFiles, totalFiles int, currentPath string) } func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, cancelCh <-chan struct{}, totalFiles int, completed *int) (map[int][]LibraryScanResult, int, error) { - resultsByIndex := make(map[int][]LibraryScanResult, len(tasks)) + return scanLibraryAudioTasksParallelWithSink( + tasks, + scanTime, + cancelCh, + totalFiles, + completed, + nil, + ) +} + +func scanLibraryAudioTasksParallelWithSink( + tasks []libraryScanTask, + scanTime string, + cancelCh <-chan struct{}, + totalFiles int, + completed *int, + sink func([]LibraryScanResult) error, +) (map[int][]LibraryScanResult, int, error) { + var resultsByIndex map[int][]LibraryScanResult + if sink == nil { + resultsByIndex = make(map[int][]LibraryScanResult, len(tasks)) + } if len(tasks) == 0 { return resultsByIndex, 0, nil } @@ -223,7 +245,14 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can GoLog("[LibraryScan] Error scanning %s: %v\n", task.info.path, err) continue } - resultsByIndex[task.index] = []LibraryScanResult{*result} + results := []LibraryScanResult{*result} + if sink != nil { + if err := sink(results); err != nil { + return resultsByIndex, errorCount, err + } + } else { + resultsByIndex[task.index] = results + } } return resultsByIndex, errorCount, nil } @@ -283,6 +312,7 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can }() errorCount := 0 + var sinkErr error for taskResult := range resultCh { *completed++ updateLibraryScanProgress(*completed, totalFiles, taskResult.path) @@ -291,7 +321,16 @@ func scanLibraryAudioTasksParallel(tasks []libraryScanTask, scanTime string, can GoLog("[LibraryScan] Error scanning %s: %v\n", taskResult.path, taskResult.err) continue } - resultsByIndex[taskResult.index] = taskResult.results + if sink != nil { + if sinkErr == nil { + sinkErr = sink(taskResult.results) + } + } else { + resultsByIndex[taskResult.index] = taskResult.results + } + } + if sinkErr != nil { + return resultsByIndex, errorCount, sinkErr } select { @@ -308,17 +347,21 @@ func SetLibraryCoverCacheDir(cacheDir string) { libraryCoverCacheMu.Unlock() } -func ScanLibraryFolder(folderPath string) (string, error) { +func scanLibraryFolderWithSink( + folderPath string, + sink func(LibraryScanResult) error, + preserveOrder bool, +) (int, error) { if folderPath == "" { - return "[]", fmt.Errorf("folder path is empty") + return 0, fmt.Errorf("folder path is empty") } info, err := os.Stat(folderPath) if err != nil { - return "[]", fmt.Errorf("folder not found: %w", err) + return 0, fmt.Errorf("folder not found: %w", err) } if !info.IsDir() { - return "[]", fmt.Errorf("path is not a folder: %s", folderPath) + return 0, fmt.Errorf("path is not a folder: %s", folderPath) } libraryScanProgressMu.Lock() @@ -335,7 +378,7 @@ func ScanLibraryFolder(folderPath string) (string, error) { audioFileInfos, err := collectLibraryAudioFiles(folderPath, cancelCh) if err != nil { - return "[]", err + return 0, err } totalFiles := len(audioFileInfos) @@ -346,51 +389,61 @@ func ScanLibraryFolder(folderPath string) (string, error) { if totalFiles == 0 { libraryScanProgressMu.Lock() libraryScanProgress.IsComplete = true + libraryScanProgress.ProgressPct = 100 libraryScanProgressMu.Unlock() - return "[]", nil + return 0, nil } GoLog("[LibraryScan] Found %d audio files to scan\n", totalFiles) - results := make([]LibraryScanResult, 0, totalFiles) scanTime := time.Now().UTC().Format(time.RFC3339) errorCount := 0 + emittedCount := 0 + emitResults := func(results []LibraryScanResult) error { + for i := range results { + if err := sink(results[i]); err != nil { + return err + } + emittedCount++ + } + return nil + } cueReferencedAudioFiles := make(map[string]bool) parsedCueFiles := make(map[string]scannedCueFileInfo) - for _, fileInfo := range audioFileInfos { filePath := fileInfo.path - ext := strings.ToLower(filepath.Ext(filePath)) - if ext == ".cue" { - sheet, err := ParseCueFile(filePath) - if err == nil && sheet.FileName != "" { - audioPath := ResolveCueAudioPath(filePath, sheet.FileName) - if audioPath != "" { - parsedCueFiles[filePath] = scannedCueFileInfo{ - sheet: sheet, - audioPath: audioPath, - } - cueReferencedAudioFiles[audioPath] = true + if strings.ToLower(filepath.Ext(filePath)) != ".cue" { + continue + } + sheet, parseErr := ParseCueFile(filePath) + if parseErr == nil && sheet.FileName != "" { + audioPath := ResolveCueAudioPath(filePath, sheet.FileName) + if audioPath != "" { + parsedCueFiles[filePath] = scannedCueFileInfo{ + sheet: sheet, + audioPath: audioPath, } + cueReferencedAudioFiles[audioPath] = true } } } - resultsByIndex := make(map[int][]LibraryScanResult, totalFiles) audioTasks := make([]libraryScanTask, 0, totalFiles) + var orderedResults map[int][]LibraryScanResult + if preserveOrder { + orderedResults = make(map[int][]LibraryScanResult, totalFiles) + } completedFiles := 0 - for i, fileInfo := range audioFileInfos { filePath := fileInfo.path select { case <-cancelCh: - return "[]", fmt.Errorf("scan cancelled") + return emittedCount, fmt.Errorf("scan cancelled") default: } ext := strings.ToLower(filepath.Ext(filePath)) - if ext == ".cue" { var cueResults []LibraryScanResult cueInfo, ok := parsedCueFiles[filePath] @@ -407,16 +460,18 @@ func ScanLibraryFolder(folderPath string) (string, error) { } else { cueResults, err = ScanCueFileForLibrary(filePath, scanTime) } + completedFiles++ + updateLibraryScanProgress(completedFiles, totalFiles, filePath) if err != nil { errorCount++ GoLog("[LibraryScan] Error scanning cue %s: %v\n", filePath, err) - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, filePath) continue } - resultsByIndex[i] = cueResults - completedFiles++ - updateLibraryScanProgress(completedFiles, totalFiles, filePath) + if preserveOrder { + orderedResults[i] = cueResults + } else if err := emitResults(cueResults); err != nil { + return emittedCount, fmt.Errorf("write scan result: %w", err) + } GoLog("[LibraryScan] CUE sheet %s: %d tracks\n", filepath.Base(filePath), len(cueResults)) continue } @@ -431,31 +486,53 @@ func ScanLibraryFolder(folderPath string) (string, error) { audioTasks = append(audioTasks, libraryScanTask{index: i, info: fileInfo}) } - audioResults, audioErrors, err := scanLibraryAudioTasksParallel( + var audioSink func([]LibraryScanResult) error + if !preserveOrder { + audioSink = emitResults + } + audioResults, audioErrors, err := scanLibraryAudioTasksParallelWithSink( audioTasks, scanTime, cancelCh, totalFiles, &completedFiles, + audioSink, ) if err != nil { - return "[]", err + return emittedCount, err } errorCount += audioErrors - for index, scanResults := range audioResults { - resultsByIndex[index] = scanResults - } - - for i := range audioFileInfos { - results = append(results, resultsByIndex[i]...) + if preserveOrder { + for index, results := range audioResults { + orderedResults[index] = results + } + for i := range audioFileInfos { + if err := emitResults(orderedResults[i]); err != nil { + return emittedCount, fmt.Errorf("write scan result: %w", err) + } + } } libraryScanProgressMu.Lock() libraryScanProgress.ErrorCount = errorCount libraryScanProgress.IsComplete = true + libraryScanProgress.ScannedFiles = totalFiles + libraryScanProgress.ProgressPct = 100 libraryScanProgressMu.Unlock() - GoLog("[LibraryScan] Scan complete: %d tracks found, %d errors\n", len(results), errorCount) + GoLog("[LibraryScan] Scan complete: %d tracks found, %d errors\n", emittedCount, errorCount) + return emittedCount, nil +} + +func ScanLibraryFolder(folderPath string) (string, error) { + results := make([]LibraryScanResult, 0) + _, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error { + results = append(results, result) + return nil + }, true) + if err != nil { + return "[]", err + } jsonBytes, err := json.Marshal(results) if err != nil { @@ -465,6 +542,43 @@ func ScanLibraryFolder(folderPath string) (string, error) { return string(jsonBytes), nil } +// ScanLibraryFolderToNDJSONFile writes one JSON object per line so mobile +// clients can decode and ingest bounded batches instead of materializing a +// full-library JSON array in both the Go and Dart heaps. +func ScanLibraryFolderToNDJSONFile(folderPath, outputPath string) (int, error) { + if outputPath == "" { + return 0, fmt.Errorf("output path is empty") + } + file, err := os.Create(outputPath) + if err != nil { + return 0, fmt.Errorf("create scan output: %w", err) + } + removeOnError := true + defer func() { + _ = file.Close() + if removeOnError { + _ = os.Remove(outputPath) + } + }() + + writer := bufio.NewWriterSize(file, 64*1024) + encoder := json.NewEncoder(writer) + count, err := scanLibraryFolderWithSink(folderPath, func(result LibraryScanResult) error { + return encoder.Encode(result) + }, false) + if err != nil { + return count, err + } + if err := writer.Flush(); err != nil { + return count, fmt.Errorf("flush scan output: %w", err) + } + if err := file.Close(); err != nil { + return count, fmt.Errorf("close scan output: %w", err) + } + removeOnError = false + return count, nil +} + func GetLibraryScanProgress() string { libraryScanProgressMu.RLock() defer libraryScanProgressMu.RUnlock() diff --git a/go_backend/library_scan_supplement_test.go b/go_backend/library_scan_supplement_test.go index 04189359..88b9719d 100644 --- a/go_backend/library_scan_supplement_test.go +++ b/go_backend/library_scan_supplement_test.go @@ -1,7 +1,9 @@ package gobackend import ( + "bufio" "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -89,6 +91,35 @@ func TestLibraryScanFullIncrementalAndMetadataFallbacks(t *testing.T) { if !foundTagged { t.Fatalf("tagged APE not found in %#v", results) } + + ndjsonPath := filepath.Join(t.TempDir(), "library.ndjson") + streamedCount, err := ScanLibraryFolderToNDJSONFile(dir, ndjsonPath) + if err != nil { + t.Fatalf("ScanLibraryFolderToNDJSONFile: %v", err) + } + ndjsonFile, err := os.Open(ndjsonPath) + if err != nil { + t.Fatal(err) + } + defer ndjsonFile.Close() + decodedCount := 0 + scanner := bufio.NewScanner(ndjsonFile) + for scanner.Scan() { + var result LibraryScanResult + if err := json.Unmarshal(scanner.Bytes(), &result); err != nil { + t.Fatalf("decode NDJSON row: %v", err) + } + if result.FilePath == "" { + t.Fatal("NDJSON row has no file path") + } + decodedCount++ + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + if decodedCount != streamedCount || decodedCount != len(results) { + t.Fatalf("NDJSON counts = decoded:%d streamed:%d array:%d", decodedCount, streamedCount, len(results)) + } if progress := GetLibraryScanProgress(); !strings.Contains(progress, `"IsComplete":true`) && !strings.Contains(progress, `"is_complete":true`) { t.Fatalf("progress = %s", progress) } @@ -161,3 +192,31 @@ func TestLibraryScanFullIncrementalAndMetadataFallbacks(t *testing.T) { CancelLibraryScan() SetLibraryCoverCacheDir("") } + +func TestScanLibraryFolderPreservesFileOrderWithParallelWorkers(t *testing.T) { + dir := t.TempDir() + for i := 0; i < 20; i++ { + name := fmt.Sprintf("%02d - Track.mp3", i) + if err := os.WriteFile(filepath.Join(dir, name), []byte("not really mp3"), 0600); err != nil { + t.Fatal(err) + } + } + + jsonText, err := ScanLibraryFolder(dir) + if err != nil { + t.Fatal(err) + } + var results []LibraryScanResult + if err := json.Unmarshal([]byte(jsonText), &results); err != nil { + t.Fatal(err) + } + if len(results) != 20 { + t.Fatalf("results = %d", len(results)) + } + for i, result := range results { + expected := filepath.Join(dir, fmt.Sprintf("%02d - Track.mp3", i)) + if result.FilePath != expected { + t.Fatalf("result %d path = %q, want %q", i, result.FilePath, expected) + } + } +} diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift index 5e54992d..024e5954 100644 --- a/ios/Runner/AppDelegate.swift +++ b/ios/Runner/AppDelegate.swift @@ -22,8 +22,8 @@ import Gobackend private var backendChannel: FlutterMethodChannel? private var pendingSessionGrantEvents: [[String: Any]] = [] - /// Currently accessed security-scoped URL for library folder - private var activeSecurityScopedURL: URL? + private let securityScopedAccessLock = NSLock() + private var securityScopedAccesses: [String: URL] = [:] /// Pending Flutter result for the native folder picker private var pendingDirectoryPickerResult: FlutterResult? @@ -1074,6 +1074,24 @@ import Gobackend let response = GobackendScanLibraryFolderJSON(folderPath, &error) if let error = error { throw error } return bridgeJsonResult(response as String? ?? "[]") + + case "scanLibraryFolderToNDJSONFile": + guard + let args = call.arguments as? [String: Any], + let folderPath = args["folder_path"] as? String, + !folderPath.isEmpty, + let outputPath = args["output_path"] as? String, + !outputPath.isEmpty + else { + throw invalidArgumentsError(call.method) + } + let count = GobackendScanLibraryFolderToNDJSONFileJSON( + folderPath, + outputPath, + &error + ) + if let error = error { throw error } + return ["path": outputPath, "count": count] case "scanLibraryFolderIncremental": let args = call.arguments as! [String: Any] @@ -1104,12 +1122,24 @@ import Gobackend return try resolveIosBookmark(bookmarkBase64) case "startAccessingIosBookmark": - let args = call.arguments as! [String: Any] - let bookmarkBase64 = args["bookmark"] as! String + guard + let args = call.arguments as? [String: Any], + let bookmarkBase64 = args["bookmark"] as? String, + !bookmarkBase64.isEmpty + else { + throw invalidArgumentsError(call.method) + } return try startAccessingIosBookmark(bookmarkBase64) case "stopAccessingIosBookmark": - stopAccessingIosBookmark() + guard + let args = call.arguments as? [String: Any], + let token = args["token"] as? String, + !token.isEmpty + else { + throw invalidArgumentsError(call.method) + } + stopAccessingIosBookmark(token: token) return nil case "createIosBookmarkFromPath": @@ -1259,13 +1289,16 @@ import Gobackend return url.path } - /// Resolve a base64-encoded bookmark, start accessing the security-scoped resource, - /// and return the resolved filesystem path. The resource stays accessed until - /// `stopAccessingIosBookmark()` is called. - private func startAccessingIosBookmark(_ bookmarkBase64: String) throws -> String { - // Stop any previously accessed resource first - stopAccessingIosBookmark() - + private func invalidArgumentsError(_ method: String) -> NSError { + return NSError( + domain: "SpotiFLAC", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid arguments for \(method)"] + ) + } + + /// Starts an independently owned security-scoped lease. + private func startAccessingIosBookmark(_ bookmarkBase64: String) throws -> [String: String] { guard let bookmarkData = Data(base64Encoded: bookmarkBase64) else { throw NSError( domain: "SpotiFLAC", @@ -1304,16 +1337,19 @@ import Gobackend ) } - activeSecurityScopedURL = url - return url.path + let token = UUID().uuidString + securityScopedAccessLock.lock() + securityScopedAccesses[token] = url + securityScopedAccessLock.unlock() + return ["path": url.path, "token": token] } - /// Stop accessing the currently active security-scoped resource, if any. - private func stopAccessingIosBookmark() { - if let url = activeSecurityScopedURL { - url.stopAccessingSecurityScopedResource() - activeSecurityScopedURL = nil - } + /// Releases only the lease identified by the caller's token. + private func stopAccessingIosBookmark(token: String) { + securityScopedAccessLock.lock() + let url = securityScopedAccesses.removeValue(forKey: token) + securityScopedAccessLock.unlock() + url?.stopAccessingSecurityScopedResource() } } diff --git a/lib/providers/download_history_models.dart b/lib/providers/download_history_models.dart index 4f5cb3fd..aaa9a7e3 100644 --- a/lib/providers/download_history_models.dart +++ b/lib/providers/download_history_models.dart @@ -81,7 +81,7 @@ class DownloadHistoryItem { 'safFileName': safFileName, 'safRepaired': safRepaired, 'service': service, - 'downloadedAt': downloadedAt.toIso8601String(), + 'downloadedAt': downloadedAt.toUtc().toIso8601String(), 'isrc': isrc, 'spotifyId': spotifyId, 'trackNumber': trackNumber, @@ -116,7 +116,7 @@ class DownloadHistoryItem { safFileName: json['safFileName'] as String?, safRepaired: json['safRepaired'] == true, service: json['service'] as String, - downloadedAt: DateTime.parse(json['downloadedAt'] as String), + downloadedAt: DateTime.parse(json['downloadedAt'] as String).toLocal(), isrc: json['isrc'] as String?, spotifyId: json['spotifyId'] as String?, trackNumber: json['trackNumber'] as int?, diff --git a/lib/providers/download_queue_provider.dart b/lib/providers/download_queue_provider.dart index fab9a56f..abf1dc84 100644 --- a/lib/providers/download_queue_provider.dart +++ b/lib/providers/download_queue_provider.dart @@ -158,9 +158,7 @@ List normalizeBatchAlbumArtists(List tracks) { final currentArtist = normalizeOptionalString(tracks[index].albumArtist); if (currentArtist == canonicalArtist) continue; normalized ??= List.of(tracks); - normalized[index] = tracks[index].copyWith( - albumArtist: canonicalArtist, - ); + normalized[index] = tracks[index].copyWith(albumArtist: canonicalArtist); } } @@ -209,9 +207,7 @@ String? _sharedBatchAlbumArtist(List tracks) { credits.add(names); } - final sharedKeys = credits.first - .map((name) => name.toLowerCase()) - .toSet(); + final sharedKeys = credits.first.map((name) => name.toLowerCase()).toSet(); for (final credit in credits.skip(1)) { final keys = credit.map((name) => name.toLowerCase()).toSet(); sharedKeys.removeWhere((name) => !keys.contains(name)); @@ -1453,7 +1449,7 @@ class DownloadQueueNotifier extends Notifier { var settings = ref.read(settingsProvider); updateSettings(settings); var isSafMode = _isSafMode(settings); - var iosDownloadBookmarkActive = false; + IosSecurityScopedAccess? iosDownloadBookmarkAccess; // Validate SAF before handing the batch to either queue implementation. // Never silently redirect a user-selected SAF destination into private app @@ -1630,11 +1626,12 @@ class DownloadQueueNotifier extends Notifier { if (!isSafMode && Platform.isIOS && settings.downloadDirectoryBookmark.isNotEmpty) { - final resolvedPath = await PlatformBridge.startAccessingIosBookmark( - settings.downloadDirectoryBookmark, - ); + iosDownloadBookmarkAccess = + await PlatformBridge.startAccessingIosBookmark( + settings.downloadDirectoryBookmark, + ); + final resolvedPath = iosDownloadBookmarkAccess?.path; if (resolvedPath != null && resolvedPath.isNotEmpty) { - iosDownloadBookmarkActive = true; if (resolvedPath != state.outputDir) { _log.i('Resolved iOS download bookmark path: $resolvedPath'); state = state.copyWith(outputDir: resolvedPath); @@ -1662,9 +1659,11 @@ class DownloadQueueNotifier extends Notifier { try { await _runQueueLoop(); } finally { - if (iosDownloadBookmarkActive) { - await PlatformBridge.stopAccessingIosBookmark(); - iosDownloadBookmarkActive = false; + if (iosDownloadBookmarkAccess != null) { + await PlatformBridge.stopAccessingIosBookmark( + iosDownloadBookmarkAccess, + ); + iosDownloadBookmarkAccess = null; } } final stoppedWhilePaused = state.isPaused; diff --git a/lib/providers/library_collections_provider.dart b/lib/providers/library_collections_provider.dart index 02da2248..a25b2637 100644 --- a/lib/providers/library_collections_provider.dart +++ b/lib/providers/library_collections_provider.dart @@ -6,8 +6,12 @@ import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; import 'package:spotiflac_android/models/track.dart'; +import 'package:spotiflac_android/services/ffmpeg_service.dart'; import 'package:spotiflac_android/services/library_collections_database.dart'; +const _playlistCoverMaxDimension = 1024; +const _playlistCoverMaxStoredBytes = 2 * 1024 * 1024; + String trackCollectionKey(Track track) { final isrc = track.isrc?.trim(); if (isrc != null && isrc.isNotEmpty) { @@ -986,12 +990,11 @@ class LibraryCollectionsNotifier extends Notifier { final playlist = state.playlistById(playlistId); if (playlist == null) return; - final coversDir = await _playlistCoversDir(); - final ext = p.extension(sourceFilePath).toLowerCase(); - final destPath = p.join(coversDir.path, '$playlistId$ext'); - if (playlist.coverImagePath == destPath) return; - - await File(sourceFilePath).copy(destPath); + final previousCoverPath = playlist.coverImagePath; + final destPath = await _normalizePlaylistCoverFile( + playlistId, + sourceFilePath, + ); final now = DateTime.now(); await _db.updatePlaylistCover( @@ -1004,6 +1007,78 @@ class LibraryCollectionsNotifier extends Notifier { return playlist.copyWith(coverImagePath: () => destPath, updatedAt: now); }); _invalidatePlaylistPickerSummaries(); + if (previousCoverPath != null && previousCoverPath != destPath) { + try { + final previous = File(previousCoverPath); + if (await previous.exists()) await previous.delete(); + } catch (_) {} + } + } + + Future _normalizePlaylistCoverFile( + String playlistId, + String sourceFilePath, + ) async { + final coversDir = await _playlistCoversDir(); + final destPath = p.join(coversDir.path, '$playlistId.jpg'); + final tempPath = p.join( + coversDir.path, + '.$playlistId.${DateTime.now().microsecondsSinceEpoch}.jpg', + ); + try { + final dimensions = await FFmpegService.probeImageDimensions( + sourceFilePath, + ); + final longestEdge = dimensions == null + ? _playlistCoverMaxDimension + : (dimensions.width > dimensions.height + ? dimensions.width + : dimensions.height); + var targetDimension = longestEdge + .clamp(64, _playlistCoverMaxDimension) + .toInt(); + while (true) { + final resized = await FFmpegService.resizeCoverArt( + inputPath: sourceFilePath, + outputPath: tempPath, + maxDimension: targetDimension, + ); + if (!resized) { + throw StateError('Unable to normalize playlist cover'); + } + if (await File(tempPath).length() <= _playlistCoverMaxStoredBytes) { + break; + } + if (targetDimension <= 64) { + throw StateError('Normalized playlist cover exceeds the size limit'); + } + targetDimension = (targetDimension * 3 ~/ 4) + .clamp(64, targetDimension) + .toInt(); + } + final destination = File(destPath); + if (await destination.exists()) await destination.delete(); + await File(tempPath).rename(destPath); + return destPath; + } finally { + try { + final temp = File(tempPath); + if (await temp.exists()) await temp.delete(); + } catch (_) {} + } + } + + Future _playlistCoverNeedsNormalization(String path) async { + final file = File(path); + if (!await file.exists()) return false; + if (p.extension(path).toLowerCase() != '.jpg' || + await file.length() > _playlistCoverMaxStoredBytes) { + return true; + } + final dimensions = await FFmpegService.probeImageDimensions(path); + return dimensions == null || + dimensions.width > _playlistCoverMaxDimension || + dimensions.height > _playlistCoverMaxDimension; } Future removePlaylistCover(String playlistId) async { @@ -1046,10 +1121,15 @@ class LibraryCollectionsNotifier extends Notifier { Future>> exportPlaylistCovers() async { await _ensureLoaded(); final covers = >{}; - for (final playlist in state.playlists) { - final path = playlist.coverImagePath; + final playlists = List.of(state.playlists); + for (final playlist in playlists) { + var path = playlist.coverImagePath; if (path == null || path.isEmpty) continue; try { + if (await _playlistCoverNeedsNormalization(path)) { + await setPlaylistCover(playlist.id, path); + path = state.playlistById(playlist.id)?.coverImagePath ?? path; + } final file = File(path); if (!await file.exists()) continue; final bytes = await file.readAsBytes(); @@ -1092,9 +1172,22 @@ class LibraryCollectionsNotifier extends Notifier { final ext = (coverEntry['ext'] as String?) ?? '.jpg'; if (data != null && data.isNotEmpty) { try { - final destPath = p.join(coversDir.path, '$id$ext'); - await File(destPath).writeAsBytes(base64Decode(data)); - newCoverPath = destPath; + final sourcePath = p.join( + coversDir.path, + '.$id.restore${ext.startsWith('.') ? ext : '.$ext'}', + ); + final source = File(sourcePath); + await source.writeAsBytes(base64Decode(data), flush: true); + try { + newCoverPath = await _normalizePlaylistCoverFile( + id, + sourcePath, + ); + } finally { + try { + if (await source.exists()) await source.delete(); + } catch (_) {} + } } catch (_) { newCoverPath = null; } diff --git a/lib/providers/local_library_provider.dart b/lib/providers/local_library_provider.dart index 2e397dd1..2814327c 100644 --- a/lib/providers/local_library_provider.dart +++ b/lib/providers/local_library_provider.dart @@ -131,6 +131,7 @@ class LocalLibraryNotifier extends Notifier { bool _hasLoadedFromDatabase = false; Future? _loadFuture; bool _scanCancelRequested = false; + bool _scanInProgress = false; static const _scanNotificationHeartbeat = Duration(seconds: 4); int _lastScanNotificationPercent = -1; int _lastScanNotificationTotalFiles = -1; @@ -237,16 +238,73 @@ class LocalLibraryNotifier extends Notifier { return false; } + Future<({int inserted, int skipped})?> _replaceFromFullScanStream({ + required String folderPath, + required bool isSaf, + required Set downloadedPathKeys, + }) async { + if (_scanCancelRequested) return null; + final scanFile = isSaf + ? await PlatformBridge.scanSafTreeToNDJSONFile( + folderPath, + isCancelled: () => _scanCancelRequested, + ) + : await PlatformBridge.scanLibraryFolderToNDJSONFile( + folderPath, + isCancelled: () => _scanCancelRequested, + ); + var skipped = 0; + try { + if (_scanCancelRequested) return null; + state = state.copyWith( + scanIsFinalizing: true, + scanProgress: state.scanProgress >= 99 ? state.scanProgress : 99, + scanCurrentFile: null, + ); + Stream> filteredRows() async* { + var decodedRows = 0; + await for (final json in scanFile.rows()) { + if (_scanCancelRequested) { + throw StateError('Library scan cancelled during ingestion'); + } + decodedRows++; + final filePath = json['filePath'] as String?; + if (_isDownloadedPath(filePath, downloadedPathKeys)) { + skipped++; + continue; + } + yield json; + } + if (decodedRows != scanFile.expectedCount) { + throw FormatException( + 'Library scan row count mismatch: decoded $decodedRows, ' + 'expected ${scanFile.expectedCount}', + ); + } + } + + final inserted = await _db.replaceAllStream(filteredRows()); + _log.i( + 'Stream-ingested $inserted/${scanFile.expectedCount} scan rows ' + '($skipped downloads excluded)', + ); + return (inserted: inserted, skipped: skipped); + } finally { + await scanFile.delete(); + } + } + Future startScan( String folderPath, { bool forceFullScan = false, String? iosBookmark, }) async { - if (state.isScanning) { + if (_scanInProgress || state.isScanning) { _log.w('Scan already in progress'); return; } + _scanInProgress = true; _scanCancelRequested = false; _log.i( 'Starting library scan: $folderPath (incremental: ${!forceFullScan})', @@ -287,13 +345,13 @@ class LocalLibraryNotifier extends Notifier { _startProgressPolling(); String? resolvedPath; - bool didStartSecurityAccess = false; + IosSecurityScopedAccess? securityAccess; if (Platform.isIOS && iosBookmark != null && iosBookmark.isNotEmpty) { - resolvedPath = await PlatformBridge.startAccessingIosBookmark( + securityAccess = await PlatformBridge.startAccessingIosBookmark( iosBookmark, ); - if (resolvedPath != null) { - didStartSecurityAccess = true; + resolvedPath = securityAccess?.path; + if (securityAccess != null) { _log.i('Started iOS security-scoped access: $resolvedPath'); } else { _log.w( @@ -326,11 +384,14 @@ class LocalLibraryNotifier extends Notifier { '(${downloadedPathKeys.length} path keys)', ); - if (forceFullScan) { - final results = isSaf - ? await PlatformBridge.scanSafTree(effectiveFolderPath) - : await PlatformBridge.scanLibraryFolder(effectiveFolderPath); - if (_scanCancelRequested) { + final useStreamingFullScan = forceFullScan || await _db.getCount() == 0; + if (useStreamingFullScan) { + final scanResult = await _replaceFromFullScanStream( + folderPath: effectiveFolderPath, + isSaf: isSaf, + downloadedPathKeys: downloadedPathKeys, + ); + if (scanResult == null || _scanCancelRequested) { state = state.copyWith( isScanning: false, scanIsFinalizing: false, @@ -346,24 +407,12 @@ class LocalLibraryNotifier extends Notifier { scanCurrentFile: null, ); - final items = []; - int skippedDownloads = 0; - for (final json in results) { - final filePath = json['filePath'] as String?; - if (_isDownloadedPath(filePath, downloadedPathKeys)) { - skippedDownloads++; - continue; - } - final item = LocalLibraryItem.fromJson(json); - items.add(item); - } + final skippedDownloads = scanResult.skipped; if (skippedDownloads > 0) { _log.i('Skipped $skippedDownloads files already in download history'); } - await _db.replaceAll(items.map((e) => e.toJson()).toList()); - final now = DateTime.now(); try { final prefs = await SharedPreferences.getInstance(); @@ -421,6 +470,9 @@ class LocalLibraryNotifier extends Notifier { Map result; try { + if (_scanCancelRequested) { + throw StateError('Library scan cancelled before native scan'); + } if (isSaf) { result = useSnapshotBridge && snapshotPath != null ? await PlatformBridge.scanSafTreeIncrementalFromSnapshot( @@ -564,6 +616,16 @@ class LocalLibraryNotifier extends Notifier { ); } } catch (e, stack) { + if (_scanCancelRequested) { + _log.i('Library scan cancelled'); + state = state.copyWith( + isScanning: false, + scanIsFinalizing: false, + scanWasCancelled: true, + ); + await _showScanCancelledNotification(); + return; + } _log.e('Library scan failed: $e', e, stack); state = state.copyWith( isScanning: false, @@ -572,11 +634,12 @@ class LocalLibraryNotifier extends Notifier { ); await _showScanFailedNotification(e.toString()); } finally { - if (didStartSecurityAccess) { - await PlatformBridge.stopAccessingIosBookmark(); + if (securityAccess != null) { + await PlatformBridge.stopAccessingIosBookmark(securityAccess); _log.i('Stopped iOS security-scoped access'); } _stopProgressPolling(); + _scanInProgress = false; } } @@ -593,6 +656,7 @@ class LocalLibraryNotifier extends Notifier { } Future _handleLibraryScanProgress(Map progress) async { + if (_scanCancelRequested) return; final nextProgress = (progress['progress_pct'] as num?)?.toDouble() ?? 0; final normalizedProgress = ((nextProgress * 10).round() / 10).clamp( 0.0, @@ -683,11 +747,7 @@ class LocalLibraryNotifier extends Notifier { _log.i('Cancelling library scan'); _scanCancelRequested = true; await PlatformBridge.cancelLibraryScan(); - state = state.copyWith( - isScanning: false, - scanIsFinalizing: false, - scanWasCancelled: true, - ); + state = state.copyWith(scanIsFinalizing: false, scanWasCancelled: true); _stopProgressPolling(); await _showScanCancelledNotification(); } @@ -761,13 +821,15 @@ class LocalLibraryNotifier extends Notifier { } Future cleanupMissingFiles({String? iosBookmark}) async { - bool didStartSecurityAccess = false; + IosSecurityScopedAccess? securityAccess; if (Platform.isIOS && iosBookmark != null && iosBookmark.isNotEmpty) { - final resolved = await PlatformBridge.startAccessingIosBookmark( + securityAccess = await PlatformBridge.startAccessingIosBookmark( iosBookmark, ); - if (resolved != null) { - didStartSecurityAccess = true; + if (securityAccess == null) { + throw const FileSystemException( + 'Cannot clean the library without folder access', + ); } } try { @@ -777,8 +839,8 @@ class LocalLibraryNotifier extends Notifier { } return removed; } finally { - if (didStartSecurityAccess) { - await PlatformBridge.stopAccessingIosBookmark(); + if (securityAccess != null) { + await PlatformBridge.stopAccessingIosBookmark(securityAccess); } } } diff --git a/lib/screens/queue_tab.dart b/lib/screens/queue_tab.dart index 15e2482a..5633aae5 100644 --- a/lib/screens/queue_tab.dart +++ b/lib/screens/queue_tab.dart @@ -481,7 +481,7 @@ class _QueueTabState extends ConsumerState { _QueueLibraryPageData? nonEmptyFallback, }) { void storePage(_QueueLibraryPageData data) { - final cached = _queueLibraryPageDataCache[request]; + final cached = _cachedQueueLibraryPageAt(request, request.offset); if (shouldRetainQueueLibraryPageSnapshot( currentIsEmpty: data.isEmpty, cachedHasContent: cached != null && !cached.isEmpty, @@ -489,6 +489,17 @@ class _QueueTabState extends ConsumerState { )) { return; } + final staleRequests = _queueLibraryPageDataCache.keys + .where( + (cachedRequest) => + cachedRequest.offset == request.offset && + cachedRequest != request && + _sameQueueLibraryPageScope(cachedRequest, request), + ) + .toList(growable: false); + for (final staleRequest in staleRequests) { + _queueLibraryPageDataCache.remove(staleRequest); + } _queueLibraryPageDataCache[request] = data; _trimQueueLibraryPageDataCache(protectedRequest: request); } @@ -503,19 +514,7 @@ class _QueueTabState extends ConsumerState { final pages = <_QueueLibraryPageData>[]; for (var offset = 0; offset <= request.offset; offset += _libraryPageSize) { - final page = - _queueLibraryPageDataCache[_QueueLibraryPageRequest( - filterMode: request.filterMode, - limit: request.limit, - offset: offset, - searchQuery: request.searchQuery, - filterSource: request.filterSource, - filterQuality: request.filterQuality, - filterFormat: request.filterFormat, - filterMetadata: request.filterMetadata, - sortMode: request.sortMode, - localLibraryEnabled: request.localLibraryEnabled, - )]; + final page = _cachedQueueLibraryPageAt(request, offset); if (page != null) pages.add(page); } @@ -569,16 +568,37 @@ class _QueueTabState extends ConsumerState { _QueueLibraryPageRequest request, _QueueLibraryPageRequest protectedRequest, ) { - return request.filterMode == protectedRequest.filterMode && + return _sameQueueLibraryPageScope(request, protectedRequest) && request.limit == protectedRequest.limit && - request.offset <= protectedRequest.offset && - request.searchQuery == protectedRequest.searchQuery && - request.filterSource == protectedRequest.filterSource && - request.filterQuality == protectedRequest.filterQuality && - request.filterFormat == protectedRequest.filterFormat && - request.filterMetadata == protectedRequest.filterMetadata && - request.sortMode == protectedRequest.sortMode && - request.localLibraryEnabled == protectedRequest.localLibraryEnabled; + request.offset <= protectedRequest.offset; + } + + bool _sameQueueLibraryPageScope( + _QueueLibraryPageRequest a, + _QueueLibraryPageRequest b, + ) { + return a.filterMode == b.filterMode && + a.limit == b.limit && + a.searchQuery == b.searchQuery && + a.filterSource == b.filterSource && + a.filterQuality == b.filterQuality && + a.filterFormat == b.filterFormat && + a.filterMetadata == b.filterMetadata && + a.sortMode == b.sortMode && + a.localLibraryEnabled == b.localLibraryEnabled; + } + + _QueueLibraryPageData? _cachedQueueLibraryPageAt( + _QueueLibraryPageRequest request, + int offset, + ) { + for (final entry in _queueLibraryPageDataCache.entries) { + if (entry.key.offset == offset && + _sameQueueLibraryPageScope(entry.key, request)) { + return entry.value; + } + } + return null; } void _trimQueueLibraryPageDataCache({ @@ -1292,19 +1312,39 @@ class _QueueTabState extends ConsumerState { nonEmptyFallback: historySnapshotCounts, ); - _QueueLibraryPageRequest pageRequest(String filterMode) => - _QueueLibraryPageRequest( - filterMode: filterMode, - limit: _libraryPageSize, - offset: _libraryPageOffsetFor(filterMode), - searchQuery: _searchQuery, - filterSource: _filterSource, - filterQuality: _filterQuality, - filterFormat: _filterFormat, - filterMetadata: _filterMetadata, - sortMode: _sortMode, - localLibraryEnabled: localLibraryEnabled, - ); + _QueueLibraryPageRequest pageRequest(String filterMode) { + final offset = _libraryPageOffsetFor(filterMode); + final baseRequest = _QueueLibraryPageRequest( + filterMode: filterMode, + limit: _libraryPageSize, + offset: offset, + searchQuery: _searchQuery, + filterSource: _filterSource, + filterQuality: _filterQuality, + filterFormat: _filterFormat, + filterMetadata: _filterMetadata, + sortMode: _sortMode, + localLibraryEnabled: localLibraryEnabled, + ); + if (offset == 0) return baseRequest; + final previousPage = _cachedQueueLibraryPageAt( + baseRequest, + offset - _libraryPageSize, + ); + return _QueueLibraryPageRequest( + filterMode: baseRequest.filterMode, + limit: baseRequest.limit, + offset: baseRequest.offset, + searchQuery: baseRequest.searchQuery, + filterSource: baseRequest.filterSource, + filterQuality: baseRequest.filterQuality, + filterFormat: baseRequest.filterFormat, + filterMetadata: baseRequest.filterMetadata, + sortMode: baseRequest.sortMode, + localLibraryEnabled: baseRequest.localLibraryEnabled, + cursor: previousPage?.nextCursor, + ); + } final activePageRequest = pageRequest(historyFilterMode); final activePageValue = ref.watch( diff --git a/lib/screens/queue_tab_helpers.dart b/lib/screens/queue_tab_helpers.dart index 247df822..44260b0d 100644 --- a/lib/screens/queue_tab_helpers.dart +++ b/lib/screens/queue_tab_helpers.dart @@ -89,6 +89,7 @@ class _QueueLibraryPageRequest { final String? filterMetadata; final String sortMode; final bool localLibraryEnabled; + final QueueLibraryDbCursor? cursor; const _QueueLibraryPageRequest({ required this.filterMode, @@ -101,6 +102,7 @@ class _QueueLibraryPageRequest { required this.filterMetadata, required this.sortMode, required this.localLibraryEnabled, + this.cursor, }); QueueLibraryDbQuery toDbQuery() => QueueLibraryDbQuery( @@ -114,6 +116,7 @@ class _QueueLibraryPageRequest { metadata: filterMetadata, sortMode: sortMode, includeLocal: localLibraryEnabled, + cursor: cursor, ); bool get allowsInMemoryHistoryFallback => @@ -141,7 +144,8 @@ class _QueueLibraryPageRequest { filterFormat == other.filterFormat && filterMetadata == other.filterMetadata && sortMode == other.sortMode && - localLibraryEnabled == other.localLibraryEnabled; + localLibraryEnabled == other.localLibraryEnabled && + cursor == other.cursor; @override int get hashCode => Object.hash( @@ -155,6 +159,7 @@ class _QueueLibraryPageRequest { filterMetadata, sortMode, localLibraryEnabled, + cursor, ); } @@ -219,6 +224,7 @@ class _QueueLibraryPageData { final List localItems; final List<_GroupedAlbum> groupedAlbums; final List<_GroupedLocalAlbum> groupedLocalAlbums; + final QueueLibraryDbCursor? nextCursor; const _QueueLibraryPageData({ this.items = const [], @@ -226,6 +232,7 @@ class _QueueLibraryPageData { this.localItems = const [], this.groupedAlbums = const [], this.groupedLocalAlbums = const [], + this.nextCursor, }); bool get isEmpty => @@ -331,6 +338,7 @@ class _QueueLibraryPageData { localItems: localItems, groupedAlbums: groupedAlbums, groupedLocalAlbums: groupedLocalAlbums, + nextCursor: pages.last.nextCursor, ); } @@ -403,7 +411,10 @@ final _queueLibraryPageProvider = FutureProvider.autoDispose } final dbQuery = request.toDbQuery(); if (request.filterMode == 'albums') { - final rows = await LibraryDatabase.instance.getQueueAlbumPage(dbQuery); + final page = await LibraryDatabase.instance.getQueueAlbumPageResult( + dbQuery, + ); + final rows = page.rows; final groupedAlbums = <_GroupedAlbum>[]; final groupedLocalAlbums = <_GroupedLocalAlbum>[]; for (final row in rows) { @@ -439,10 +450,14 @@ final _queueLibraryPageProvider = FutureProvider.autoDispose return _QueueLibraryPageData( groupedAlbums: groupedAlbums, groupedLocalAlbums: groupedLocalAlbums, + nextCursor: page.nextCursor, ); } - final rows = await LibraryDatabase.instance.getQueueTrackPage(dbQuery); + final page = await LibraryDatabase.instance.getQueueTrackPageResult( + dbQuery, + ); + final rows = page.rows; final items = []; final historyItems = []; final localItems = []; @@ -463,6 +478,7 @@ final _queueLibraryPageProvider = FutureProvider.autoDispose items: items, historyItems: historyItems, localItems: localItems, + nextCursor: page.nextCursor, ); }); diff --git a/lib/screens/track_metadata_edit_sheet.dart b/lib/screens/track_metadata_edit_sheet.dart index c7a1d3aa..eed04e01 100644 --- a/lib/screens/track_metadata_edit_sheet.dart +++ b/lib/screens/track_metadata_edit_sheet.dart @@ -105,6 +105,9 @@ class _MetadataCandidateArtworkState extends State<_MetadataCandidateArtwork> { width: 56, height: 56, fit: BoxFit.cover, + cacheWidth: (56 * MediaQuery.devicePixelRatioOf(context)).round(), + cacheHeight: (56 * MediaQuery.devicePixelRatioOf(context)).round(), + filterQuality: FilterQuality.low, ) : coverUrl != null ? CachedCoverImage( @@ -2434,6 +2437,11 @@ class _EditMetadataSheetState extends State<_EditMetadataSheet> { width: 112, height: 112, fit: BoxFit.cover, + cacheWidth: (112 * MediaQuery.devicePixelRatioOf(context)) + .round(), + cacheHeight: (112 * MediaQuery.devicePixelRatioOf(context)) + .round(), + filterQuality: FilterQuality.low, semanticLabel: context.l10n.editMetadataAutoFillCoverAvailable, errorBuilder: (_, _, _) => const SizedBox.shrink(), @@ -2710,6 +2718,11 @@ class _EditMetadataSheetState extends State<_EditMetadataSheet> { height: 160, width: 160, fit: BoxFit.cover, + cacheWidth: (160 * MediaQuery.devicePixelRatioOf(context)) + .round(), + cacheHeight: (160 * MediaQuery.devicePixelRatioOf(context)) + .round(), + filterQuality: FilterQuality.low, errorBuilder: (_, _, _) => Container( width: 160, height: 160, diff --git a/lib/services/conversion_library_service.dart b/lib/services/conversion_library_service.dart index 5442f9b8..5caa9a3f 100644 --- a/lib/services/conversion_library_service.dart +++ b/lib/services/conversion_library_service.dart @@ -111,7 +111,7 @@ class ConversionLibraryService { final converted = source.toJson() ..['id'] = convertedLibraryItemId(source.id, newFilePath) ..['filePath'] = newFilePath - ..['downloadedAt'] = DateTime.now().toIso8601String() + ..['downloadedAt'] = DateTime.now().toUtc().toIso8601String() ..['quality'] = newQuality ..['format'] = normalizedConvertedAudioFormat(targetFormat) ..['bitrate'] = convertedBitrate diff --git a/lib/services/cover_download_service.dart b/lib/services/cover_download_service.dart index 029ad8b3..00a5c130 100644 --- a/lib/services/cover_download_service.dart +++ b/lib/services/cover_download_service.dart @@ -31,7 +31,7 @@ class CoverDownloadService { : safeBaseName.trim(); final tempDir = await Directory.systemTemp.createTemp('save_cover_'); final tempPath = p.join(tempDir.path, 'cover.image'); - var iosBookmarkActive = false; + IosSecurityScopedAccess? iosBookmarkAccess; try { final download = await PlatformBridge.downloadCoverToFile( @@ -74,13 +74,13 @@ class CoverDownloadService { var outputDirectory = settings.downloadDirectory.trim(); if (Platform.isIOS && settings.downloadDirectoryBookmark.isNotEmpty) { - final resolved = await PlatformBridge.startAccessingIosBookmark( + iosBookmarkAccess = await PlatformBridge.startAccessingIosBookmark( settings.downloadDirectoryBookmark, ); + final resolved = iosBookmarkAccess?.path; if (resolved == null || resolved.trim().isEmpty) { throw const FileSystemException('No storage access'); } - iosBookmarkActive = true; outputDirectory = resolved.trim(); } if (outputDirectory.isEmpty) { @@ -100,8 +100,8 @@ class CoverDownloadService { location: outputPath, ); } finally { - if (iosBookmarkActive) { - await PlatformBridge.stopAccessingIosBookmark(); + if (iosBookmarkAccess != null) { + await PlatformBridge.stopAccessingIosBookmark(iosBookmarkAccess); } try { if (await tempDir.exists()) await tempDir.delete(recursive: true); diff --git a/lib/services/history_database.dart b/lib/services/history_database.dart index bb0ab487..af2217e3 100644 --- a/lib/services/history_database.dart +++ b/lib/services/history_database.dart @@ -65,7 +65,7 @@ class HistoryBatchLookupRequest { } class HistoryDatabase { - static const int schemaVersion = 10; + static const int schemaVersion = 11; static final HistoryDatabase instance = HistoryDatabase._init(); static final sqlite.SingleFlightInitializer _database = sqlite.SingleFlightInitializer(); @@ -123,7 +123,14 @@ class HistoryDatabase { isrc_norm TEXT, match_key TEXT, album_key TEXT, - search_text TEXT + search_text TEXT, + sort_track TEXT, + sort_artist TEXT, + sort_album TEXT, + sort_album_artist TEXT, + sort_genre TEXT, + sort_release TEXT, + sort_added INTEGER ) '''); @@ -139,6 +146,7 @@ class HistoryDatabase { 'CREATE INDEX IF NOT EXISTS idx_history_track_artist ON history(track_name, artist_name)', ); await _createNormalizedIndexes(db); + await _createQueueIndexes(db); await _createPathKeyTable(db); _log.i('Database schema created with indexes'); @@ -204,6 +212,23 @@ class HistoryDatabase { await _backfillNormalizedColumns(db); await _createNormalizedIndexes(db); } + if (oldVersion < 11) { + await sqlite.addColumnIfMissing(db, 'history', 'sort_track', 'TEXT'); + await sqlite.addColumnIfMissing(db, 'history', 'sort_artist', 'TEXT'); + await sqlite.addColumnIfMissing(db, 'history', 'sort_album', 'TEXT'); + await sqlite.addColumnIfMissing( + db, + 'history', + 'sort_album_artist', + 'TEXT', + ); + await sqlite.addColumnIfMissing(db, 'history', 'sort_genre', 'TEXT'); + await sqlite.addColumnIfMissing(db, 'history', 'sort_release', 'TEXT'); + await sqlite.addColumnIfMissing(db, 'history', 'sort_added', 'INTEGER'); + await _backfillQueueSortColumns(db); + await _createQueueIndexes(db); + _log.i('Added persisted queue sort columns'); + } } static String normalizeLookupText(String? value) => @@ -262,6 +287,27 @@ class HistoryDatabase { ); } + Future _createQueueIndexes(DatabaseExecutor db) async { + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_history_queue_added ON history(sort_added DESC, sort_track, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_history_queue_track ON history(sort_track, sort_artist, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_history_queue_artist ON history(sort_artist, sort_track, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_history_queue_album ON history(sort_album, sort_track, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_history_queue_genre ON history(sort_genre, sort_track, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_history_queue_release ON history(sort_release, sort_track, id)', + ); + } + Future _backfillNormalizedColumns(Database db) async { final rows = await db.query( 'history', @@ -322,6 +368,65 @@ class HistoryDatabase { }; } + Map _queueSortColumns({ + required String? trackName, + required String? artistName, + required String? albumName, + required String? albumArtist, + required String? genre, + required String? releaseDate, + required Object? downloadedAt, + }) { + final parsedDownloadedAt = downloadedAt is DateTime + ? downloadedAt + : DateTime.tryParse(downloadedAt?.toString() ?? ''); + return { + 'sort_track': normalizeLookupText(trackName), + 'sort_artist': normalizeLookupText(artistName), + 'sort_album': normalizeLookupText(albumName), + 'sort_album_artist': normalizeLookupText( + (albumArtist ?? '').trim().isEmpty ? artistName : albumArtist, + ), + 'sort_genre': normalizeLookupText(genre), + 'sort_release': releaseDate?.trim() ?? '', + 'sort_added': parsedDownloadedAt?.millisecondsSinceEpoch ?? 0, + }; + } + + Future _backfillQueueSortColumns(Database db) async { + final rows = await db.query( + 'history', + columns: [ + 'id', + 'track_name', + 'artist_name', + 'album_name', + 'album_artist', + 'genre', + 'release_date', + 'downloaded_at', + ], + ); + final batch = db.batch(); + for (final row in rows) { + batch.update( + 'history', + _queueSortColumns( + trackName: row['track_name'] as String?, + artistName: row['artist_name'] as String?, + albumName: row['album_name'] as String?, + albumArtist: row['album_artist'] as String?, + genre: row['genre'] as String?, + releaseDate: row['release_date'] as String?, + downloadedAt: row['downloaded_at'], + ), + where: 'id = ?', + whereArgs: [row['id']], + ); + } + await batch.commit(noResult: true); + } + Future _createPathKeyTable(DatabaseExecutor db) => sqlite.createPathKeyTable(db, 'history_path_keys'); @@ -474,6 +579,10 @@ class HistoryDatabase { } Map _jsonToDbRow(Map json) { + final downloadedAt = json['downloadedAt']; + final parsedDownloadedAt = downloadedAt is DateTime + ? downloadedAt + : DateTime.tryParse(downloadedAt?.toString() ?? ''); final row = { 'id': json['id'], 'track_name': json['trackName'], @@ -488,7 +597,9 @@ class HistoryDatabase { 'saf_file_name': json['safFileName'], 'saf_repaired': json['safRepaired'] == true ? 1 : 0, 'service': json['service'], - 'downloaded_at': json['downloadedAt'], + 'downloaded_at': + parsedDownloadedAt?.toUtc().toIso8601String() ?? + downloadedAt?.toString(), 'isrc': json['isrc'], 'spotify_id': json['spotifyId'], 'track_number': json['trackNumber'], @@ -507,6 +618,17 @@ class HistoryDatabase { 'label': json['label'], 'copyright': json['copyright'], }; + row.addAll( + _queueSortColumns( + trackName: json['trackName'] as String?, + artistName: json['artistName'] as String?, + albumName: json['albumName'] as String?, + albumArtist: json['albumArtist'] as String?, + genre: json['genre'] as String?, + releaseDate: json['releaseDate'] as String?, + downloadedAt: parsedDownloadedAt, + ), + ); row.addAll( _normalizedColumns( spotifyId: json['spotifyId'] as String?, @@ -599,7 +721,7 @@ class HistoryDatabase { final db = await database; final rows = await db.query( 'history', - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', limit: limit, offset: offset, ); @@ -634,7 +756,7 @@ class HistoryDatabase { 'history', where: 'match_key = ?', whereArgs: [key], - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', limit: 1, ); if (rows.isEmpty) return null; @@ -664,7 +786,7 @@ class HistoryDatabase { FROM history h JOIN history_path_keys hpk ON hpk.item_id = h.id WHERE hpk.path_key IN ($placeholders) - ORDER BY h.downloaded_at DESC + ORDER BY h.sort_added DESC, h.id DESC LIMIT 1 ''', pathKeys); if (rows.isEmpty) return null; @@ -677,7 +799,7 @@ class HistoryDatabase { 'history', where: 'spotify_id = ?', whereArgs: [spotifyId], - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', limit: 1, ); if (rows.isEmpty) return null; @@ -690,7 +812,7 @@ class HistoryDatabase { 'history', where: 'isrc = ?', whereArgs: [isrc], - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', limit: 1, ); if (rows.isEmpty) return null; @@ -712,7 +834,7 @@ class HistoryDatabase { where: 'spotify_id IN ($placeholders) OR spotify_id_norm IN ($placeholders)', whereArgs: [...spotifyCandidates, ...normalized], - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', limit: 1, ); if (rows.isNotEmpty) return _dbRowToJson(rows.first); @@ -725,7 +847,7 @@ class HistoryDatabase { columns: columns, where: 'isrc_norm = ?', whereArgs: [isrcNorm], - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', limit: 1, ); if (rows.isNotEmpty) return _dbRowToJson(rows.first); @@ -738,7 +860,7 @@ class HistoryDatabase { columns: columns, where: 'match_key = ?', whereArgs: [matchKey], - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', limit: 1, ); if (rows.isNotEmpty) return _dbRowToJson(rows.first); @@ -771,7 +893,7 @@ class HistoryDatabase { rawValues: rawValues, destination: destination, mapRow: _dbRowToJson, - orderBy: 'downloaded_at DESC', + orderBy: 'sort_added DESC, id DESC', ); } @@ -997,7 +1119,7 @@ class HistoryDatabase { 'saf_file_name', ], where: 'file_path IS NOT NULL AND file_path != ""', - orderBy: 'downloaded_at DESC, id DESC', + orderBy: 'sort_added DESC, id DESC', limit: limit, offset: offset, ); diff --git a/lib/services/library_database.dart b/lib/services/library_database.dart index 6c09498e..57603d9c 100644 --- a/lib/services/library_database.dart +++ b/lib/services/library_database.dart @@ -16,7 +16,7 @@ final _log = AppLogger('LibraryDatabase'); class LibraryDatabase { static final LibraryDatabase instance = LibraryDatabase._init(); - static const int schemaVersion = 9; + static const int schemaVersion = 10; static const int audioMetadataScanVersion = 1; static final sqlite.SingleFlightInitializer _database = sqlite.SingleFlightInitializer(); @@ -87,7 +87,11 @@ class LibraryDatabase { album_name_norm TEXT, album_artist_norm TEXT, match_key TEXT, - album_key TEXT + album_key TEXT, + search_text TEXT, + sort_genre TEXT, + sort_release TEXT, + sort_added INTEGER ) '''); @@ -102,6 +106,7 @@ class LibraryDatabase { 'CREATE INDEX idx_library_file_path ON library(file_path)', ); await _createNormalizedIndexes(db); + await _createQueueIndexes(db); await _createPathKeyTable(db); _log.i('Library database schema created with indexes'); @@ -173,6 +178,15 @@ class LibraryDatabase { ); _log.i('Marked existing rows for one-time audio metadata rescan'); } + if (oldVersion < 10) { + await sqlite.addColumnIfMissing(db, 'library', 'search_text', 'TEXT'); + await sqlite.addColumnIfMissing(db, 'library', 'sort_genre', 'TEXT'); + await sqlite.addColumnIfMissing(db, 'library', 'sort_release', 'TEXT'); + await sqlite.addColumnIfMissing(db, 'library', 'sort_added', 'INTEGER'); + await _backfillQueueColumns(db); + await _createQueueIndexes(db); + _log.i('Added persisted queue sort/search columns'); + } } Future _createPathKeyTable(DatabaseExecutor db) => @@ -217,6 +231,33 @@ class LibraryDatabase { ); } + Future _createQueueIndexes(DatabaseExecutor db) async { + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_library_queue_added ' + 'ON library(sort_added DESC, track_name_norm, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_library_queue_track ' + 'ON library(track_name_norm, artist_name_norm, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_library_queue_artist ' + 'ON library(artist_name_norm, track_name_norm, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_library_queue_album ' + 'ON library(album_name_norm, track_name_norm, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_library_queue_genre ' + 'ON library(sort_genre, track_name_norm, id)', + ); + await db.execute( + 'CREATE INDEX IF NOT EXISTS idx_library_queue_release ' + 'ON library(sort_release, track_name_norm, id)', + ); + } + Future _backfillNormalizedColumns(Database db) async { final rows = await db.query( 'library', @@ -269,7 +310,77 @@ class LibraryDatabase { }; } + Map _queueColumns({ + required String? trackName, + required String? artistName, + required String? albumName, + required String? albumArtist, + required String? genre, + required String? releaseDate, + required int? fileModTime, + required String? scannedAt, + }) { + final trackNorm = normalizeLookupText(trackName); + final artistNorm = normalizeLookupText(artistName); + final albumNorm = normalizeLookupText(albumName); + final albumArtistNorm = normalizeLookupText( + (albumArtist ?? '').trim().isEmpty ? artistName : albumArtist, + ); + return { + 'search_text': [ + trackNorm, + artistNorm, + albumNorm, + albumArtistNorm, + ].where((value) => value.isNotEmpty).join(' '), + 'sort_genre': normalizeLookupText(genre), + 'sort_release': releaseDate?.trim() ?? '', + 'sort_added': + fileModTime ?? + DateTime.tryParse(scannedAt ?? '')?.millisecondsSinceEpoch ?? + 0, + }; + } + + Future _backfillQueueColumns(Database db) async { + final rows = await db.query( + 'library', + columns: [ + 'id', + 'track_name', + 'artist_name', + 'album_name', + 'album_artist', + 'genre', + 'release_date', + 'file_mod_time', + 'scanned_at', + ], + ); + final batch = db.batch(); + for (final row in rows) { + batch.update( + 'library', + _queueColumns( + trackName: row['track_name'] as String?, + artistName: row['artist_name'] as String?, + albumName: row['album_name'] as String?, + albumArtist: row['album_artist'] as String?, + genre: row['genre'] as String?, + releaseDate: row['release_date'] as String?, + fileModTime: (row['file_mod_time'] as num?)?.toInt(), + scannedAt: row['scanned_at'] as String?, + ), + where: 'id = ?', + whereArgs: [row['id']], + ); + } + await batch.commit(noResult: true); + } + Map _jsonToDbRow(Map json) { + final fileModTime = (json['fileModTime'] as num?)?.toInt(); + final scannedAt = json['scannedAt'] as String?; final row = { 'id': json['id'], 'track_name': json['trackName'], @@ -299,6 +410,18 @@ class LibraryDatabase { (json['audioMetadataScanVersion'] as num?)?.toInt() ?? audioMetadataScanVersion, }; + row.addAll( + _queueColumns( + trackName: json['trackName'] as String?, + artistName: json['artistName'] as String?, + albumName: json['albumName'] as String?, + albumArtist: json['albumArtist'] as String?, + genre: json['genre'] as String?, + releaseDate: json['releaseDate'] as String?, + fileModTime: fileModTime, + scannedAt: scannedAt, + ), + ); row.addAll( _normalizedColumns( trackName: json['trackName'] as String? ?? '', @@ -406,6 +529,52 @@ class LibraryDatabase { _log.i('Replaced library with ${items.length} items'); } + /// Atomically replaces the Library while consuming bounded scan batches. + /// The stream may represent tens of thousands of tracks without requiring a + /// second full list of models/maps on the Dart heap. + Future replaceAllStream( + Stream> items, { + int batchSize = 300, + }) async { + if (batchSize <= 0) { + throw ArgumentError.value(batchSize, 'batchSize', 'Must be positive'); + } + final db = await database; + var inserted = 0; + await db.transaction((txn) async { + await txn.delete('library_path_keys'); + await txn.delete('library'); + + var batch = txn.batch(); + var pending = 0; + Future flush() async { + if (pending == 0) return; + await batch.commit(noResult: true); + batch = txn.batch(); + pending = 0; + } + + await for (final json in items) { + final id = json['id'] as String?; + if (id == null || id.trim().isEmpty) { + throw const FormatException('Library scan row has no valid id'); + } + batch.insert( + 'library', + _jsonToDbRow(json), + conflictAlgorithm: ConflictAlgorithm.replace, + ); + _putPathKeysInBatch(batch, id, json['filePath'] as String?); + inserted++; + pending++; + if (pending >= batchSize) await flush(); + } + await flush(); + }); + _log.i('Stream-replaced library with $inserted items'); + return inserted; + } + Future>> getAll({int? limit, int? offset}) async { final db = await database; final rows = await db.query( @@ -441,26 +610,46 @@ class LibraryDatabase { Future>> getQueueTrackPage( QueueLibraryDbQuery request, + ) async { + return (await getQueueTrackPageResult(request)).rows; + } + + Future getQueueTrackPageResult( + QueueLibraryDbQuery request, ) async { final db = await database; await _ensureHistoryAttached(db); final args = []; - final unionSql = _queueTrackUnionSql(request, args); + final orderTerms = _queueTrackOrderTerms(request.sortMode); + final usesCursor = + request.cursor != null && + request.cursor!.values.length == orderTerms.length; + final unionSql = _queueTrackUnionSql( + request, + args, + orderTerms: orderTerms, + usesCursor: usesCursor, + ); final rows = await db.rawQuery( ''' SELECT * FROM ($unionSql) ORDER BY ${_queueTrackOrderBy(request.sortMode)} - LIMIT ? OFFSET ? + LIMIT ? ${usesCursor ? '' : 'OFFSET ?'} ''', - [...args, request.limit, request.offset], + [...args, request.limit, if (!usesCursor) request.offset], + ); + return QueueLibraryDbPage( + rows: rows.map(_queueTrackRowToJson).toList(growable: false), + nextCursor: _queueCursorFromRow(rows.lastOrNull, orderTerms), ); - return rows.map(_queueTrackRowToJson).toList(growable: false); } Future getQueueCounts(QueueLibraryDbQuery request) async { final db = await database; await _ensureHistoryAttached(db); + final fastCounts = await _getUnfilteredQueueCounts(db, request); + if (fastCounts != null) return fastCounts; final parts = []; final args = []; @@ -539,23 +728,115 @@ class LibraryDatabase { ); } + /// The default Library badges do not need a row-by-row join against album + /// counts. Aggregate the covering album-key indexes directly and reserve the + /// more expensive filtered query for active search/quality/metadata filters. + Future _getUnfilteredQueueCounts( + Database db, + QueueLibraryDbQuery request, + ) async { + if (normalizeLookupText(request.searchQuery).isNotEmpty || + request.quality != null || + request.format != null || + request.metadata != null) { + return null; + } + final source = request.source; + if (source != null && source != 'downloaded' && source != 'local') { + return null; + } + + final parts = []; + if (source != 'local') { + parts.add(''' + SELECT + COALESCE(SUM(track_count), 0) AS all_count, + COALESCE(SUM(CASE WHEN track_count > 1 THEN 1 ELSE 0 END), 0) AS album_count, + COALESCE(SUM(CASE WHEN track_count = 1 THEN 1 ELSE 0 END), 0) AS single_count + FROM ( + SELECT album_key, COUNT(*) AS track_count + FROM history_db.history + GROUP BY album_key + ) + '''); + } + if (request.includeLocal && source != 'downloaded') { + parts.add(''' + SELECT + COALESCE(SUM(track_count), 0) AS all_count, + COALESCE(SUM(CASE WHEN track_count > 1 THEN 1 ELSE 0 END), 0) AS album_count, + COALESCE(SUM(CASE WHEN track_count = 1 THEN 1 ELSE 0 END), 0) AS single_count + FROM ( + SELECT l.album_key, COUNT(*) AS track_count + FROM library l + WHERE NOT EXISTS ( + SELECT 1 + FROM library_path_keys lpk + JOIN history_db.history_path_keys hpk ON hpk.path_key = lpk.path_key + WHERE lpk.item_id = l.id + ) + GROUP BY l.album_key + ) + '''); + } + if (parts.isEmpty) { + return const QueueLibraryCounts( + allTrackCount: 0, + albumCount: 0, + singleTrackCount: 0, + ); + } + + final rows = await db.rawQuery(''' + SELECT + COALESCE(SUM(all_count), 0) AS all_count, + COALESCE(SUM(album_count), 0) AS album_count, + COALESCE(SUM(single_count), 0) AS single_count + FROM (${parts.join(' UNION ALL ')}) + '''); + final row = rows.isEmpty ? const {} : rows.first; + return QueueLibraryCounts( + allTrackCount: (row['all_count'] as num?)?.toInt() ?? 0, + albumCount: (row['album_count'] as num?)?.toInt() ?? 0, + singleTrackCount: (row['single_count'] as num?)?.toInt() ?? 0, + ); + } + Future>> getQueueAlbumPage( QueueLibraryDbQuery request, + ) async { + return (await getQueueAlbumPageResult(request)).rows; + } + + Future getQueueAlbumPageResult( + QueueLibraryDbQuery request, ) async { final db = await database; await _ensureHistoryAttached(db); final args = []; - final unionSql = _queueAlbumUnionSql(request, args); + final orderTerms = _queueAlbumOrderTerms(request.sortMode); + final usesCursor = + request.cursor != null && + request.cursor!.values.length == orderTerms.length; + final unionSql = _queueAlbumUnionSql( + request, + args, + orderTerms: orderTerms, + usesCursor: usesCursor, + ); final rows = await db.rawQuery( ''' SELECT * FROM ($unionSql) ORDER BY ${_queueAlbumOrderBy(request.sortMode)} - LIMIT ? OFFSET ? + LIMIT ? ${usesCursor ? '' : 'OFFSET ?'} ''', - [...args, request.limit, request.offset], + [...args, request.limit, if (!usesCursor) request.offset], + ); + return QueueLibraryDbPage( + rows: rows.toList(growable: false), + nextCursor: _queueCursorFromRow(rows.lastOrNull, orderTerms), ); - return rows.toList(growable: false); } Future>> getQueueLocalAlbumTracks( @@ -1081,7 +1362,7 @@ class LibraryDatabase { for (final entry in fileModTimes.entries) { batch.update( 'library', - {'file_mod_time': entry.value}, + {'file_mod_time': entry.value, 'sort_added': entry.value}, where: 'file_path = ?', whereArgs: [entry.key], ); diff --git a/lib/services/library_database_models.dart b/lib/services/library_database_models.dart index a6a4924e..fcfd473e 100644 --- a/lib/services/library_database_models.dart +++ b/lib/services/library_database_models.dart @@ -232,6 +232,7 @@ class QueueLibraryDbQuery { final String? metadata; final String sortMode; final bool includeLocal; + final QueueLibraryDbCursor? cursor; const QueueLibraryDbQuery({ this.limit = 100, @@ -244,9 +245,44 @@ class QueueLibraryDbQuery { this.metadata, this.sortMode = 'latest', this.includeLocal = true, + this.cursor, }); } +/// Opaque seek cursor for queue Library pagination. +/// +/// Values follow the active SQL order (including its unique tie-breaker), so +/// later pages can seek from the last row instead of making SQLite discard an +/// ever-growing OFFSET prefix. +class QueueLibraryDbCursor { + final List values; + + const QueueLibraryDbCursor(this.values); + + @override + bool operator ==(Object other) { + if (identical(this, other)) return true; + if (other is! QueueLibraryDbCursor || + other.values.length != values.length) { + return false; + } + for (var i = 0; i < values.length; i++) { + if (values[i] != other.values[i]) return false; + } + return true; + } + + @override + int get hashCode => Object.hashAll(values); +} + +class QueueLibraryDbPage { + final List> rows; + final QueueLibraryDbCursor? nextCursor; + + const QueueLibraryDbPage({required this.rows, required this.nextCursor}); +} + class QueueLibraryCounts { final int allTrackCount; final int albumCount; diff --git a/lib/services/library_database_queue_sql.dart b/lib/services/library_database_queue_sql.dart index 0a5f3871..8f2b1319 100644 --- a/lib/services/library_database_queue_sql.dart +++ b/lib/services/library_database_queue_sql.dart @@ -2,8 +2,20 @@ part of 'library_database.dart'; // SQL builders for the queue tab's history+local union queries. +class _QueueOrderTerm { + final String column; + final bool descending; + + const _QueueOrderTerm(this.column, {this.descending = false}); +} + extension _LibraryDbQueueSql on LibraryDatabase { - String _queueTrackUnionSql(QueueLibraryDbQuery request, List args) { + String _queueTrackUnionSql( + QueueLibraryDbQuery request, + List args, { + required List<_QueueOrderTerm> orderTerms, + required bool usesCursor, + }) { final parts = []; if (request.source != 'local') { final where = []; @@ -18,7 +30,8 @@ extension _LibraryDbQueueSql on LibraryDatabase { ) '''); } - parts.add(''' + final selectSql = + ''' SELECT 'downloaded' AS queue_source, 'dl_' || h.id AS unified_id, @@ -56,15 +69,24 @@ extension _LibraryDbQueueSql on LibraryDatabase { NULL AS file_mod_time, h.bitrate, h.format, - LOWER(h.track_name) AS sort_track, - LOWER(h.artist_name) AS sort_artist, - LOWER(h.album_name) AS sort_album, - LOWER(COALESCE(h.genre, '')) AS sort_genre, - h.release_date AS sort_release, - CAST(strftime('%s', h.downloaded_at) AS INTEGER) * 1000 AS sort_added + h.sort_track, + h.sort_artist, + h.sort_album, + h.sort_genre, + h.sort_release, + h.sort_added FROM history_db.history h ${where.isEmpty ? '' : 'WHERE ${where.join(' AND ')}'} - '''); + '''; + parts.add( + _boundedQueuePart( + selectSql, + request, + args, + orderTerms, + usesCursor: usesCursor, + ), + ); } if (request.includeLocal && request.source != 'downloaded') { @@ -95,7 +117,8 @@ extension _LibraryDbQueueSql on LibraryDatabase { ) '''); } - parts.add(''' + final selectSql = + ''' SELECT 'local' AS queue_source, 'local_' || l.id AS unified_id, @@ -136,12 +159,21 @@ extension _LibraryDbQueueSql on LibraryDatabase { l.track_name_norm AS sort_track, l.artist_name_norm AS sort_artist, l.album_name_norm AS sort_album, - LOWER(COALESCE(l.genre, '')) AS sort_genre, - l.release_date AS sort_release, - COALESCE(l.file_mod_time, CAST(strftime('%s', l.scanned_at) AS INTEGER) * 1000) AS sort_added + l.sort_genre, + l.sort_release, + l.sort_added FROM library l ${where.isEmpty ? '' : 'WHERE ${where.join(' AND ')}'} - '''); + '''; + parts.add( + _boundedQueuePart( + selectSql, + request, + args, + orderTerms, + usesCursor: usesCursor, + ), + ); } if (parts.isEmpty) { @@ -195,12 +227,18 @@ extension _LibraryDbQueueSql on LibraryDatabase { return parts.join(' UNION ALL '); } - String _queueAlbumUnionSql(QueueLibraryDbQuery request, List args) { + String _queueAlbumUnionSql( + QueueLibraryDbQuery request, + List args, { + required List<_QueueOrderTerm> orderTerms, + required bool usesCursor, + }) { final parts = []; if (request.source != 'local') { final where = []; _appendQueueHistoryFilters(where, args, request); - parts.add(''' + final selectSql = + ''' SELECT 'downloaded' AS queue_source, c.album_key, @@ -211,16 +249,16 @@ extension _LibraryDbQueueSql on LibraryDatabase { MAX(h.file_path) AS sample_file_path, COUNT(*) AS track_count, c.latest_added AS sort_added, - MIN(LOWER(COALESCE(h.album_name, ''))) AS sort_album, - MIN(LOWER(COALESCE(h.album_artist, h.artist_name, ''))) AS sort_artist, - MAX(h.release_date) AS sort_release, - MAX(LOWER(COALESCE(h.genre, ''))) AS sort_genre + MIN(COALESCE(h.sort_album, '')) AS sort_album, + MIN(COALESCE(h.sort_album_artist, '')) AS sort_artist, + COALESCE(MAX(h.release_date), '') AS sort_release, + COALESCE(MAX(h.sort_genre), '') AS sort_genre FROM history_db.history h JOIN ( SELECT album_key, COUNT(*) AS track_count, - MAX(CAST(strftime('%s', downloaded_at) AS INTEGER) * 1000) AS latest_added + MAX(COALESCE(sort_added, 0)) AS latest_added FROM history_db.history GROUP BY album_key HAVING COUNT(*) > 1 @@ -228,7 +266,16 @@ extension _LibraryDbQueueSql on LibraryDatabase { ON c.album_key = h.album_key ${where.isEmpty ? '' : 'WHERE ${where.join(' AND ')}'} GROUP BY c.album_key - '''); + '''; + parts.add( + _boundedQueuePart( + selectSql, + request, + args, + orderTerms, + usesCursor: usesCursor, + ), + ); } if (request.includeLocal && request.source != 'downloaded') { @@ -243,7 +290,8 @@ extension _LibraryDbQueueSql on LibraryDatabase { ''', ]; _appendQueueLocalFilters(where, args, request); - parts.add(''' + final selectSql = + ''' SELECT 'local' AS queue_source, c.album_key, @@ -256,14 +304,14 @@ extension _LibraryDbQueueSql on LibraryDatabase { c.latest_added AS sort_added, MIN(l.album_name_norm) AS sort_album, MIN(l.album_artist_norm) AS sort_artist, - MAX(l.release_date) AS sort_release, - MAX(LOWER(COALESCE(l.genre, ''))) AS sort_genre + COALESCE(MAX(l.release_date), '') AS sort_release, + COALESCE(MAX(l.sort_genre), '') AS sort_genre FROM library l JOIN ( SELECT album_key, COUNT(*) AS track_count, - MAX(COALESCE(file_mod_time, CAST(strftime('%s', scanned_at) AS INTEGER) * 1000)) AS latest_added + MAX(COALESCE(sort_added, 0)) AS latest_added FROM library WHERE NOT EXISTS ( SELECT 1 @@ -276,7 +324,16 @@ extension _LibraryDbQueueSql on LibraryDatabase { ) c ON c.album_key = l.album_key ${where.isEmpty ? '' : 'WHERE ${where.join(' AND ')}'} GROUP BY c.album_key - '''); + '''; + parts.add( + _boundedQueuePart( + selectSql, + request, + args, + orderTerms, + usesCursor: usesCursor, + ), + ); } if (parts.isEmpty) { @@ -339,15 +396,8 @@ extension _LibraryDbQueueSql on LibraryDatabase { final query = LibraryDatabase.normalizeLookupText(request.searchQuery); if (query.isNotEmpty) { final like = '%${_escapeLikePattern(query)}%'; - where.add(''' - ( - l.track_name_norm LIKE ? ESCAPE '\\' OR - l.artist_name_norm LIKE ? ESCAPE '\\' OR - l.album_name_norm LIKE ? ESCAPE '\\' OR - l.album_artist_norm LIKE ? ESCAPE '\\' - ) - '''); - args.addAll([like, like, like, like]); + where.add("l.search_text LIKE ? ESCAPE '\\'"); + args.add(like); } _appendQueueCommonFilters( where, @@ -472,37 +522,226 @@ extension _LibraryDbQueueSql on LibraryDatabase { } String _queueTrackOrderBy(String sortMode) { - return switch (sortMode) { - 'oldest' => 'sort_added ASC, sort_track ASC', - 'a-z' => 'sort_track ASC, sort_artist ASC', - 'z-a' => 'sort_track DESC, sort_artist DESC', - 'artist-asc' => 'sort_artist ASC, sort_track ASC', - 'artist-desc' => 'sort_artist DESC, sort_track ASC', - 'album-asc' => 'sort_album ASC, sort_track ASC', - 'album-desc' => 'sort_album DESC, sort_track ASC', - 'release-oldest' => 'sort_release ASC, sort_track ASC', - 'release-newest' => 'sort_release DESC, sort_track ASC', - 'genre-asc' => 'sort_genre ASC, sort_track ASC', - 'genre-desc' => 'sort_genre DESC, sort_track ASC', - _ => 'sort_added DESC, sort_track ASC', - }; + return _queueOrderBy(_queueTrackOrderTerms(sortMode)); } String _queueAlbumOrderBy(String sortMode) { + return _queueOrderBy(_queueAlbumOrderTerms(sortMode)); + } + + String _boundedQueuePart( + String selectSql, + QueueLibraryDbQuery request, + List args, + List<_QueueOrderTerm> orderTerms, { + required bool usesCursor, + }) { + final cursorPredicate = usesCursor + ? _queueCursorPredicate(request.cursor, orderTerms, args) + : ''; + final branchLimit = usesCursor + ? request.limit + : request.limit + request.offset; + args.add(branchLimit); + final branchOrder = orderTerms + .where((term) => term.column != 'queue_source') + .toList(growable: false); + return ''' + SELECT * FROM ( + SELECT * + FROM ($selectSql) + ${cursorPredicate.isEmpty ? '' : 'WHERE $cursorPredicate'} + ORDER BY ${_queueOrderBy(branchOrder)} + LIMIT ? + ) + '''; + } + + List<_QueueOrderTerm> _queueTrackOrderTerms(String sortMode) { return switch (sortMode) { - 'oldest' => 'sort_added ASC, sort_album ASC', - 'a-z' || 'album-asc' => 'sort_album ASC, sort_artist ASC', - 'z-a' || 'album-desc' => 'sort_album DESC, sort_artist DESC', - 'artist-asc' => 'sort_artist ASC, sort_album ASC', - 'artist-desc' => 'sort_artist DESC, sort_album ASC', - 'release-oldest' => 'sort_release ASC, sort_album ASC', - 'release-newest' => 'sort_release DESC, sort_album ASC', - 'genre-asc' => 'sort_genre ASC, sort_album ASC', - 'genre-desc' => 'sort_genre DESC, sort_album ASC', - _ => 'sort_added DESC, sort_album ASC', + 'oldest' => const [ + _QueueOrderTerm('sort_added'), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'a-z' => const [ + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('sort_artist'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'z-a' => const [ + _QueueOrderTerm('sort_track', descending: true), + _QueueOrderTerm('sort_artist', descending: true), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'artist-asc' => const [ + _QueueOrderTerm('sort_artist'), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'artist-desc' => const [ + _QueueOrderTerm('sort_artist', descending: true), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'album-asc' => const [ + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'album-desc' => const [ + _QueueOrderTerm('sort_album', descending: true), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'release-oldest' => const [ + _QueueOrderTerm('sort_release'), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'release-newest' => const [ + _QueueOrderTerm('sort_release', descending: true), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'genre-asc' => const [ + _QueueOrderTerm('sort_genre'), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + 'genre-desc' => const [ + _QueueOrderTerm('sort_genre', descending: true), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], + _ => const [ + _QueueOrderTerm('sort_added', descending: true), + _QueueOrderTerm('sort_track'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('id'), + ], }; } + List<_QueueOrderTerm> _queueAlbumOrderTerms(String sortMode) { + return switch (sortMode) { + 'oldest' => const [ + _QueueOrderTerm('sort_added'), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'a-z' || 'album-asc' => const [ + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('sort_artist'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'z-a' || 'album-desc' => const [ + _QueueOrderTerm('sort_album', descending: true), + _QueueOrderTerm('sort_artist', descending: true), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'artist-asc' => const [ + _QueueOrderTerm('sort_artist'), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'artist-desc' => const [ + _QueueOrderTerm('sort_artist', descending: true), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'release-oldest' => const [ + _QueueOrderTerm('sort_release'), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'release-newest' => const [ + _QueueOrderTerm('sort_release', descending: true), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'genre-asc' => const [ + _QueueOrderTerm('sort_genre'), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + 'genre-desc' => const [ + _QueueOrderTerm('sort_genre', descending: true), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + _ => const [ + _QueueOrderTerm('sort_added', descending: true), + _QueueOrderTerm('sort_album'), + _QueueOrderTerm('queue_source'), + _QueueOrderTerm('album_key'), + ], + }; + } + + String _queueOrderBy(List<_QueueOrderTerm> terms) => terms + .map((term) => '${term.column} ${term.descending ? 'DESC' : 'ASC'}') + .join(', '); + + String _queueCursorPredicate( + QueueLibraryDbCursor? cursor, + List<_QueueOrderTerm> terms, + List args, + ) { + if (cursor == null || cursor.values.length != terms.length) return ''; + final clauses = []; + final first = terms.first; + final coarseOperator = first.descending ? '<=' : '>='; + args.add(cursor.values.first); + for (var i = 0; i < terms.length; i++) { + final comparisons = []; + for (var j = 0; j < i; j++) { + comparisons.add('${terms[j].column} = ?'); + args.add(cursor.values[j]); + } + comparisons.add( + '${terms[i].column} ${terms[i].descending ? '<' : '>'} ?', + ); + args.add(cursor.values[i]); + clauses.add('(${comparisons.join(' AND ')})'); + } + return '(${first.column} $coarseOperator ?) AND (${clauses.join(' OR ')})'; + } + + QueueLibraryDbCursor? _queueCursorFromRow( + Map? row, + List<_QueueOrderTerm> terms, + ) { + if (row == null) return null; + final values = []; + for (final term in terms) { + final value = row[term.column]; + if (value is! Object) return null; + values.add(value); + } + return QueueLibraryDbCursor(List.unmodifiable(values)); + } + Map _queueTrackRowToJson(Map row) { final source = row['queue_source'] as String? ?? ''; if (source == 'local') { diff --git a/lib/services/platform_bridge.dart b/lib/services/platform_bridge.dart index 38c7450b..60e09d14 100644 --- a/lib/services/platform_bridge.dart +++ b/lib/services/platform_bridge.dart @@ -20,6 +20,38 @@ bool isForegroundServiceStartNotAllowed(Object error) { } Object? _decodeJsonInBackground(String json) => jsonDecode(json); +String _encodeJsonInBackground(Object? value) => jsonEncode(value); + +class LibraryScanNDJSONFile { + final File file; + final int expectedCount; + + const LibraryScanNDJSONFile({ + required this.file, + required this.expectedCount, + }); + + Stream> rows() async* { + await for (final line + in file + .openRead() + .transform(utf8.decoder) + .transform(const LineSplitter())) { + if (line.trim().isEmpty) continue; + final decoded = jsonDecode(line); + if (decoded is! Map) { + throw const FormatException('Library scan NDJSON row is not an object'); + } + yield Map.from(decoded); + } + } + + Future delete() async { + try { + if (await file.exists()) await file.delete(); + } catch (_) {} + } +} class ExtensionSessionGrantEvent { final String extensionId; @@ -40,6 +72,13 @@ class IosPickedDirectory { const IosPickedDirectory({required this.path, required this.bookmark}); } +class IosSecurityScopedAccess { + final String path; + final String token; + + const IosSecurityScopedAccess({required this.path, required this.token}); +} + class InstallationState { final bool markerExisted; final bool markerCreated; @@ -113,6 +152,7 @@ class PlatformBridge { static const _urlHandleCacheTtl = Duration(minutes: 5); static const _customSearchCacheTtl = Duration(minutes: 2); static const _bridgeCacheMaxEntries = 256; + static const _lookupCachePersistDebounce = Duration(milliseconds: 500); static const _metadataPersistentCacheKey = 'bridge_metadata_lookup_cache_v1'; static const _downloadProgressEvents = EventChannel( 'com.zarz.spotiflac/download_progress_stream', @@ -132,6 +172,9 @@ class PlatformBridge { _homeFeedInFlight = {}; static Future? _persistentLookupCacheLoadFuture; static int _lookupCacheGeneration = 0; + static Timer? _lookupCachePersistTimer; + static Future? _lookupCachePersistInFlight; + static bool _lookupCachePersistDirty = false; static int _extensionRequestSequence = 0; static final StreamController _extensionSessionGrantEvents = @@ -259,8 +302,10 @@ class PlatformBridge { value: _copyStringMap(value), expiresAt: DateTime.now().add(ttl), ); - unawaited( - _persistLookupCache(cache, persistentCacheKey, _lookupCacheGeneration), + _scheduleLookupCachePersist( + cache, + persistentCacheKey, + _lookupCacheGeneration, ); } @@ -383,14 +428,69 @@ class PlatformBridge { 'value': entry.value.value, }, }; + final encoded = data.length >= 32 + ? await compute(_encodeJsonInBackground, data) + : jsonEncode(data); + if (generation != _lookupCacheGeneration) return; final prefs = await SharedPreferences.getInstance(); if (generation != _lookupCacheGeneration) return; - await prefs.setString(prefsKey, jsonEncode(data)); + await prefs.setString(prefsKey, encoded); } catch (e) { _log.w('Failed to persist bridge lookup cache: $e'); } } + static void _scheduleLookupCachePersist( + Map cache, + String prefsKey, + int generation, + ) { + _lookupCachePersistDirty = true; + if (_lookupCachePersistInFlight != null) return; + _lookupCachePersistTimer?.cancel(); + _lookupCachePersistTimer = Timer(_lookupCachePersistDebounce, () { + _lookupCachePersistTimer = null; + unawaited(_flushLookupCachePersist(cache, prefsKey, generation)); + }); + } + + static Future _flushLookupCachePersist( + Map cache, + String prefsKey, + int generation, + ) { + final active = _lookupCachePersistInFlight; + if (active != null) { + _lookupCachePersistDirty = true; + return active; + } + + late final Future flush; + flush = + () async { + do { + _lookupCachePersistDirty = false; + if (generation != _lookupCacheGeneration) return; + await _persistLookupCache(cache, prefsKey, generation); + } while (_lookupCachePersistDirty && + generation == _lookupCacheGeneration); + }().whenComplete(() { + if (identical(_lookupCachePersistInFlight, flush)) { + _lookupCachePersistInFlight = null; + } + }); + _lookupCachePersistInFlight = flush; + return flush; + } + + static Future _cancelLookupCachePersistence() async { + _lookupCachePersistTimer?.cancel(); + _lookupCachePersistTimer = null; + _lookupCachePersistDirty = false; + final active = _lookupCachePersistInFlight; + if (active != null) await active; + } + static Future _clearPersistentLookupCaches() async { try { final prefs = await SharedPreferences.getInstance(); @@ -404,6 +504,7 @@ class PlatformBridge { static Future _clearLookupCaches() async { _lookupCacheGeneration++; + await _cancelLookupCachePersistence(); _persistentLookupCacheLoadFuture = null; _metadataCache.clear(); _urlHandleCache.clear(); @@ -1839,6 +1940,17 @@ class PlatformBridge { return _decodeMapListResultAsync(result, 'scanLibraryFolder'); } + static Future scanLibraryFolderToNDJSONFile( + String folderPath, { + bool Function()? isCancelled, + }) { + return _scanToNDJSONFile( + method: 'scanLibraryFolderToNDJSONFile', + arguments: {'folder_path': folderPath}, + isCancelled: isCancelled, + ); + } + static Future> scanLibraryFolderIncremental( String folderPath, Map existingFiles, @@ -1878,6 +1990,67 @@ class PlatformBridge { return _decodeMapListResultAsync(result, 'scanSafTree'); } + static Future scanSafTreeToNDJSONFile( + String treeUri, { + bool Function()? isCancelled, + }) { + return _scanToNDJSONFile( + method: 'scanSafTreeToNDJSONFile', + arguments: {'tree_uri': treeUri}, + isCancelled: isCancelled, + ); + } + + static Future _scanToNDJSONFile({ + required String method, + required Map arguments, + bool Function()? isCancelled, + }) async { + final tempDir = await getTemporaryDirectory(); + final output = File( + '${tempDir.path}${Platform.pathSeparator}' + 'library_scan_${DateTime.now().microsecondsSinceEpoch}.ndjson', + ); + try { + if (isCancelled?.call() == true) { + throw StateError('Library scan cancelled before native scan'); + } + final result = await _channel.invokeMethod(method, { + ...arguments, + 'output_path': output.path, + }); + if (result is! Map) { + throw FormatException('$method returned ${result.runtimeType}'); + } + if (result['cancelled'] == true) { + throw FormatException('$method returned a cancelled partial scan'); + } + final pathValue = result['path']; + final countValue = result['count']; + if (pathValue is! String || pathValue.trim().isEmpty) { + throw FormatException('$method returned an invalid output path'); + } + if (countValue is! num || + !countValue.isFinite || + countValue < 0 || + countValue != countValue.toInt()) { + throw FormatException('$method returned an invalid row count'); + } + final path = pathValue; + final count = countValue.toInt(); + final file = File(path); + if (!await file.exists()) { + throw FormatException('$method did not create its output file'); + } + return LibraryScanNDJSONFile(file: file, expectedCount: count); + } catch (_) { + try { + if (await output.exists()) await output.delete(); + } catch (_) {} + rethrow; + } + } + static Future> scanSafTreeIncremental( String treeUri, Map existingFiles, @@ -2103,24 +2276,43 @@ class PlatformBridge { } /// Resolve a base64-encoded iOS security-scoped bookmark and start accessing - /// the resource. Returns the resolved filesystem path. - /// The resource stays accessed until [stopAccessingIosBookmark] is called. - static Future startAccessingIosBookmark(String bookmark) async { + /// the resource. The returned lease must be passed to + /// [stopAccessingIosBookmark] by the operation that acquired it. + static Future startAccessingIosBookmark( + String bookmark, + ) async { try { final result = await _channel.invokeMethod('startAccessingIosBookmark', { 'bookmark': bookmark, }); - return result as String?; + if (result is! Map) { + throw FormatException( + 'startAccessingIosBookmark returned ${result.runtimeType}', + ); + } + final path = result['path']; + final token = result['token']; + if (path is! String || + path.trim().isEmpty || + token is! String || + token.trim().isEmpty) { + throw const FormatException('Invalid iOS bookmark access lease'); + } + return IosSecurityScopedAccess(path: path, token: token); } catch (e) { _log.w('Failed to start accessing iOS bookmark: $e'); return null; } } - /// Stop accessing the currently active iOS security-scoped resource. - static Future stopAccessingIosBookmark() async { + /// Releases exactly the security-scoped lease acquired by the caller. + static Future stopAccessingIosBookmark( + IosSecurityScopedAccess access, + ) async { try { - await _channel.invokeMethod('stopAccessingIosBookmark'); + await _channel.invokeMethod('stopAccessingIosBookmark', { + 'token': access.token, + }); } catch (e) { _log.w('Failed to stop accessing iOS bookmark: $e'); } diff --git a/lib/widgets/cross_extension_share_sheet.dart b/lib/widgets/cross_extension_share_sheet.dart index e2c8465a..c4139f7c 100644 --- a/lib/widgets/cross_extension_share_sheet.dart +++ b/lib/widgets/cross_extension_share_sheet.dart @@ -158,7 +158,7 @@ class _CrossExtensionShareTile extends StatelessWidget { borderRadius: BorderRadius.circular(12), ), clipBehavior: Clip.antiAlias, - child: _buildIcon(colorScheme), + child: _buildIcon(context, colorScheme), ), title: Text( result.displayName, @@ -205,7 +205,7 @@ class _CrossExtensionShareTile extends StatelessWidget { return Opacity(opacity: 0.5, child: tile); } - Widget _buildIcon(ColorScheme colorScheme) { + Widget _buildIcon(BuildContext context, ColorScheme colorScheme) { final fallbackIcon = Icon( Icons.extension_rounded, color: colorScheme.onSurfaceVariant, @@ -219,6 +219,9 @@ class _CrossExtensionShareTile extends StatelessWidget { width: 44, height: 44, fit: BoxFit.cover, + cacheWidth: (44 * MediaQuery.devicePixelRatioOf(context)).round(), + cacheHeight: (44 * MediaQuery.devicePixelRatioOf(context)).round(), + filterQuality: FilterQuality.low, errorBuilder: (_, _, _) => fallbackIcon, ); } diff --git a/lib/widgets/extension_row.dart b/lib/widgets/extension_row.dart index 7895439c..4c2a4166 100644 --- a/lib/widgets/extension_row.dart +++ b/lib/widgets/extension_row.dart @@ -49,6 +49,9 @@ class ExtensionAvatar extends StatelessWidget { width: size, height: size, fit: BoxFit.cover, + cacheWidth: (size * MediaQuery.devicePixelRatioOf(context)).round(), + cacheHeight: (size * MediaQuery.devicePixelRatioOf(context)).round(), + filterQuality: FilterQuality.low, errorBuilder: (_, _, _) => fallback, ); } else if (imageUrl != null && imageUrl!.isNotEmpty) { @@ -57,6 +60,9 @@ class ExtensionAvatar extends StatelessWidget { width: size, height: size, fit: BoxFit.cover, + cacheWidth: (size * MediaQuery.devicePixelRatioOf(context)).round(), + cacheHeight: (size * MediaQuery.devicePixelRatioOf(context)).round(), + filterQuality: FilterQuality.low, errorBuilder: (_, _, _) => fallback, loadingBuilder: (context, child, progress) { if (progress == null) return child; diff --git a/lib/widgets/playlist_picker_sheet.dart b/lib/widgets/playlist_picker_sheet.dart index 1e1cf82b..ef7a44a0 100644 --- a/lib/widgets/playlist_picker_sheet.dart +++ b/lib/widgets/playlist_picker_sheet.dart @@ -335,7 +335,7 @@ class _PlaylistPickerThumbnail extends StatelessWidget { children: [ ClipRRect( borderRadius: borderRadius, - child: _buildCoverImage(colorScheme, size), + child: _buildCoverImage(context, colorScheme, size), ), if (isSelected) ...[ Positioned.fill( @@ -368,7 +368,11 @@ class _PlaylistPickerThumbnail extends StatelessWidget { ); } - Widget _buildCoverImage(ColorScheme colorScheme, double size) { + Widget _buildCoverImage( + BuildContext context, + ColorScheme colorScheme, + double size, + ) { final customCoverPath = playlist.coverImagePath; if (customCoverPath != null && customCoverPath.isNotEmpty) { return Image.file( @@ -376,6 +380,9 @@ class _PlaylistPickerThumbnail extends StatelessWidget { width: size, height: size, fit: BoxFit.cover, + cacheWidth: (size * MediaQuery.devicePixelRatioOf(context)).round(), + cacheHeight: (size * MediaQuery.devicePixelRatioOf(context)).round(), + filterQuality: FilterQuality.low, errorBuilder: (_, _, _) => _iconFallback(colorScheme, size), ); } diff --git a/test/download_history_logic_test.dart b/test/download_history_logic_test.dart index 1100f6ec..b93cf9ca 100644 --- a/test/download_history_logic_test.dart +++ b/test/download_history_logic_test.dart @@ -22,6 +22,16 @@ DownloadHistoryItem _historyItem({ void main() { group('download history identity', () { + test('serializes download timestamps with an explicit UTC offset', () { + final item = _historyItem( + id: 'utc', + filePath: '/music/Album/Same Song.flac', + downloadedAt: DateTime(2026, 7, 22, 12, 30), + ); + + expect(item.toJson()['downloadedAt'], endsWith('Z')); + }); + test('same track metadata does not merge files from different albums', () { final first = _historyItem( id: 'first', diff --git a/test/platform_bridge_metadata_test.dart b/test/platform_bridge_metadata_test.dart index 79dc8efb..8529d91e 100644 --- a/test/platform_bridge_metadata_test.dart +++ b/test/platform_bridge_metadata_test.dart @@ -167,4 +167,25 @@ void main() { expect(results.single['status'], 'found'); expect(results.single['relative_dir'], 'Artist/Album'); }); + + test('iOS bookmark access releases the matching native lease', () async { + final calls = []; + TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .setMockMethodCallHandler(backendChannel, (call) async { + calls.add(call); + if (call.method == 'startAccessingIosBookmark') { + return {'path': '/music', 'token': 'lease-1'}; + } + if (call.method == 'stopAccessingIosBookmark') return null; + fail('Unexpected method: ${call.method}'); + }); + + final access = await PlatformBridge.startAccessingIosBookmark('bookmark'); + expect(access?.path, '/music'); + expect(access?.token, 'lease-1'); + await PlatformBridge.stopAccessingIosBookmark(access!); + + expect(calls[1].method, 'stopAccessingIosBookmark'); + expect(calls[1].arguments, {'token': 'lease-1'}); + }); }