CalvinBackup/SpotiFLAC-Mobile
1
0
Fork 0
mirror of https://github.com/zarzet/SpotiFLAC-Mobile.git synced 2026-08-27 05:12:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

perf(library): stream scans and optimize queue queries

Browse Source
This commit is contained in:
zarzet
2026-08-18 11:03:05 +07:00
parent 29609b6084
commit 7fc27e8314
28 changed files with 2145 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android/app/src/main/kotlin/com/zarz/spotiflac/MainActivity.kt
+25
View File
@@ -2165,6 +2165,23 @@ class MainActivity: FlutterFragmentActivity() {
}
result.success(response)
}
"scanLibraryFolderToNDJSONFile" -> {
val folderPath = call.argument<String>("folder_path") ?: ""
val outputPath = call.argument<String>("output_path") ?: ""
val count = withContext(Dispatchers.IO) {
safScanActive = false
Gobackend.scanLibraryFolderToNDJSONFileJSON(
folderPath,
outputPath,
)
}
result.success(
mapOf(
"path" to outputPath,
"count" to count,
)
)
}
"scanLibraryFolderIncremental" -> {
val folderPath = call.argument<String>("folder_path") ?: ""
val existingFiles = call.argument<String>("existing_files") ?: "{}"
@@ -2197,6 +2214,14 @@ class MainActivity: FlutterFragmentActivity() {
}
result.success(response)
}
"scanSafTreeToNDJSONFile" -> {
val treeUri = call.argument<String>("tree_uri") ?: ""
val outputPath = call.argument<String>("output_path") ?: ""
val response = withContext(Dispatchers.IO) {
scanSafTree(treeUri, outputPath)
}
result.success(response)
}
"scanSafTreeIncremental" -> {
val treeUri = call.argument<String>("tree_uri") ?: ""
val existingFiles = call.argument<String>("existing_files") ?: "{}"
android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt
+126 -52
View File
@@ -3,9 +3,11 @@ package com.zarz.spotiflac
import android.app.Activity
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
import android.os.Build
import android.os.Bundle
import android.provider.DocumentsContract
import androidx.activity.OnBackPressedCallback
import androidx.activity.result.contract.ActivityResultContracts
import androidx.documentfile.provider.DocumentFile
@@ -34,6 +36,7 @@ import org.json.JSONTokener
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.IOException
import java.security.MessageDigest
import java.util.Locale
@@ -460,21 +463,13 @@ internal fun MainActivity.getSafChildFileLookup(
): Map<String, DocumentFile> {
val dirKey = dir.uri.toString()
return cache.getOrPut(dirKey) {
try {
buildMap {
for (child in dir.listFiles()) {
if (!child.isFile) continue
val childName = child.name?.trim().orEmpty()
if (childName.isBlank()) continue
put(childName.lowercase(Locale.ROOT), child)
}
buildMap {
for (child in listSafChildrenOrThrow(dir)) {
if (!child.isFile) continue
val childName = child.name?.trim().orEmpty()
if (childName.isBlank()) continue
put(childName.lowercase(Locale.ROOT), child)
}
} catch (e: Exception) {
android.util.Log.w(
"SpotiFLAC",
"Failed to build SAF child lookup for $dirKey: ${e.message}",
)
emptyMap()
}
}
}
@@ -510,11 +505,80 @@ internal fun MainActivity.resolveCueAudioSibling(
return null
}
internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
if (treeUriStr.isBlank()) return "[]"
internal fun MainActivity.listSafChildrenOrThrow(dir: DocumentFile): List<DocumentFile> {
val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree(
dir.uri,
DocumentsContract.getDocumentId(dir.uri),
)
val cursor = contentResolver.query(
childrenUri,
arrayOf(DocumentsContract.Document.COLUMN_DOCUMENT_ID),
null,
null,
null,
) ?: throw IOException("SAF provider returned no cursor for ${dir.uri}")
return cursor.use {
val documentIdIndex = it.getColumnIndexOrThrow(
DocumentsContract.Document.COLUMN_DOCUMENT_ID,
)
buildList {
while (it.moveToNext()) {
val childUri = DocumentsContract.buildDocumentUriUsingTree(
dir.uri,
it.getString(documentIdIndex),
)
val child = DocumentFile.fromSingleUri(this@listSafChildrenOrThrow, childUri)
?: throw IOException("Invalid SAF child URI: $childUri")
add(child)
}
}
}
}
internal fun MainActivity.resolveReadableSafTreeOrThrow(
treeUriStr: String,
): Pair<Uri, DocumentFile> {
if (treeUriStr.isBlank()) {
throw IllegalArgumentException("SAF tree URI is empty")
}
val treeUri = Uri.parse(treeUriStr)
val root = DocumentFile.fromTreeUri(this, treeUri) ?: return "[]"
val hasReadPermission = contentResolver.persistedUriPermissions.any {
it.uri == treeUri && it.isReadPermission
} || checkUriPermission(
treeUri,
android.os.Process.myPid(),
android.os.Process.myUid(),
Intent.FLAG_GRANT_READ_URI_PERMISSION,
) == PackageManager.PERMISSION_GRANTED
if (!hasReadPermission) {
throw SecurityException("Read access to the SAF tree has been revoked")
}
val root = DocumentFile.fromTreeUri(this, treeUri)
?: throw IOException("Unable to resolve SAF tree")
if (!root.exists() || !root.canRead()) {
throw IOException("SAF tree is unavailable or unreadable")
}
return treeUri to root
}
internal fun MainActivity.scanSafTree(
treeUriStr: String,
ndjsonOutputPath: String? = null,
): Any {
fun emptyResult(): Any {
if (ndjsonOutputPath == null) return "[]"
File(ndjsonOutputPath).writeText("", Charsets.UTF_8)
return mapOf("path" to ndjsonOutputPath, "count" to 0)
}
fun cancelledResult(): Any {
updateSafScanProgress { it.isComplete = true }
if (ndjsonOutputPath == null) return "[]"
try { File(ndjsonOutputPath).delete() } catch (_: Exception) {}
throw java.util.concurrent.CancellationException("SAF library scan cancelled")
}
val (_, root) = resolveReadableSafTreeOrThrow(treeUriStr)
resetSafScanProgress()
safScanCancel = false
@@ -535,8 +599,7 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
while (queue.isNotEmpty()) {
if (safScanCancel) {
updateSafScanProgress { it.isComplete = true }
return "[]"
return cancelledResult()
}
val (dir, path) = queue.removeFirst()
@@ -546,7 +609,7 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
}
val children = try {
dir.listFiles()
listSafChildrenOrThrow(dir)
} catch (e: Exception) {
traversalErrors++
updateSafScanProgress { it.errorCount = traversalErrors }
@@ -559,8 +622,7 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
for (child in children) {
if (safScanCancel) {
updateSafScanProgress { it.isComplete = true }
return "[]"
return cancelledResult()
}
try {
@@ -592,6 +654,10 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
}
}
if (traversalErrors > 0) {
throw IOException("SAF traversal failed for $traversalErrors entries")
}
val totalItems = audioFiles.size + cueFiles.size
updateSafScanProgress {
it.totalFiles = totalItems
@@ -602,19 +668,28 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
it.isComplete = true
it.progressPct = 100.0
}
return "[]"
return emptyResult()
}
// Stream results to a spill file: a full-library scan's JSONArray plus
// its serialized string would otherwise hold the whole payload on the
// Java heap several times over.
val spill = this.SpillJsonWriter()
val spill = if (ndjsonOutputPath == null) this.SpillJsonWriter() else null
val ndjsonWriter = ndjsonOutputPath?.let {
File(it).bufferedWriter(Charsets.UTF_8, 64 * 1024)
}
var resultCount = 0
fun putResult(obj: JSONObject) {
spill.raw(if (resultCount == 0) "[" else ",")
spill.raw(obj.toString())
if (ndjsonWriter != null) {
ndjsonWriter.write(obj.toString())
ndjsonWriter.newLine()
} else {
spill!!.raw(if (resultCount == 0) "[" else ",")
spill.raw(obj.toString())
}
resultCount++
}
try {
var scanned = 0
var errors = traversalErrors
@@ -622,9 +697,9 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
for ((cueDoc, parentDir) in cueFiles) {
if (safScanCancel) {
updateSafScanProgress { it.isComplete = true }
spill.abandon()
return "[]"
ndjsonWriter?.close()
spill?.abandon()
return cancelledResult()
}
val cueName = try { cueDoc.name ?: "" } catch (_: Exception) { "" }
@@ -722,9 +797,9 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
for ((doc, _) in audioFiles) {
if (safScanCancel) {
updateSafScanProgress { it.isComplete = true }
spill.abandon()
return "[]"
ndjsonWriter?.close()
spill?.abandon()
return cancelledResult()
}
if (cueReferencedAudioUris.contains(doc.uri.toString())) {
@@ -780,8 +855,20 @@ internal fun MainActivity.scanSafTree(treeUriStr: String): Any {
it.progressPct = 100.0
}
spill.raw(if (resultCount == 0) "[]" else "]")
if (ndjsonWriter != null) {
ndjsonWriter.close()
return mapOf("path" to ndjsonOutputPath, "count" to resultCount)
}
spill!!.raw(if (resultCount == 0) "[]" else "]")
return spill.result()
} catch (e: Exception) {
try { ndjsonWriter?.close() } catch (_: Exception) {}
spill?.abandon()
if (ndjsonOutputPath != null) {
try { File(ndjsonOutputPath).delete() } catch (_: Exception) {}
}
throw e
}
}
/**
@@ -809,24 +896,7 @@ internal fun MainActivity.scanSafTreeIncremental(
treeUriStr: String,
existingFiles: Map<String, Long>,
): Any {
if (treeUriStr.isBlank()) {
val result = JSONObject()
result.put("files", JSONArray())
result.put("removedUris", JSONArray())
result.put("skippedCount", 0)
result.put("totalFiles", 0)
return result.toString()
}
val treeUri = Uri.parse(treeUriStr)
val root = DocumentFile.fromTreeUri(this, treeUri) ?: run {
val result = JSONObject()
result.put("files", JSONArray())
result.put("removedUris", JSONArray())
result.put("skippedCount", 0)
result.put("totalFiles", 0)
return result.toString()
}
val (_, root) = resolveReadableSafTreeOrThrow(treeUriStr)
resetSafScanProgress()
safScanCancel = false
@@ -875,7 +945,7 @@ internal fun MainActivity.scanSafTreeIncremental(
}
val children = try {
dir.listFiles()
listSafChildrenOrThrow(dir)
} catch (e: Exception) {
traversalErrors++
updateSafScanProgress { it.errorCount = traversalErrors }
@@ -954,6 +1024,10 @@ internal fun MainActivity.scanSafTreeIncremental(
}
}
if (traversalErrors > 0) {
throw IOException("SAF traversal failed for $traversalErrors entries")
}
val removedUris = existingFiles.keys.filter { !currentUris.contains(it) }
val totalFiles = currentUris.size
val filesToProcess = audioFiles.size + cueFilesToScan.size
android/app/src/main/kotlin/com/zarz/spotiflac/NativeDownloadFinalizer.kt
+98 -4
View File
@@ -39,7 +39,7 @@ object NativeDownloadFinalizer {
const val NATIVE_WORKER_CONTRACT_VERSION = 1
// Native finalizer owns background-safe history writes while Flutter may be suspended.
// Keep this schema contract in sync with Dart HistoryDatabase before bumping either side.
const val HISTORY_SCHEMA_VERSION = 10
const val HISTORY_SCHEMA_VERSION = 11
internal val activeFFmpegSessionIds = mutableSetOf<Long>()
internal val nativeFFmpegSessionIds = mutableSetOf<Long>()
internal val activeFFmpegSessionLock = Any()
@@ -94,6 +94,13 @@ object NativeDownloadFinalizer {
"match_key",
"album_key",
"search_text",
"sort_track",
"sort_artist",
"sort_album",
"sort_album_artist",
"sort_genre",
"sort_release",
"sort_added",
)
private val androidStoragePathAliases = listOf(
"/storage/emulated/0",
@@ -1363,7 +1370,14 @@ object NativeDownloadFinalizer {
isrc_norm TEXT,
match_key TEXT,
album_key TEXT,
search_text TEXT
search_text TEXT,
sort_track TEXT,
sort_artist TEXT,
sort_album TEXT,
sort_album_artist TEXT,
sort_genre TEXT,
sort_release TEXT,
sort_added INTEGER
)
""".trimIndent()
)
@@ -1382,6 +1396,13 @@ object NativeDownloadFinalizer {
ensureHistoryColumn(db, "match_key", "ALTER TABLE history ADD COLUMN match_key TEXT")
ensureHistoryColumn(db, "album_key", "ALTER TABLE history ADD COLUMN album_key TEXT")
ensureHistoryColumn(db, "search_text", "ALTER TABLE history ADD COLUMN search_text TEXT")
ensureHistoryColumn(db, "sort_track", "ALTER TABLE history ADD COLUMN sort_track TEXT")
ensureHistoryColumn(db, "sort_artist", "ALTER TABLE history ADD COLUMN sort_artist TEXT")
ensureHistoryColumn(db, "sort_album", "ALTER TABLE history ADD COLUMN sort_album TEXT")
ensureHistoryColumn(db, "sort_album_artist", "ALTER TABLE history ADD COLUMN sort_album_artist TEXT")
ensureHistoryColumn(db, "sort_genre", "ALTER TABLE history ADD COLUMN sort_genre TEXT")
ensureHistoryColumn(db, "sort_release", "ALTER TABLE history ADD COLUMN sort_release TEXT")
ensureHistoryColumn(db, "sort_added", "ALTER TABLE history ADD COLUMN sort_added INTEGER")
ensureHistoryPathKeyTable(db)
if (needsBackfill) {
backfillNormalizedHistoryColumns(db)
@@ -1397,6 +1418,12 @@ object NativeDownloadFinalizer {
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_isrc_norm ON history(isrc_norm)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_match_key ON history(match_key)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_album_key ON history(album_key)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_added ON history(sort_added DESC, sort_track, id)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_track ON history(sort_track, sort_artist, id)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_artist ON history(sort_artist, sort_track, id)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_album ON history(sort_album, sort_track, id)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_genre ON history(sort_genre, sort_track, id)")
db.execSQL("CREATE INDEX IF NOT EXISTS idx_history_queue_release ON history(sort_release, sort_track, id)")
if (db.version < HISTORY_SCHEMA_VERSION) db.version = HISTORY_SCHEMA_VERSION
if (deduplicateTrack) deleteDuplicateHistoryRows(db, values)
db.insertWithOnConflict("history", null, values, SQLiteDatabase.CONFLICT_REPLACE)
@@ -1523,8 +1550,8 @@ object NativeDownloadFinalizer {
private fun backfillNormalizedHistoryColumns(db: SQLiteDatabase) {
db.query(
"history",
arrayOf("id", "spotify_id", "isrc", "track_name", "artist_name", "album_name", "album_artist"),
"spotify_id_norm IS NULL OR isrc_norm IS NULL OR match_key IS NULL OR album_key IS NULL OR search_text IS NULL",
arrayOf("id", "spotify_id", "isrc", "track_name", "artist_name", "album_name", "album_artist", "genre", "release_date", "downloaded_at"),
"spotify_id_norm IS NULL OR isrc_norm IS NULL OR match_key IS NULL OR album_key IS NULL OR search_text IS NULL OR sort_track IS NULL OR sort_release IS NULL OR sort_added IS NULL",
null,
null,
null,
@@ -1537,6 +1564,9 @@ object NativeDownloadFinalizer {
val artistIndex = cursor.getColumnIndex("artist_name")
val albumIndex = cursor.getColumnIndex("album_name")
val albumArtistIndex = cursor.getColumnIndex("album_artist")
val genreIndex = cursor.getColumnIndex("genre")
val releaseDateIndex = cursor.getColumnIndex("release_date")
val downloadedAtIndex = cursor.getColumnIndex("downloaded_at")
while (cursor.moveToNext()) {
if (idIndex < 0) continue
val values = ContentValues()
@@ -1556,6 +1586,16 @@ object NativeDownloadFinalizer {
albumName = albumName,
albumArtist = albumArtist,
)
putQueueSortHistoryColumns(
values,
trackName = trackName,
artistName = artistName,
albumName = albumName,
albumArtist = albumArtist,
genre = cursor.getNullableString(genreIndex),
releaseDate = cursor.getNullableString(releaseDateIndex),
downloadedAt = cursor.getNullableString(downloadedAtIndex),
)
db.update("history", values, "id = ?", arrayOf(cursor.getString(idIndex)))
}
}
@@ -1599,6 +1639,60 @@ object NativeDownloadFinalizer {
albumName = values.getAsString("album_name"),
albumArtist = values.getAsString("album_artist"),
)
putQueueSortHistoryColumns(
values,
trackName = values.getAsString("track_name"),
artistName = values.getAsString("artist_name"),
albumName = values.getAsString("album_name"),
albumArtist = values.getAsString("album_artist"),
genre = values.getAsString("genre"),
releaseDate = values.getAsString("release_date"),
downloadedAt = values.getAsString("downloaded_at"),
)
}
private fun putQueueSortHistoryColumns(
values: ContentValues,
trackName: String?,
artistName: String?,
albumName: String?,
albumArtist: String?,
genre: String?,
releaseDate: String?,
downloadedAt: String?,
) {
values.put("sort_track", normalizeLookupText(trackName))
values.put("sort_artist", normalizeLookupText(artistName))
values.put("sort_album", normalizeLookupText(albumName))
values.put(
"sort_album_artist",
normalizeLookupText(albumArtist?.takeIf { it.trim().isNotEmpty() } ?: artistName),
)
values.put("sort_genre", normalizeLookupText(genre))
values.put("sort_release", releaseDate?.trim().orEmpty())
val sortAdded = parseHistoryTimestampMillis(downloadedAt)
values.put("sort_added", sortAdded)
}
private fun parseHistoryTimestampMillis(value: String?): Long {
val timestamp = value?.trim().orEmpty()
if (timestamp.isEmpty()) return 0L
return try {
java.time.Instant.parse(timestamp).toEpochMilli()
} catch (_: Exception) {
try {
java.time.OffsetDateTime.parse(timestamp).toInstant().toEpochMilli()
} catch (_: Exception) {
try {
java.time.LocalDateTime.parse(timestamp)
.atZone(java.time.ZoneId.systemDefault())
.toInstant()
.toEpochMilli()
} catch (_: Exception) {
0L
}
}
}
}
private fun putAlbumSearchHistoryColumns(