diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt index 41009edb..ba16342c 100644 --- a/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/MainActivitySafScan.kt @@ -1,11 +1,7 @@ package com.zarz.spotiflac -import android.content.Context -import android.content.Intent -import android.content.pm.PackageManager import android.net.Uri import android.provider.DocumentsContract -import android.provider.DocumentsContract.Document.MIME_TYPE_DIR import androidx.documentfile.provider.DocumentFile import org.json.JSONArray import org.json.JSONObject @@ -14,12 +10,6 @@ import java.io.FileOutputStream import java.io.IOException import java.security.MessageDigest import java.util.Locale -import java.util.concurrent.ExecutionException -import java.util.concurrent.Executors -import java.util.concurrent.Future -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.ThreadPoolExecutor -import java.util.concurrent.TimeUnit // SAF library-scan subsystem: tree walking, incremental diff, CUE resolution, // and scan-progress state shared with the progress stream in MainActivity. @@ -97,9 +87,6 @@ internal fun MainActivity.loadExistingFilesFromSnapshot(snapshotPath: String): M ".ape", ".wv", ".mpc", ".wav", ".aiff", ".aif", ".dsf", ".dff" ) -// Provider round trips dominate on SD cards, USB drives and network shares. -private const val SAF_LIST_CONCURRENCY = 4 -private const val SAF_READ_WORKERS = 6 private const val SAF_SCAN_PAUSE_POLL_MS = 100L /** @@ -113,244 +100,6 @@ internal fun MainActivity.safScanStopRequested(cancelled: () -> Boolean = { fals return safScanCancel || cancelled() } -/** - * Prefetches the listings of directories already waiting in a breadth-first - * queue. The caller still dequeues, deduplicates and records errors serially, - * so traversal order and results match a one-directory-at-a-time walk. - * Idle daemon threads exit on their own, so abandoned walks leak nothing. - */ -private class SafTreeLister(private val context: Context) { - private val executor = ThreadPoolExecutor( - SAF_LIST_CONCURRENCY, - SAF_LIST_CONCURRENCY, - 1, - TimeUnit.SECONDS, - LinkedBlockingQueue(), - ) { runnable -> Thread(runnable, "saf-scan-list").apply { isDaemon = true } } - .apply { allowCoreThreadTimeOut(true) } - private val prefetched = mutableMapOf>>>() - - private fun submit(dir: DocumentFile) = executor.submit>> { - try { - Result.success(context.listSafChildrenOrThrow(dir)) - } catch (e: Exception) { - Result.failure(e) - } - } - - fun list( - dir: DocumentFile, - queue: Collection>, - visited: Set, - ): Result> { - val uri = dir.uri.toString() - val current = prefetched.remove(uri) ?: submit(dir) - for ((next, _) in queue) { - if (prefetched.size >= SAF_LIST_CONCURRENCY * 2) break - val nextUri = next.uri.toString() - if (nextUri == uri || nextUri in visited || nextUri in prefetched) continue - prefetched[nextUri] = submit(next) - } - return try { - current.get() - } catch (e: ExecutionException) { - throw e.cause ?: e - } - } -} - -/** - * Runs [task] on a bounded pool and passes each result to [consume] in input - * order. A sliding window keeps workers busy past one slow file. Returns - * false when cancelled. - */ -private fun runSafReadsInOrder( - items: List, - cancelled: () -> Boolean, - task: (T) -> R, - consume: (T, Result) -> Unit, -): Boolean { - if (items.isEmpty()) return true - val workers = minOf(SAF_READ_WORKERS, items.size) - val executor = Executors.newFixedThreadPool(workers) - try { - val window = workers * 4 - val inFlight = ArrayDeque>>(window) - var next = 0 - while (next < items.size || inFlight.isNotEmpty()) { - if (cancelled()) { - inFlight.forEach { it.second.cancel(true) } - return false - } - while (inFlight.size < window && next < items.size) { - val item = items[next++] - inFlight.addLast(item to executor.submit { task(item) }) - } - val (item, future) = inFlight.removeFirst() - val result = try { - Result.success(future.get()) - } catch (e: ExecutionException) { - Result.failure(e.cause ?: e) - } catch (e: Exception) { - Result.failure(e) - } - consume(item, result) - } - return true - } finally { - executor.shutdownNow() - } -} - -internal data class SafChildEntry( - val doc: DocumentFile, - val name: String?, - val isDirectory: Boolean, - val lastModified: Long, - val isFile: Boolean, -) - -internal fun Context.listSafChildrenOrThrow( - dir: DocumentFile, - includeLastModified: Boolean = true, -): List { - for (attempt in 0..2) { - try { - return querySafChildrenOrThrow(dir, includeLastModified) - } catch (error: SafProviderLoadingException) { - if (attempt == 2) throw error - Thread.sleep(300L * (attempt + 1)) - } - } - throw IOException("SAF directory listing did not complete") -} - -private class SafProviderLoadingException(uri: Uri) : - IOException("SAF provider is still loading $uri; retry the scan when it is ready") - -private fun Context.querySafChildrenOrThrow( - dir: DocumentFile, - includeLastModified: Boolean, -): List { - val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree( - dir.uri, - DocumentsContract.getDocumentId(dir.uri), - ) - val projection = mutableListOf( - DocumentsContract.Document.COLUMN_DOCUMENT_ID, - DocumentsContract.Document.COLUMN_DISPLAY_NAME, - DocumentsContract.Document.COLUMN_MIME_TYPE, - ) - if (includeLastModified) projection.add(DocumentsContract.Document.COLUMN_LAST_MODIFIED) - val cursor = try { - contentResolver.query(childrenUri, projection.toTypedArray(), null, null, null) - } catch (_: Exception) { - // A few older providers reject the richer projection; retry with IDs. - contentResolver.query( - childrenUri, - arrayOf(DocumentsContract.Document.COLUMN_DOCUMENT_ID), - null, - null, - null, - ) - } ?: throw IOException("SAF provider returned no cursor for ${dir.uri}") - return cursor.use { - // Network DocumentsProviders may return cached/partial rows while - // fetching a directory. Never commit that as an empty/full scan. - if (it.extras.getBoolean(DocumentsContract.EXTRA_LOADING, false)) { - throw SafProviderLoadingException(dir.uri) - } - val documentIdIndex = it.getColumnIndexOrThrow( - DocumentsContract.Document.COLUMN_DOCUMENT_ID, - ) - val displayNameIndex = it.getColumnIndex( - DocumentsContract.Document.COLUMN_DISPLAY_NAME, - ) - val mimeTypeIndex = it.getColumnIndex( - DocumentsContract.Document.COLUMN_MIME_TYPE, - ) - val lastModifiedIndex = it.getColumnIndex( - DocumentsContract.Document.COLUMN_LAST_MODIFIED, - ) - buildList { - while (it.moveToNext()) { - val childUri = DocumentsContract.buildDocumentUriUsingTree( - dir.uri, - it.getString(documentIdIndex), - ) - val child = DocumentFile.fromTreeUri(this@querySafChildrenOrThrow, childUri) - ?: throw IOException("Invalid SAF child URI: $childUri") - val name = if (displayNameIndex >= 0 && !it.isNull(displayNameIndex)) { - it.getString(displayNameIndex) - } else { - try { child.name } catch (_: Exception) { null } - } - val mimeType = if (mimeTypeIndex >= 0 && !it.isNull(mimeTypeIndex)) { - it.getString(mimeTypeIndex) - } else { - null - } - val isDirectory = if (mimeType != null) { - mimeType == MIME_TYPE_DIR - } else { - try { child.isDirectory } catch (_: Exception) { false } - } - val isFile = if (mimeType != null) { - mimeType.isNotEmpty() && !isDirectory - } else { - !isDirectory && try { child.isFile } catch (_: Exception) { false } - } - val lastModified = if (!includeLastModified) { - 0L - } else if ( - lastModifiedIndex >= 0 && !it.isNull(lastModifiedIndex) - ) { - it.getLong(lastModifiedIndex) - } else { - try { child.lastModified() } catch (_: Exception) { 0L } - } - add(SafChildEntry(child, name, isDirectory, lastModified, isFile)) - } - } - } - } - -internal fun Context.resolveReadableSafTreeOrThrow( - treeUriStr: String, - ): Triple> { - if (treeUriStr.isBlank()) { - throw IllegalArgumentException("SAF tree URI is empty") - } - val treeUri = Uri.parse(treeUriStr) - val root = DocumentFile.fromTreeUri(this, treeUri) - ?: throw IOException("Unable to resolve SAF tree") - val hasReadPermission = contentResolver.persistedUriPermissions.any { - it.uri == treeUri && it.isReadPermission - } || checkUriPermission( - root.uri, - android.os.Process.myPid(), - android.os.Process.myUid(), - Intent.FLAG_GRANT_READ_URI_PERMISSION, - ) == PackageManager.PERMISSION_GRANTED - if (!hasReadPermission) { - throw SecurityException("Read access to the SAF tree has been revoked") - } - // canRead() also requires a non-empty MIME type from queryDocument. - // Remote providers can omit that metadata after reconnecting even - // though queryChildDocuments can read the folder. The real listing is - // authoritative; reuse it in the scan instead of another round trip. - for (attempt in 0..2) { - val children = listSafChildrenOrThrow(root) - if (children.isNotEmpty() || root.exists()) { - return Triple(treeUri, root, children) - } - // An empty listing without a confirmed root is inconclusive, - // never evidence that previously indexed songs were deleted. - if (attempt < 2) Thread.sleep(300L * (attempt + 1)) - } - throw IOException("SAF provider could not confirm the library folder; reconnect it and retry") - } - internal fun MainActivity.scanSafTree( treeUriStr: String, ndjsonOutputPath: String? = null, diff --git a/android/app/src/main/kotlin/com/zarz/spotiflac/SafTreeListing.kt b/android/app/src/main/kotlin/com/zarz/spotiflac/SafTreeListing.kt new file mode 100644 index 00000000..7bac4bf7 --- /dev/null +++ b/android/app/src/main/kotlin/com/zarz/spotiflac/SafTreeListing.kt @@ -0,0 +1,262 @@ +package com.zarz.spotiflac + +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.net.Uri +import android.provider.DocumentsContract +import android.provider.DocumentsContract.Document.MIME_TYPE_DIR +import androidx.documentfile.provider.DocumentFile +import java.io.IOException +import java.util.concurrent.ExecutionException +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.ThreadPoolExecutor +import java.util.concurrent.TimeUnit + +// SAF tree listing for the library scan: child queries with provider-loading +// retries, readable tree resolution, listing prefetch and the ordered +// metadata read pool. + +// Provider round trips dominate on SD cards, USB drives and network shares. +private const val SAF_LIST_CONCURRENCY = 4 +private const val SAF_READ_WORKERS = 6 + +/** + * Prefetches the listings of directories already waiting in a breadth-first + * queue. The caller still dequeues, deduplicates and records errors serially, + * so traversal order and results match a one-directory-at-a-time walk. + * Idle daemon threads exit on their own, so abandoned walks leak nothing. + */ +internal class SafTreeLister(private val context: Context) { + private val executor = ThreadPoolExecutor( + SAF_LIST_CONCURRENCY, + SAF_LIST_CONCURRENCY, + 1, + TimeUnit.SECONDS, + LinkedBlockingQueue(), + ) { runnable -> Thread(runnable, "saf-scan-list").apply { isDaemon = true } } + .apply { allowCoreThreadTimeOut(true) } + private val prefetched = mutableMapOf>>>() + + private fun submit(dir: DocumentFile) = executor.submit>> { + try { + Result.success(context.listSafChildrenOrThrow(dir)) + } catch (e: Exception) { + Result.failure(e) + } + } + + fun list( + dir: DocumentFile, + queue: Collection>, + visited: Set, + ): Result> { + val uri = dir.uri.toString() + val current = prefetched.remove(uri) ?: submit(dir) + for ((next, _) in queue) { + if (prefetched.size >= SAF_LIST_CONCURRENCY * 2) break + val nextUri = next.uri.toString() + if (nextUri == uri || nextUri in visited || nextUri in prefetched) continue + prefetched[nextUri] = submit(next) + } + return try { + current.get() + } catch (e: ExecutionException) { + throw e.cause ?: e + } + } +} + +/** + * Runs [task] on a bounded pool and passes each result to [consume] in input + * order. A sliding window keeps workers busy past one slow file. Returns + * false when cancelled. + */ +internal fun runSafReadsInOrder( + items: List, + cancelled: () -> Boolean, + task: (T) -> R, + consume: (T, Result) -> Unit, +): Boolean { + if (items.isEmpty()) return true + val workers = minOf(SAF_READ_WORKERS, items.size) + val executor = Executors.newFixedThreadPool(workers) + try { + val window = workers * 4 + val inFlight = ArrayDeque>>(window) + var next = 0 + while (next < items.size || inFlight.isNotEmpty()) { + if (cancelled()) { + inFlight.forEach { it.second.cancel(true) } + return false + } + while (inFlight.size < window && next < items.size) { + val item = items[next++] + inFlight.addLast(item to executor.submit { task(item) }) + } + val (item, future) = inFlight.removeFirst() + val result = try { + Result.success(future.get()) + } catch (e: ExecutionException) { + Result.failure(e.cause ?: e) + } catch (e: Exception) { + Result.failure(e) + } + consume(item, result) + } + return true + } finally { + executor.shutdownNow() + } +} + +internal data class SafChildEntry( + val doc: DocumentFile, + val name: String?, + val isDirectory: Boolean, + val lastModified: Long, + val isFile: Boolean, +) + +internal fun Context.listSafChildrenOrThrow( + dir: DocumentFile, + includeLastModified: Boolean = true, +): List { + for (attempt in 0..2) { + try { + return querySafChildrenOrThrow(dir, includeLastModified) + } catch (error: SafProviderLoadingException) { + if (attempt == 2) throw error + Thread.sleep(300L * (attempt + 1)) + } + } + throw IOException("SAF directory listing did not complete") +} + +private class SafProviderLoadingException(uri: Uri) : + IOException("SAF provider is still loading $uri; retry the scan when it is ready") + +private fun Context.querySafChildrenOrThrow( + dir: DocumentFile, + includeLastModified: Boolean, +): List { + val childrenUri = DocumentsContract.buildChildDocumentsUriUsingTree( + dir.uri, + DocumentsContract.getDocumentId(dir.uri), + ) + val projection = mutableListOf( + DocumentsContract.Document.COLUMN_DOCUMENT_ID, + DocumentsContract.Document.COLUMN_DISPLAY_NAME, + DocumentsContract.Document.COLUMN_MIME_TYPE, + ) + if (includeLastModified) projection.add(DocumentsContract.Document.COLUMN_LAST_MODIFIED) + val cursor = try { + contentResolver.query(childrenUri, projection.toTypedArray(), null, null, null) + } catch (_: Exception) { + // A few older providers reject the richer projection; retry with IDs. + contentResolver.query( + childrenUri, + arrayOf(DocumentsContract.Document.COLUMN_DOCUMENT_ID), + null, + null, + null, + ) + } ?: throw IOException("SAF provider returned no cursor for ${dir.uri}") + return cursor.use { + // Network DocumentsProviders may return cached/partial rows while + // fetching a directory. Never commit that as an empty/full scan. + if (it.extras.getBoolean(DocumentsContract.EXTRA_LOADING, false)) { + throw SafProviderLoadingException(dir.uri) + } + val documentIdIndex = it.getColumnIndexOrThrow( + DocumentsContract.Document.COLUMN_DOCUMENT_ID, + ) + val displayNameIndex = it.getColumnIndex( + DocumentsContract.Document.COLUMN_DISPLAY_NAME, + ) + val mimeTypeIndex = it.getColumnIndex( + DocumentsContract.Document.COLUMN_MIME_TYPE, + ) + val lastModifiedIndex = it.getColumnIndex( + DocumentsContract.Document.COLUMN_LAST_MODIFIED, + ) + buildList { + while (it.moveToNext()) { + val childUri = DocumentsContract.buildDocumentUriUsingTree( + dir.uri, + it.getString(documentIdIndex), + ) + val child = DocumentFile.fromTreeUri(this@querySafChildrenOrThrow, childUri) + ?: throw IOException("Invalid SAF child URI: $childUri") + val name = if (displayNameIndex >= 0 && !it.isNull(displayNameIndex)) { + it.getString(displayNameIndex) + } else { + try { child.name } catch (_: Exception) { null } + } + val mimeType = if (mimeTypeIndex >= 0 && !it.isNull(mimeTypeIndex)) { + it.getString(mimeTypeIndex) + } else { + null + } + val isDirectory = if (mimeType != null) { + mimeType == MIME_TYPE_DIR + } else { + try { child.isDirectory } catch (_: Exception) { false } + } + val isFile = if (mimeType != null) { + mimeType.isNotEmpty() && !isDirectory + } else { + !isDirectory && try { child.isFile } catch (_: Exception) { false } + } + val lastModified = if (!includeLastModified) { + 0L + } else if ( + lastModifiedIndex >= 0 && !it.isNull(lastModifiedIndex) + ) { + it.getLong(lastModifiedIndex) + } else { + try { child.lastModified() } catch (_: Exception) { 0L } + } + add(SafChildEntry(child, name, isDirectory, lastModified, isFile)) + } + } + } + } + +internal fun Context.resolveReadableSafTreeOrThrow( + treeUriStr: String, + ): Triple> { + if (treeUriStr.isBlank()) { + throw IllegalArgumentException("SAF tree URI is empty") + } + val treeUri = Uri.parse(treeUriStr) + val root = DocumentFile.fromTreeUri(this, treeUri) + ?: throw IOException("Unable to resolve SAF tree") + val hasReadPermission = contentResolver.persistedUriPermissions.any { + it.uri == treeUri && it.isReadPermission + } || checkUriPermission( + root.uri, + android.os.Process.myPid(), + android.os.Process.myUid(), + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) == PackageManager.PERMISSION_GRANTED + if (!hasReadPermission) { + throw SecurityException("Read access to the SAF tree has been revoked") + } + // canRead() also requires a non-empty MIME type from queryDocument. + // Remote providers can omit that metadata after reconnecting even + // though queryChildDocuments can read the folder. The real listing is + // authoritative; reuse it in the scan instead of another round trip. + for (attempt in 0..2) { + val children = listSafChildrenOrThrow(root) + if (children.isNotEmpty() || root.exists()) { + return Triple(treeUri, root, children) + } + // An empty listing without a confirmed root is inconclusive, + // never evidence that previously indexed songs were deleted. + if (attempt < 2) Thread.sleep(300L * (attempt + 1)) + } + throw IOException("SAF provider could not confirm the library folder; reconnect it and retry") + }